@soulcraft/cor 3.0.0-rc.1

package/dist/hnsw/NativeDiskAnnWrapper.d.ts

@@ -0,0 +1,489 @@
+/**
+ * @module hnsw/NativeDiskAnnWrapper
+ * @description TypeScript wrapper around cor's native **Adaptive
+ * DiskANN** engine. Satisfies brainy 8.0's `VectorIndexProvider`
+ * contract (see [`providerContracts.ts`](../providerContracts.ts))
+ * so brainy treats it as the standard vector index slot —
+ * `addItem` / `search` / `rebuild` / `flush` / ... Underneath,
+ * every operation routes through cor's pure-Rust DiskANN engine
+ * (`native/diskann/`).
+ *
+ * @example
+ * ```typescript
+ * import { Brainy } from '@soulcraft/brainy'
+ *
+ * // Cor is auto-detected by brainy's plugin system on init.
+ * const brain = new Brainy({
+ *   storage: { type: 'filesystem', path: '/data/idx' }
+ * })
+ * await brain.init()  // [cor] Adaptive DiskANN engaged (mode auto)
+ *
+ * await brain.add({ data: 'native rust acceleration', type: 'concept' })
+ * const hits = await brain.search('billion scale ann', 10)
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Default zero-config — the adaptive selector picks Mode 1 / 2 / 3
+ * // from observed memory pressure. Most users only set `recall`.
+ * const brain = new Brainy({
+ *   storage: { type: 'filesystem', path: '/data/idx' },
+ *   vector: {
+ *     recall: 'balanced',  // 'fast' | 'balanced' | 'accurate'
+ *   }
+ * })
+ * ```
+ *
+ * ## Operating model
+ *
+ * Adaptive DiskANN is build-once, query-many: the on-disk file
+ * embeds the Vamana graph plus — depending on the build mode — an
+ * optional PQ codebook + codes section + the full vectors, all in
+ * a single contiguous mmap-able layout (see `native/diskann/src/format.rs`).
+ * Dynamic insertions go to a small in-memory **delta buffer** that
+ * brute-force-searches alongside the main index until the next
+ * `rebuild()` folds them in. This matches FreshDiskANN's published
+ * online-update model.
+ *
+ * The wrapper picks an operating mode (in-memory / hybrid /
+ * on-disk) at build time and open time via the
+ * {@link AdaptiveDiskAnnModeSelector}; see that module for the
+ * decision tree. Operators set nothing — observed memory drives
+ * the choice.
+ *
+ * ## Search path
+ *
+ * 1. Query the main index via the native DiskANN searcher:
+ *    - **Mode 1**: exact-distance walk over full vectors (no PQ,
+ *      no rerank — the walk's top-L is the exact top-L).
+ *    - **Mode 2/3**: PQ-greedy walk in RAM (codes pinned or paged
+ *      depending on mode), full-vector re-rank of the **whole walked
+ *      candidate list** by default (`paddingFactor` is an optional cap
+ *      for bounding cold-NVMe re-rank I/O — see {@link RECALL_PRESETS}).
+ * 2. Brute-force the delta buffer (typically <0.1% of total
+ *    size after a recent rebuild).
+ * 3. Merge + sort + truncate to `k`.
+ *
+ * ## When this wrapper engages
+ *
+ * The cor plugin (`src/plugin.ts`) registers this wrapper under
+ * brainy 8.0's canonical `'vector'` provider key. Whenever cor is
+ * installed, brainy's vector index slot resolves to Adaptive DiskANN.
+ */
+import type { Vector, VectorDocument, DistanceFunction, StorageAdapter } from '@soulcraft/brainy';
+import type { VectorIndexProvider } from '../providerContracts.js';
+import type { EntityIdMapperLike } from '../utils/columnStoreTypes.js';
+import type { MigrationCoordinator, MigrationStatus } from '../migration/MigrationCoordinator.js';
+import { type AdaptiveDiskAnnMode, type ModeSelection } from './AdaptiveDiskAnnModeSelector.js';
+/**
+ * **Public configuration surface** for the Adaptive DiskANN wrapper.
+ *
+ * The zero-config principle: the wrapper picks every algorithm-level
+ * tuning value (PQ subspaces, max degree, search list size, alpha
+ * pruning, mmap-vs-RAM adjacency) from observed runtime signals.
+ * Operators never tune the algorithm. The only knobs users touch are:
+ *
+ * 1. `dimensions` — required; can't be auto-detected
+ * 2. `recall` — the single quality/latency dial
+ * 3. `indexPath` — optional override; derived from storage by default
+ * 4. `mode` — explicit escape hatch for benchmarking / pinned-profile
+ *    deployments. The adaptive selector picks correctly for ~99% of
+ *    workloads.
+ */
+export interface DiskAnnIndexConfig {
+    /** Vector dimension (e.g. 384 for all-MiniLM-L6-v2). */
+    dimensions: number;
+    /**
+     * Output path for the on-disk DiskANN file. When omitted, the
+     * wrapper derives the path from `options.storage` using the
+     * canonical brainy 8.0 convention:
+     *
+     *   `{storage-root}/_system/vector-index/{shard}.dkann`
+     *
+     * For a single-shard brain, `{shard}` is `main`. Living under
+     * `_system/` means brainy's `db.persist(path)` and operator
+     * backup tools (`gsutil` / `rclone`) sweep the DiskANN file for
+     * free — no special inclusion rule needed. Explicit `indexPath`
+     * always wins (sharded deployments that want custom paths).
+     */
+    indexPath?: string;
+    /**
+     * **Recall preset** — the single quality/latency dial. Brainy 8.0
+     * exposes the same names on its JS HNSW path, so the user-facing
+     * trade-off is identical regardless of which vector index runs
+     * underneath.
+     *
+     * - `"fast"` — ~92-95% recall@10; lowest latency.
+     * - `"balanced"` (default) — ~96-98% recall@10.
+     * - `"accurate"` — ~99%+ recall@10; highest latency.
+     */
+    recall?: 'fast' | 'balanced' | 'accurate';
+    /**
+     * **Adaptive DiskANN mode escape hatch** (Piece I). Defaults to
+     * `'auto'`, which lets the selector pick `in-memory` / `hybrid` /
+     * `on-disk` from observed memory pressure
+     * (`/proc/meminfo` + V8 heap + active instances). Set to an
+     * explicit mode only when you have a specific reason — benchmarking,
+     * a pinned-profile deployment, etc.
+     */
+    mode?: 'auto' | AdaptiveDiskAnnMode;
+}
+/**
+ * @description **Scale-aware `l_search` (E3 zero-config adaptation).** The walk
+ * width a fixed default serves degrades with corpus size: recall@10 at
+ * `l_search=100` MEASURED 0.9942 @ 1M → 0.9385 @ 10M → **0.8747 @ 100M**
+ * (SIFT sweeps, bxl9000 2026-07-01: sift10m-recall + sift100m-nvme runs;
+ * summaries in `docs/verification/`). Holding ≥0.95 needed `l_search≈150` at
+ * 10M (0.9647) and `≈256` at 100M (0.9559) — i.e. the required width grows
+ * ~×1.8 per decade of corpus size. This resolver widens the *default* walk to
+ * match: `base` up to 2M nodes (where the measured recall is ≥0.99 already),
+ * then `base × 1.8^log10(n/2e6)`, capped at 512 (the latency guard; the cap is
+ * also where the 1B PROJECTION ends — 1B ≈ 489 stays a projection until the
+ * SIFT1B run measures it).
+ *
+ * Zero-config contract: this scales the DEFAULT only. An explicit
+ * `defaultLSearch` in the construction config pins the width verbatim (the
+ * escape hatch), and a `recall` preset scales proportionally from its base
+ * (`fast`/`accurate` keep their relative latency/recall intent at every scale).
+ *
+ * @param base - The preset/base walk width (e.g. 100 for 'balanced').
+ * @param nodeCount - Current corpus size (main index + delta).
+ * @returns The scale-adjusted walk width, in `[base, 512]` (never below base).
+ * @example
+ * scaleAwareLSearch(100, 1e6)  // 100  (measured 0.9942 — base is ample)
+ * scaleAwareLSearch(100, 1e7)  // 151  (measured anchor: 150 → 0.9647)
+ * scaleAwareLSearch(100, 1e8)  // 271  (measured anchor: 256 → 0.9559)
+ * scaleAwareLSearch(100, 1e9)  // 489  (PROJECTED — SIFT1B measures it)
+ */
+export declare function scaleAwareLSearch(base: number, nodeCount: number): number;
+export declare class NativeDiskAnnWrapper implements VectorIndexProvider {
+    private config;
+    private distanceFunction;
+    private storage;
+    private persistMode;
+    /** Live searcher instance — null until the first build. */
+    private native;
+    /** Newly added entries since the last build. Brute-force searched. */
+    private delta;
+    /** Removed entries — filtered out at search time. */
+    private tombstones;
+    /**
+     * **#72 Phase A+B — lazy resolver to the canonical entity-id mapper**
+     * (the metadata index's mapper, the SAME one brainy assigns entity
+     * ints with at write time). Wired by `src/plugin.ts` exactly as the
+     * graph index's `mapperResolver` (#68). The wrapper resolves
+     * `uuid → int` (slotmap writes, pushdown, membership) and `int → uuid`
+     * (search-result hydration) through it, which is why there is no
+     * longer a resident `slotByUuid` / `uuidBySlot` pair on the JS heap —
+     * the off-heap mmap slotmaps (`main.slotmap` / `main.slotrev`) plus
+     * this mapper replace them entirely (the 10B-RAM gate).
+     */
+    private mapperResolver?;
+    /**
+     * Lazy resolver to the shared #18 {@link MigrationCoordinator} (the same getter
+     * the metadata + graph providers receive). Undefined when no migration path is
+     * wired (e.g. a raw-napi construction in tests).
+     */
+    private readonly migrationCoordinatorGetter?;
+    /**
+     * True when the construction config carried an EXPLICIT `defaultLSearch` —
+     * the operator pinned the walk width, so {@link resolveLSearch} uses it
+     * verbatim instead of the scale-aware default (see {@link scaleAwareLSearch}).
+     * A `recall` preset does NOT pin: presets set the BASE and still scale with
+     * corpus size, preserving their relative recall/latency intent at every scale.
+     */
+    private lSearchPinned;
+    /**
+     * The in-flight fold (async rebuild) promise, or `null`. Concurrent
+     * `rebuild()` calls coalesce onto it, `addItem()`'s hard-cap backpressure
+     * awaits it, and `removeItem()` consults it so a delta-only removal that
+     * may already be baked into the incoming index still gets a tombstone
+     * (no resurrection at swap).
+     */
+    private rebuildInFlight;
+    /** In-flight L0 flush, or null (see {@link flushDeltaToL0}). */
+    private flushInFlight;
+    /** Live L0 segments, ascending by id (oldest first; search iterates newest→oldest). */
+    private l0Segments;
+    /** Next segment id (persisted in the manifest). */
+    private manifestNextId;
+    /** Segment-manifest cold-load memo (see {@link ensureSegments}). */
+    private segmentsLoaded;
+    private segmentsLoading;
+    /** Serializes manifest writes: the storage adapter has no per-key write
+     * ordering, so two in-flight saves (a flush racing a consolidation) could
+     * physically land out of order and persist a STALE segment list — losing a
+     * just-flushed segment's only reference after a restart. Each save chains
+     * behind the previous and builds its payload AT EXECUTION TIME. */
+    private manifestSaveChain;
+    /**
+     * **Piece I — Adaptive DiskANN.** The selector's most recent
+     * decision, exposed via {@link getLastModeSelection} for telemetry
+     * and tests. `null` until the first build or open.
+     */
+    private lastSelection;
+    constructor(config: DiskAnnIndexConfig & {
+        distanceFunction?: DistanceFunction;
+    }, distanceFunction: DistanceFunction, options?: {
+        storage?: StorageAdapter | null;
+        persistMode?: 'immediate' | 'deferred';
+        /**
+         * **#72 Phase A+B** — lazy resolver to the canonical entity-id
+         * mapper (the metadata index's mapper). Required for any operation
+         * that touches the main index (rebuild / search / removeItem):
+         * the wrapper resolves `slot ↔ uuid` through the off-heap mmap
+         * slotmaps + this mapper instead of resident JS maps. `src/plugin.ts`
+         * passes `() => metadataIndexInstance?.getIdMapper()`, mirroring the
+         * graph index's #68 wiring; the resolver is lazy so factory-call
+         * order between the providers doesn't matter.
+         */
+        mapperResolver?: () => EntityIdMapperLike | undefined;
+        /**
+         * **#18** — lazy resolver to the shared {@link MigrationCoordinator}. The
+         * plugin passes the SAME getter to all three derived-index providers so
+         * Brainy's feature-detected lock methods answer consistently across them.
+         */
+        migrationCoordinator?: () => MigrationCoordinator | undefined;
+    });
+    /**
+     * Resolve the canonical entity-id mapper, or throw if it isn't wired.
+     * Every main-index operation (rebuild, search-result hydration,
+     * pushdown, membership) resolves `slot ↔ uuid` through this mapper +
+     * the off-heap slotmaps; without it the wrapper cannot translate the
+     * native engine's entity ints to UUIDs. In production `src/plugin.ts`
+     * always wires it; the throw surfaces a misconfiguration loudly rather
+     * than silently returning empty results.
+     */
+    private mapper;
+    /**
+     * Append an entry to the delta buffer. Persisted by the next
+     * `rebuild()` call, which folds the delta into the main index.
+     */
+    addItem(item: VectorDocument): Promise<string>;
+    /**
+     * Mark an entry as removed. Filtered out at search time; physically
+     * removed at the next `rebuild()`.
+     */
+    removeItem(id: string): Promise<boolean>;
+    /**
+     * Hydrate the delta buffer from the canonical nouns persisted in
+     * storage — the cold-rebuild path used by {@link rebuild} after a
+     * `restore()` / migration, when the wrapper has no in-memory state to
+     * fold but storage holds the authoritative records. Streams via
+     * paginated `getNouns()` rather than fetching every noun at once.
+     *
+     * NOTE: like the existing delta path this materializes the vectors it
+     * loads into the JS delta. A billion-noun cold rebuild should instead
+     * stream straight from a storage-backed vector source (the native
+     * `build_from_file` path used by the SIFT billion-scale builder); that
+     * scale optimization is tracked separately. For the migration sizes
+     * this serves today the delta route is correct, and it logs the count
+     * it loaded so an operator can see the reconstruction happen.
+     *
+     * @param dim - Index dimension; nouns whose vector width differs are skipped.
+     */
+    private hydrateDeltaFromStorage;
+    search(queryVector: Vector, k?: number, filter?: (id: string) => Promise<boolean>, options?: {
+        rerank?: {
+            multiplier: number;
+        };
+        candidateIds?: string[];
+        /**
+         * Metadata∩graph universe (entity UUIDs) from the `find()`
+         * composition. When supplied and selective, it is PUSHED INTO the
+         * native beam walk (predicate pushdown, #10) so the walk collects only
+         * allowed slots — far higher recall than post-filtering the unfiltered
+         * top-k. Above {@link PUSHDOWN_MAX_SELECTIVITY} it is folded into the
+         * post-filter predicate instead (cheaper, and post-filter suffices).
+         */
+        allowedIds?: ReadonlySet<string>;
+    }): Promise<Array<[string, number]>>;
+    /**
+     * Predicate-pushdown search (#10). Translates the metadata∩graph entity
+     * universe to main-index slots, pushes them into the native beam walk
+     * (which traverses *through* all nodes but only collects allowed slots),
+     * and inflates `l_search` by ~1/selectivity so a selective predicate still
+     * surfaces k allowed neighbours. Delta-buffer entries (no slot yet) are
+     * brute-forced. This is the high-recall path for selective filtered
+     * `find()` — post-filtering the unfiltered top-k loses recall when few of
+     * those candidates pass the predicate.
+     *
+     * @param queryVector - The query embedding.
+     * @param k - Number of nearest allowed neighbours to return.
+     * @param allowedIds - Entity UUIDs permitted in the result (the find() universe).
+     * @param filter - Optional additional async predicate, composed with allowedIds.
+     * @param options - Optional rerank multiplier override.
+     * @returns Up to k `[uuid, distance]` pairs, ascending by distance.
+     */
+    private searchPushdown;
+    /**
+     * @description **#18 migration lock — feature-detected by Brainy.** `true` while
+     * the brain is undergoing (or has failed) the coordinated 7.x → 8.0 rebuild, so
+     * Brainy holds reads+writes behind the lock (and skips its lazy first-query
+     * force-rebuild of the vector index). Delegates to the shared {@link
+     * MigrationCoordinator}; `false` when none is wired.
+     * @returns `true` when the brain must not be read from or written to.
+     */
+    isMigrating(): boolean;
+    /**
+     * @description **#18 migration lock — structured progress** Brainy relays into
+     * `getIndexStatus().migration`. `null` unless a migration is in flight.
+     * @returns The current {@link MigrationStatus}, or `null`.
+     */
+    migrationStatus(): MigrationStatus | null;
+    /**
+     * @description Resolve the search-time walk width for this query: the
+     * scale-aware default ({@link scaleAwareLSearch} over the current corpus size)
+     * unless the operator pinned `defaultLSearch` explicitly, floored at `k × 2`
+     * so a large `k` is never starved. This is the E3 fix for the measured
+     * fixed-default recall fade (0.9942 @ 1M → 0.8747 @ 100M at l_search=100).
+     * @param k - The requested result count.
+     * @returns The walk width to pass to the native searcher.
+     */
+    /**
+     * **Adaptive DiskANN mode selection (Piece I)** — build the native config for
+     * an index of `totalCount` vectors: the selector picks "in-memory" for brains
+     * that fit in RAM (no PQ, sub-ms search), "hybrid" for medium scale, "on-disk"
+     * when memory is tight or many brains share the box. Shared by the full
+     * consolidation ({@link runRebuild}) and the O(delta) L0 segment flush
+     * ({@link runFlushToL0}); records the selection on {@link lastSelection}.
+     */
+    private nativeBuildCfg;
+    private resolveLSearch;
+    /** Total entries across the base index + all L0 segments (no delta). */
+    private indexedSize;
+    size(): number;
+    clear(): void;
+    /**
+     * Rebuild the main index from scratch: concatenate (current main −
+     * tombstones) ∪ delta, run a full DiskANN build, swap the searcher
+     * atomically.
+     *
+     * At billion-scale this is the expensive operation (hours-to-days of
+     * build time) — but it is NON-BLOCKING: the build runs on a worker
+     * thread (`rebuildFromExistingAsync`), so the JS event loop keeps
+     * serving reads and accepting writes for the entire fold. Writes that
+     * land mid-fold stay in the live delta (reconciled against the fold's
+     * snapshot at swap) and roll into the next fold. Concurrent calls
+     * coalesce onto the in-flight fold. The delta is bounded: a soft
+     * threshold kicks a background fold automatically and a hard cap
+     * applies backpressure in `addItem` (see `DELTA_SOFT_FOLD_BYTES`).
+     *
+     * The optional `recall` argument overrides the wrapper's
+     * construction-time recall preset for this rebuild only — useful
+     * when an operator wants to ship a higher-quality index after a
+     * data-quality push but kept the brain on `'balanced'` originally.
+     * Omit to keep the wrapper's current preset.
+     */
+    rebuild(options?: {
+        recall?: 'fast' | 'balanced' | 'accurate';
+    }): Promise<void>;
+    /** The fold body — see {@link rebuild} for the public contract. */
+    private runRebuild;
+    /**
+     * Flush the delta buffer to disk. For DiskANN the delta is in-memory
+     * by design (a few MB at most between rebuilds); returns the buffer
+     * size for parity with HNSW's flush contract.
+     */
+    flush(): Promise<number>;
+    getPersistMode(): 'immediate' | 'deferred';
+    /** Absolute path of a segment file (lives next to the base indexPath). */
+    private segPath;
+    /** Membership across the base + every L0 segment (int → any slot). */
+    private hasIntAnywhere;
+    /**
+     * @description Provider init — brainy feature-detects and awaits it. Cold-loads
+     * the segment manifest so a reopened brain serves its L0 segments from the
+     * first query (the wrapper's own async paths also call {@link ensureSegments},
+     * so a caller that skips init() still gets the segments lazily).
+     */
+    init(): Promise<void>;
+    /** Memoized segment-manifest cold-load. */
+    private ensureSegments;
+    /**
+     * Read `__vector_index_manifest__` and open every live L0 segment. A missing
+     * manifest is the legacy single-index layout (the base at `indexPath` IS the
+     * whole vector index) — existing brains upgrade seamlessly; the first flush
+     * writes the first manifest. A segment file the manifest references but that
+     * fails to open is SKIPPED loudly (crash-orphan tolerance; its entries remain
+     * safe in canonical storage and the next consolidation heals the set).
+     */
+    private loadSegmentManifest;
+    /**
+     * Persist the live segment set (AFTER the referenced files exist on disk).
+     * Saves are SERIALIZED and each snapshot the segment list at write time, so
+     * concurrent flush/consolidation saves can never persist out of order.
+     */
+    private saveSegmentManifest;
+    private writeSegmentManifestNow;
+    /**
+     * @description **#73 MVP-C — flush the delta into a new immutable L0 segment.**
+     * O(new-data): builds a small self-contained `.dkann` from the delta snapshot
+     * alone (existing index untouched), registers it in the manifest, and
+     * reconciles the live delta. Concurrent calls coalesce. Crash ordering: the
+     * segment file is fully written (with its slotmap sidecars) BEFORE the
+     * manifest references it.
+     */
+    private flushDeltaToL0;
+    /** The flush body — see {@link flushDeltaToL0}. */
+    private runFlushToL0;
+    private tryOpenExisting;
+    /**
+     * Remove any leftover `main.slots.json` written by a pre-#72 RC build.
+     * The native `main.slotmap` is now authoritative; a stale JSON sidecar
+     * is never read, but we delete it (best-effort) so it can never shadow
+     * the off-heap slotmap and to reclaim its (potentially large) bytes.
+     */
+    private removeLegacySlotsSidecar;
+    /**
+     * **Piece I.** Resolve the open-time mode for an existing file.
+     * Respects an explicit `config.mode` override; otherwise consults
+     * the adaptive selector.
+     */
+    private resolveOpenMode;
+    /**
+     * **Piece I.** Resolve the build-time mode. Same override-respecting
+     * shape as {@link resolveOpenMode}; the build-time decision
+     * additionally lets the selector choose Mode 1 (no PQ) when the
+     * brain fits in RAM.
+     */
+    private resolveBuildMode;
+    /**
+     * **Piece I.** The selector's most recent decision (either at the
+     * last `tryOpenExisting` or `rebuild`). Returns `null` if no
+     * brain has been opened or built yet. Surface for telemetry and
+     * tests.
+     */
+    getLastModeSelection(): ModeSelection | null;
+    /**
+     * **Test-only hook** for exercising the build-adjacency resolver
+     * without spinning up a real build. Mirrors the
+     * `_testInjectOsMemoryProbe` pattern (Piece J); not part of the
+     * public contract.
+     * @internal
+     */
+    _testResolveAdjacencyBackend(nodeCount: number): {
+        kind: 'mmap';
+        mmapPath: string;
+    } | {
+        kind: 'ram';
+    };
+    /**
+     * Resolve the build-time adjacency backend from the config plus
+     * the actual node count. `useMmapAdjacency: 'auto'` (the default)
+     * picks file-backed once `nodeCount > MMAP_ADJACENCY_AUTO_THRESHOLD`;
+     * explicit `true` / `false` always wins. Per the zero-config
+     * principle, operators don't have to know the 100 M threshold —
+     * the wrapper observes `totalCount` and flips automatically.
+     */
+    private resolveAdjacencyBackend;
+    /**
+     * Count tombstoned entities that are present in the MAIN index (so
+     * `size()` can subtract them). Membership is the off-heap reverse
+     * slotmap: resolve uuid → int → `hasInt`. Returns 0 without touching
+     * the mapper when there are no tombstones or no main index, so the
+     * common `size()` call stays allocation- and FFI-free.
+     */
+    private countMainTombstones;
+}
+//# sourceMappingURL=NativeDiskAnnWrapper.d.ts.map