@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js

@@ -1,12 +1,30 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HierarchicalNavigableSmallWorld = void 0;
+exports.derivedValuesEqual = derivedValuesEqual;
+const node_fs_1 = require("node:fs");
 const vector_ts_1 = require("./vector.js");
 const msgpackr_1 = require("msgpackr");
 const logger_ts_1 = require("../../utility/logging/logger.js");
 const hdbError_ts_1 = require("../../utility/errors/hdbError.js");
+const hnswDerivedIndex_ts_1 = require("./hnswDerivedIndex.js");
+const derivedIndexRuntime_ts_1 = require("../derivedIndexRuntime.js");
 const extended_iterable_1 = require("@harperfast/extended-iterable");
+const rocksdb_js_1 = require("@harperfast/rocksdb-js");
+const node_crypto_1 = require("node:crypto");
+const hnswPlaneBinding_ts_1 = require("./hnswPlaneBinding.js");
 const logger = (0, logger_ts_1.loggerWithTag)('HNSW');
+/** Element-wise equality of the array-like projections a derived index stores; anything else that failed `===` counts as changed. */
+function derivedValuesEqual(a, b) {
+    if (a === b)
+        return true;
+    if (a == null || b == null || typeof a.length !== 'number' || a.length !== b.length)
+        return false;
+    for (let i = 0; i < a.length; i++)
+        if (a[i] !== b[i])
+            return false;
+    return true;
+}
 // int8 scalar quantization of stored graph nodes is ON by default. Each node holds the
 // vector as a compact int8 `bin` plus a per-vector `scale`, roughly a 5x size reduction
 // over float32 and ~10x cheaper to decode (a single typed-array view instead of decoding
@@ -85,6 +103,10 @@ const ROUTING_EF = 1;
 // index's own auto-scaled ceiling so the worst case stays the same order. Schema and per-query ef
 // pins are authoritative cost ceilings; only an automatically scaled index widens from `limit`.
 const LIMIT_EF_MAX = 2 * AUTO_EF_CEILING;
+// Rerank candidates per row a bounded nearest-neighbor query consumes, with a floor: each
+// candidate costs a record load and an exact distance, so the rerank must not scale with ef.
+const RERANK_FACTOR = 4;
+const RERANK_MIN = 32;
 // Auto-scaled construction ef, used only when an index does not explicitly configure efConstruction.
 // At a constant efConstruction, edge quality erodes as the graph grows until true neighbours become
 // unreachable at ANY search ef; the cap bounds per-insert cost. Measurements and policy in
@@ -103,6 +125,60 @@ function autoScaleEfConstruction(nodeCount) {
 // ef moves with the square root of the count and is capped, so a slightly stale size is immaterial;
 // this only has to be short enough that a table growing from empty picks up a larger ef promptly.
 const NODE_COUNT_TTL = 10_000;
+// Native traversal-plane geometry (DESIGN.md § Native HNSW plane). The layer-0 cap is derived from
+// M/optimizeRouting at creation to cover the JS graph's effective cap (grace overshoot above it
+// truncates by distance); a configuration deriving past this ceiling is refused as ineligible
+// rather than silently truncated. maxNodes is a fixed sparse reservation — pages materialize on
+// write — and ids at or past it are rejected by the crate, which disables the plane.
+const PLANE_LAYER0_CAP_MAX = 1024;
+const PLANE_MAX_NODES = 1 << 24;
+// Default inline primary-key bytes per plane slot (msgpack-encoded); 40 fits UUIDs inside the slot
+// padding. `nativePlaneKeyCap` raises it for tables whose keys are longer.
+const PLANE_KEY_CAP = 40;
+const PLANE_KEY_CAP_MAX = 65535;
+// An existing plane file that cannot be opened is normally another worker mid-create (retry);
+// past this age it is a crashed create and is deleted so the audit-backed runtime can rebuild it.
+const PLANE_STALE_CREATE_MS = 60_000;
+// Retry cadence while another worker holds the create; its header lands shortly after exclusive open.
+const PLANE_ATTACH_RETRY_MS = 250;
+// Marks an error thrown by an app-supplied filter during a plane search: the caller re-raises
+// it as an ordinary query failure instead of disabling the (healthy) plane.
+const NOT_A_PLANE_FAILURE = Symbol('notAPlaneFailure');
+function numericOption(name, value) {
+    if (value === undefined || value === null)
+        return undefined;
+    if ((typeof value !== 'number' && typeof value !== 'string') || (typeof value === 'string' && value.trim() === ''))
+        throw new hdbError_ts_1.ClientError(`${name} must be a finite number`);
+    const numericValue = Number(value);
+    if (!Number.isFinite(numericValue))
+        throw new hdbError_ts_1.ClientError(`${name} must be a finite number`);
+    return numericValue;
+}
+function normalizeOptimizeRoutingDeclaration(value) {
+    if (value === true || value === 'true')
+        return 1;
+    if (value === false || value === 'false')
+        return 0;
+    return value;
+}
+function nativePlaneMaxNodes(options) {
+    const maxNodes = numericOption('nativePlaneMaxNodes', options?.nativePlaneMaxNodes) ?? PLANE_MAX_NODES;
+    if (!Number.isSafeInteger(maxNodes) || maxNodes < 1 || maxNodes >= hnswPlaneBinding_ts_1.PLANE_NO_ID) {
+        throw new hdbError_ts_1.ClientError('nativePlaneMaxNodes must be a positive integer below 2^32-1');
+    }
+    return maxNodes;
+}
+function nativePlaneKeyCap(options) {
+    const keyCap = numericOption('nativePlaneKeyCap', options?.nativePlaneKeyCap) ?? PLANE_KEY_CAP;
+    if (!Number.isSafeInteger(keyCap) || keyCap < 8 || keyCap > PLANE_KEY_CAP_MAX) {
+        throw new hdbError_ts_1.ClientError('nativePlaneKeyCap must be an integer between 8 and 65535');
+    }
+    return keyCap;
+}
+function nativePlaneDefaultDisabled() {
+    const value = process.env.HNSW_NO_NATIVE_DEFAULT;
+    return value != null && value !== '' && value !== '0' && value.toLowerCase() !== 'false';
+}
 class MinHeap {
     data = [];
     get size() {
@@ -163,6 +239,10 @@ function bisectInsert(arr, distance) {
  */
 const ENTRY_POINT = Symbol.for('entryPoint');
 const KEY_PREFIX = Symbol.for('key');
+/** A Map key for a primary key: composite keys by their encoding, since arrays compare by reference. */
+function pendingMappingKey(primaryKey) {
+    return Array.isArray(primaryKey) ? `\0${(0, msgpackr_1.pack)(primaryKey).toString('latin1')}` : primaryKey;
+}
 const MAX_LEVEL = 10; // should give good high-level skip list performance up to trillions of nodes
 // Visit budget for the post-delete connectivity probe (repairSeveredNeighbors, #1712). Severed
 // islands are small (bounded by the deleted node's neighborhood), so a probe that visits this many
@@ -180,11 +260,131 @@ const PROBE_VISIT_LIMIT = 256;
  */
 class HierarchicalNavigableSmallWorld {
     static useObjectStore = true;
+    static numericOptions = new Set([
+        'M',
+        'efConstruction',
+        'efConstructionSearch',
+        'mL',
+        'optimizeRouting',
+        'filterExpansion',
+        'nativePlaneMaxNodes',
+        'nativePlaneKeyCap',
+        'maxLagMilliseconds',
+    ]);
+    static truthyStructuralOptions = new Set(['nativePlane']);
+    static normalizeOptionValue(name, value) {
+        if (name !== 'optimizeRouting')
+            return value;
+        if (value === true)
+            return 1;
+        if (value === false)
+            return 0;
+        if (typeof value === 'string' &&
+            (value === 'true' || value === 'false' || (value.trim() !== '' && Number(value) === 0)))
+            // Legacy strings are truthy at runtime; keep them structurally distinct from false/0 so #1357 rebuilds.
+            return `legacy:${value}`;
+        return value;
+    }
+    static normalizeDeclarationOptions(options, persistedOptions) {
+        for (const name of HierarchicalNavigableSmallWorld.numericOptions) {
+            if (options[name] === undefined)
+                continue;
+            if (persistedOptions && Object.hasOwn(persistedOptions, name) && Object.is(options[name], persistedOptions[name]))
+                continue;
+            if (name === 'optimizeRouting' &&
+                typeof persistedOptions?.[name] === 'string' &&
+                options[name] !== false &&
+                options[name] !== 'false') {
+                try {
+                    const declared = numericOption(name, normalizeOptimizeRoutingDeclaration(options[name]));
+                    const persisted = numericOption(name, normalizeOptimizeRoutingDeclaration(persistedOptions[name]));
+                    if (Object.is(declared, persisted)) {
+                        options[name] = persistedOptions[name];
+                        continue;
+                    }
+                }
+                catch { }
+            }
+            if (options[name] === null) {
+                delete options[name];
+                continue;
+            }
+            const value = name === 'optimizeRouting' ? normalizeOptimizeRoutingDeclaration(options[name]) : options[name];
+            options[name] = numericOption(name, value);
+        }
+        if (Object.hasOwn(options, 'nativePlaneMaxNodes'))
+            nativePlaneMaxNodes(options);
+        if (Object.hasOwn(options, 'nativePlaneKeyCap'))
+            nativePlaneKeyCap(options);
+    }
+    static normalizeNativePlaneDeclaration(value) {
+        if (value === undefined || value === null)
+            return undefined;
+        if (value === true || value === 'true')
+            return true;
+        if (value === false || value === 'false')
+            return false;
+        throw new hdbError_ts_1.ClientError('nativePlane must be true or false');
+    }
+    static validateNativePlaneOptions(rootStore, options) {
+        if (!(rootStore instanceof rocksdb_js_1.RocksDatabase)) {
+            throw new hdbError_ts_1.ClientError('nativePlane requires the RocksDB storage engine; set nativePlane: false to use the JS index');
+        }
+        const nativeM = numericOption('M', options?.M);
+        const nativeEfConstruction = numericOption('efConstruction', options?.efConstruction);
+        const nativeML = numericOption('mL', options?.mL);
+        const nativeOptimizeRouting = numericOption('optimizeRouting', normalizeOptimizeRoutingDeclaration(options?.optimizeRouting));
+        if (options?.M === null ||
+            options?.efConstruction === null ||
+            options?.mL === null ||
+            options?.optimizeRouting === null ||
+            (nativeM !== undefined && nativeM !== 16) ||
+            (nativeEfConstruction !== undefined && nativeEfConstruction !== 200) ||
+            (nativeML !== undefined && nativeML !== 1 / Math.log(16)) ||
+            (nativeOptimizeRouting !== undefined && nativeOptimizeRouting !== 0.5)) {
+            throw new hdbError_ts_1.ClientError('nativePlane requires M=16, efConstruction=200, mL=1/ln(16), and optimizeRouting=0.5; set nativePlane: false to use the JS index');
+        }
+        const maxNodes = nativePlaneMaxNodes(options);
+        const keyCap = nativePlaneKeyCap(options);
+        if (options?.quantization === 'none' || options?.distance === 'euclidean' || options?.distance === 'dotProduct') {
+            throw new hdbError_ts_1.ClientError('nativePlane requires an int8-quantized cosine HNSW index; set nativePlane: false to use the JS index');
+        }
+        return { nativeM, nativeEfConstruction, nativeML, nativeOptimizeRouting, maxNodes, keyCap };
+    }
+    static canRunNativePlane(rootStore, options, warnForMissingBinding = true) {
+        nativePlaneMaxNodes(options);
+        nativePlaneKeyCap(options);
+        if (!(rootStore instanceof rocksdb_js_1.RocksDatabase) || (0, hnswPlaneBinding_ts_1.getPlaneBinding)(warnForMissingBinding) == null)
+            return false;
+        if (options?.M === null ||
+            options?.efConstruction === null ||
+            options?.mL === null ||
+            options?.optimizeRouting === null)
+            return false;
+        const configuredM = numericOption('M', options?.M);
+        const configuredEfConstruction = numericOption('efConstruction', options?.efConstruction);
+        const configuredML = numericOption('mL', options?.mL);
+        const configuredOptimizeRouting = numericOption('optimizeRouting', normalizeOptimizeRoutingDeclaration(options?.optimizeRouting));
+        const baseEligible = options?.quantization !== 'none' &&
+            options?.distance !== 'euclidean' &&
+            options?.distance !== 'dotProduct' &&
+            (configuredM === undefined || configuredM === 16) &&
+            (configuredEfConstruction === undefined || configuredEfConstruction === 200) &&
+            (configuredML === undefined || configuredML === 1 / Math.log(16)) &&
+            (configuredOptimizeRouting === undefined || configuredOptimizeRouting === 0.5);
+        if (!baseEligible)
+            return false;
+        return true;
+    }
+    static canDefaultToNativePlane(rootStore, options) {
+        return (!nativePlaneDefaultDisabled() && HierarchicalNavigableSmallWorld.canRunNativePlane(rootStore, options, false));
+    }
     // Index options that only affect search, not the stored graph — changing them must not trigger a
     // reindex (databases.ts persists the new value but skips rebuilding). efConstructionSearch is the
     // search-time candidate-list size; the build uses efConstruction/M/distance, which are structural.
-    // filterExpansion is the visit-budget multiplier for predicate-aware (filtered) traversal.
-    static searchOnlyOptions = ['efConstructionSearch', 'filterExpansion'];
+    // filterExpansion is the visit-budget multiplier for predicate-aware (filtered) traversal;
+    // maxLagMilliseconds controls delivery admission rather than the graph itself.
+    static searchOnlyOptions = ['efConstructionSearch', 'filterExpansion', 'maxLagMilliseconds'];
     // Signals to search.ts that this index accepts a per-record predicate in search() and applies it
     // during traversal (predicate-aware / ACORN-style filtering), so companion conditions and RBAC can
     // be pushed down instead of post-filtering an under-filled candidate set (#1241).
@@ -198,6 +398,10 @@ class HierarchicalNavigableSmallWorld {
     // a value of 1 is extremely aggressive.
     optimizeRouting = 0.5;
     nodesVisitedCount = 0;
+    // Test seam: source of randomness for level assignment. Tests that assert exact result-set
+    // equality across search strategies replace this with a seeded PRNG so the graph shape is
+    // reproducible; greedy routing legitimately diverges on rare unlucky graphs otherwise.
+    random = Math.random;
     // Visit-budget multiplier for predicate-aware traversal (#1241). Under-filled filtered searches
     // stop after the resolved budget ef * filterExpansion visits; automatic search ef contributes at
     // most AUTO_EF_MAX, while explicit schema/query ef remains authoritative. A filter that fills its
@@ -219,6 +423,22 @@ class HierarchicalNavigableSmallWorld {
     convertedNodes = new WeakMap();
     nodeCount = 0;
     nodeCountAt = 0;
+    // Native file-primary index. The RocksDB index store holds identity mappings and replay cursors;
+    // graph nodes and adjacency exist only in this file.
+    // undefined = not yet attached (may retry), null = unavailable or disabled for this process.
+    plane;
+    planeEligible = false;
+    planeRetryAt = 0;
+    planeDisabledLogged = false;
+    filePrimary = false;
+    nativePlaneMaxNodes = PLANE_MAX_NODES;
+    nativePlaneKeyCap = PLANE_KEY_CAP;
+    // Installed by attachDerivedIndexes on every worker: shared readiness of the index and the way
+    // to ask its owner for a rebuild. Only the owning worker's runtime ever destroys native state.
+    derivedHost;
+    // keyed by pendingMappingKey(pk): a composite key's arrays never compare equal by reference
+    pendingDerivedMappings = new Map();
+    postCommit;
     constructor(indexStore, options) {
         this.indexStore = indexStore;
         if (indexStore) {
@@ -226,10 +446,16 @@ class HierarchicalNavigableSmallWorld {
             // (we would actually like to use float16 if it were available)
             this.indexStore.encoder.useFloat32 = msgpackr_1.FLOAT32_OPTIONS.ALWAYS;
         }
+        const configuredM = options?.M;
+        const configuredEfConstruction = options?.efConstruction;
+        const configuredEfConstructionSearch = options?.efConstructionSearch;
+        const configuredML = options?.mL;
+        const configuredOptimizeRouting = options?.optimizeRouting;
+        const configuredFilterExpansion = options?.filterExpansion;
         this.int8 = options?.quantization !== 'none';
         // Respect an explicitly-configured ef (efConstruction seeds the search ef too); otherwise auto-scale both.
-        this.efSearchConfigured = options?.efConstructionSearch !== undefined || options?.efConstruction !== undefined;
-        this.efConstructionConfigured = options?.efConstruction !== undefined;
+        this.efSearchConfigured = configuredEfConstructionSearch !== undefined || configuredEfConstruction !== undefined;
+        this.efConstructionConfigured = configuredEfConstruction !== undefined;
         this.distance =
             options?.distance === 'euclidean'
                 ? vector_ts_1.euclideanDistance
@@ -238,23 +464,499 @@ class HierarchicalNavigableSmallWorld {
                     : vector_ts_1.cosineDistance;
         if (options) {
             // allow all the HNSW parameters to be configured/tuned
-            if (options.M !== undefined) {
-                this.M = options.M;
+            if (configuredM !== undefined) {
+                this.M = configuredM;
                 this.mL = 1 / Math.log(this.M); // recalculate
             }
-            if (options.efConstruction !== undefined)
-                this.efConstruction = this.efConstructionSearch = options.efConstruction;
-            if (options.efConstructionSearch !== undefined)
-                this.efConstructionSearch = options.efConstructionSearch;
-            if (options.mL !== undefined)
-                this.mL = options.mL;
-            if (options.optimizeRouting !== undefined)
-                this.optimizeRouting = options.optimizeRouting;
-            if (options.filterExpansion !== undefined)
-                this.filterExpansion = options.filterExpansion;
+            if (configuredEfConstruction !== undefined)
+                this.efConstruction = this.efConstructionSearch = configuredEfConstruction;
+            if (configuredEfConstructionSearch !== undefined)
+                this.efConstructionSearch = configuredEfConstructionSearch;
+            if (configuredML !== undefined)
+                this.mL = configuredML;
+            if (configuredOptimizeRouting !== undefined)
+                this.optimizeRouting = configuredOptimizeRouting;
+            if (configuredFilterExpansion !== undefined)
+                this.filterExpansion = configuredFilterExpansion;
+        }
+        if (options?.nativePlane) {
+            const { nativeM, nativeEfConstruction, nativeML, nativeOptimizeRouting, maxNodes, keyCap } = HierarchicalNavigableSmallWorld.validateNativePlaneOptions(indexStore?.rootStore, options);
+            if (nativeM !== undefined)
+                this.M = nativeM;
+            if (nativeML !== undefined)
+                this.mL = nativeML;
+            if (nativeOptimizeRouting !== undefined)
+                this.optimizeRouting = nativeOptimizeRouting;
+            this.efConstruction = nativeEfConstruction ?? 200;
+            this.nativePlaneMaxNodes = maxNodes;
+            this.nativePlaneKeyCap = keyCap;
+            // The plane stores int8 bins and computes asymmetric cosine only, so the flag is a
+            // no-op for float (quantization: "none") and non-cosine indexes; a graph whose derived
+            // layer-0 cap exceeds the plane maximum is refused rather than silently truncated.
+            this.planeEligible =
+                this.int8 && this.distance === vector_ts_1.cosineDistance && this.planeLayer0Cap() <= PLANE_LAYER0_CAP_MAX;
+            if (!this.planeEligible) {
+                throw new hdbError_ts_1.ClientError('nativePlane requires an int8-quantized cosine HNSW index; set nativePlane: false to use the JS index');
+            }
+            this.filePrimary = true;
+            this.postCommit = true;
+        }
+    }
+    /** Remove derived native state after the option is disabled. */
+    cleanupDisabledPlane() {
+        if (this.planeEligible)
+            return;
+        const filePath = this.planeFilePath();
+        if (!filePath)
+            return;
+        try {
+            if ((0, node_fs_1.existsSync)(filePath))
+                this.invalidatePlaneFile(filePath, this.plane);
+            (0, node_fs_1.unlinkSync)(filePath);
+            logger.info?.('deleted the HNSW plane file of an index no longer using nativePlane');
+        }
+        catch (error) {
+            if (error?.code !== 'ENOENT') {
+                // A later re-enable must not adopt a file that missed mutations while disabled.
+                logger.warn?.('could not delete the HNSW plane file; marking it stale', error);
+                this.invalidatePlaneFile(filePath);
+            }
+        }
+    }
+    /** Absolute path of this index's plane file, or undefined when the store exposes no path. */
+    planeFilePath() {
+        const storePath = this.indexStore?.path;
+        const storeName = this.indexStore?.name;
+        if (typeof storePath !== 'string' || typeof storeName !== 'string')
+            return undefined;
+        return (0, hnswPlaneBinding_ts_1.planeFilePathFor)(storePath, storeName);
+    }
+    /**
+     * Open or lazily create the native file. An exclusive create resolves multi-worker races; the
+     * derived-index runtime owns population, replay, and publication.
+     */
+    getPlane(dims, dimsFromVector = false) {
+        if (this.plane !== undefined)
+            return this.plane;
+        if (!this.planeEligible)
+            return (this.plane = null);
+        const now = Date.now();
+        if (now < this.planeRetryAt)
+            return null;
+        const Plane = (0, hnswPlaneBinding_ts_1.getPlaneBinding)();
+        if (!Plane)
+            return (this.plane = null); // the loader warned once already
+        if (!('keyCap' in Plane.prototype)) {
+            if (!this.planeDisabledLogged) {
+                this.planeDisabledLogged = true;
+                logger.error?.('the installed @harperfast/hnsw predates 0.3.0; the native HNSW index needs 0.3.0 or later');
+            }
+            return (this.plane = null);
+        }
+        const filePath = this.planeFilePath();
+        if (!filePath) {
+            this.disablePlane(new Error('the index store exposes no path to place the plane file next to'));
+            return null;
+        }
+        try {
+            // a tombstone marks a plane a previous unlink could not remove (Windows EBUSY while
+            // mapped): the file is stale and must never be opened over a fresh graph
+            const stalePath = (0, hnswPlaneBinding_ts_1.planeStalePathFor)(filePath);
+            if ((0, node_fs_1.existsSync)(stalePath)) {
+                try {
+                    // force: either artifact may already be gone (the documented rollback deletes the
+                    // plane file by hand), and an ENOENT here disables the plane on every later attach
+                    (0, node_fs_1.rmSync)(filePath, { force: true });
+                    (0, node_fs_1.rmSync)(stalePath, { force: true });
+                }
+                catch {
+                    this.planeRetryAt = now + NODE_COUNT_TTL;
+                    return null;
+                }
+            }
+            if ((0, node_fs_1.existsSync)(filePath)) {
+                try {
+                    // Crash recovery is per-slot inside the crate. The clean flag is advisory;
+                    // another worker may still be constructing this shared file.
+                    const opened = Plane.open(filePath);
+                    if (this.filePrimary && opened.keyCap !== this.nativePlaneKeyCap) {
+                        // created under another nativePlaneKeyCap: rebuild to the configured layout
+                        opened.invalidateFile();
+                        throw new Error(`plane keyCap ${opened.keyCap} differs from the configured ${this.nativePlaneKeyCap}`);
+                    }
+                    return (this.plane = opened);
+                }
+                catch (openError) {
+                    if (now - (0, node_fs_1.statSync)(filePath).mtimeMs <= PLANE_STALE_CREATE_MS) {
+                        // another worker is between its exclusive create and the header write
+                        this.planeRetryAt = now + PLANE_ATTACH_RETRY_MS;
+                        return null;
+                    }
+                    logger.warn?.('deleting an unopenable HNSW plane file left by an interrupted create', openError);
+                    (0, node_fs_1.unlinkSync)(filePath);
+                    if (this.filePrimary) {
+                        // The surviving mappings still name node ids from the file just removed.
+                        // Creating a fresh plane here would resolve them against an empty graph;
+                        // only reconstruction, which clears the mappings first, is safe.
+                        this.planeRetryAt = now + PLANE_ATTACH_RETRY_MS;
+                        return null;
+                    }
+                }
+            }
+            if (!dims)
+                return null; // open-only call and no file: nothing to attach yet
+            // A search target must not pin an empty index's dimensionality: creation is deferred to
+            // the first committed vector or to the rebuild scan. Return before the exclusive create
+            // rather than creating and unlinking — a concurrent insert that saw the empty file would
+            // read it as another worker's in-progress create and 503 for PLANE_STALE_CREATE_MS.
+            if (!dimsFromVector)
+                return null;
+            let fd;
+            try {
+                fd = (0, node_fs_1.openSync)(filePath, 'wx');
+            }
+            catch {
+                // another worker won the create race; its header lands within moments
+                this.planeRetryAt = now + PLANE_ATTACH_RETRY_MS;
+                return null;
+            }
+            (0, node_fs_1.closeSync)(fd);
+            try {
+                return (this.plane = Plane.create(filePath, dims, this.planeLayer0Cap(), this.nativePlaneMaxNodes, this.nativePlaneKeyCap));
+            }
+            catch (createError) {
+                // Never leave a partial file that a later process could trust as current.
+                try {
+                    (0, node_fs_1.unlinkSync)(filePath);
+                }
+                catch {
+                    // the disable below already forces the JS path for this process
+                }
+                this.disablePlane(createError);
+                return null;
+            }
+        }
+        catch (error) {
+            this.planeRetryAt = now + NODE_COUNT_TTL;
+            logger.warn?.('could not attach the HNSW plane file; will retry', error);
+            return null;
+        }
+    }
+    /** True only while the owning runtime publishes the index as ready, on whichever worker owns it. */
+    planeSearchReady(plane) {
+        if (plane.invalidated()) {
+            this.plane = undefined;
+            return false;
         }
+        return this.derivedReadiness() === 'ready';
+    }
+    derivedReadiness() {
+        return this.derivedHost?.readiness().state ?? 'unknown';
+    }
+    /** The JS graph's effective layer-0 cap for this configuration; sizes the plane's slots. */
+    planeLayer0Cap() {
+        return this.optimizeRouting ? this.M << 3 : this.M << 1;
+    }
+    /**
+     * Detach this process from the plane after a failure and ask the index's owner for a rebuild.
+     * Only the owner destroys native state (`resetDerivedStorage` under its epoch), so a failure
+     * observed here after a peer has already replaced the file cannot take out the replacement.
+     */
+    disablePlane(error) {
+        this.plane = undefined;
+        this.planeRetryAt = Date.now() + PLANE_ATTACH_RETRY_MS;
+        if (!this.planeDisabledLogged) {
+            this.planeDisabledLogged = true;
+            logger.error?.('the HNSW native index failed; requesting a rebuild from its owner', error);
+        }
+        this.derivedHost?.requestRebuild();
+    }
+    /**
+     * Delete the derived plane state before an audit-backed reconstruction. Path invalidation
+     * makes peers stop using an old mapping even when unlink leaves their mmap inode alive.
+     */
+    resetDerivedStorage() {
+        this.pendingDerivedMappings.clear();
+        const attached = this.plane;
+        this.plane = undefined;
+        this.planeRetryAt = 0;
+        const filePath = this.planeFilePath();
+        if (!filePath)
+            return;
+        if (this.filePrimary && (0, node_fs_1.existsSync)(filePath))
+            this.invalidatePlaneFile(filePath, attached);
+        try {
+            (0, node_fs_1.unlinkSync)(filePath);
+        }
+        catch (error) {
+            if (error?.code !== 'ENOENT') {
+                // a stale file that cannot be deleted (e.g. Windows EBUSY while mapped) must not
+                // be reopened as if current — mark it so no process ever adopts it
+                this.plane = null;
+                logger.warn?.('could not delete the HNSW plane file; marking it stale', error);
+                this.invalidatePlaneFile(filePath, attached);
+            }
+        }
+    }
+    /** True while any primary-key mapping survives, which is the only proof this index holds nodes. */
+    hasNodeMappings() {
+        for (const { value } of this.indexStore.getRange({})) {
+            if (value && typeof value === 'object' && typeof value.id === 'number')
+                return true;
+        }
+        return false;
+    }
+    /** Make an undeletable plane unadoptable before this process releases it. */
+    invalidatePlaneFile(filePath, attached) {
+        try {
+            (0, hnswPlaneBinding_ts_1.invalidatePlaneFile)(filePath, attached);
+        }
+        catch (error) {
+            logger.warn?.('could not invalidate the stale HNSW plane file', error);
+        }
+    }
+    /**
+     * Native search over the plane: one NAPI crossing, traversal on the libuv pool, every hit
+     * carrying its primary key out of the slot. The predicate adapter runs on this thread's event
+     * loop (batched over a ThreadsafeFunction), so this promise must never be awaited by code the
+     * predicate itself blocks on; the normal request path awaits it safely.
+     */
+    searchPlane(plane, target, k, ef, filter, filterState) {
+        const query = Float32Array.from(target);
+        let resultPromise;
+        let predicateError;
+        let planeError;
+        if (filter && filterState) {
+            const predicate = (ids, keys, keyEnds) => {
+                const verdicts = new Uint8Array(ids.length);
+                if (predicateError !== undefined || planeError !== undefined)
+                    return verdicts;
+                for (let i = 0; i < ids.length; i++) {
+                    const start = i === 0 ? 0 : keyEnds[i - 1];
+                    const end = keyEnds[i];
+                    if (end === start)
+                        continue;
+                    let primaryKey;
+                    try {
+                        primaryKey = (0, msgpackr_1.unpack)(keys.subarray(start, end));
+                    }
+                    catch (error) {
+                        planeError = error;
+                        break;
+                    }
+                    try {
+                        if (this.admit(filter, filterState, primaryKey))
+                            verdicts[i] = 1;
+                    }
+                    catch (error) {
+                        predicateError = error;
+                        break;
+                    }
+                }
+                return verdicts;
+            };
+            resultPromise = plane.searchWithPredicate(query, k, ef, predicate, undefined, filterState.maxVisits);
+        }
+        else {
+            resultPromise = plane.search(query, k, ef);
+        }
+        return resultPromise.then((hits) => {
+            if (planeError !== undefined)
+                throw planeError;
+            if (predicateError !== undefined) {
+                // the plane itself is healthy; mark the failure as the application's so the caller
+                // re-raises it rather than disabling the plane and retrying
+                try {
+                    predicateError[NOT_A_PLANE_FAILURE] = true;
+                }
+                catch {
+                    // a frozen/primitive throw still propagates, it just also disables the plane
+                }
+                throw predicateError;
+            }
+            const entries = [];
+            const { distances, keys, keyEnds } = hits;
+            // a key appears twice only after a crash (see applyDerivedValue); keep the nearer hit
+            const seen = new Set();
+            for (let i = 0; i < keyEnds.length; i++) {
+                const start = i === 0 ? 0 : keyEnds[i - 1];
+                const end = keyEnds[i];
+                if (end === start)
+                    continue;
+                const encoded = keys.toString('latin1', start, end);
+                if (seen.has(encoded))
+                    continue;
+                seen.add(encoded);
+                entries.push({ key: (0, msgpackr_1.unpack)(keys.subarray(start, end)), distance: distances[i] });
+            }
+            // nodesVisited stays 0 here: layer-0 visits happen inside the native traversal
+            // (filterEvaluations is still counted by the predicate adapter)
+            return withStats(entries, filterState);
+        });
+    }
+    attachDerivedHost(host) {
+        this.derivedHost = host;
+    }
+    queryCoverage(maxLagMilliseconds) {
+        const coverage = this.derivedHost?.coverage(maxLagMilliseconds);
+        if (!coverage || coverage.state === 'unknown') {
+            const age = coverage?.lagUpperBoundMilliseconds;
+            throw new hdbError_ts_1.DerivedIndexLagError(`Cannot certify native HNSW index coverage within ${maxLagMilliseconds} ms` +
+                (age === undefined ? '; freshness is unknown' : `; last certified ${Math.ceil(age)} ms ago`) +
+                '; retry this query');
+        }
+        return coverage;
+    }
+    /**
+     * The commit path only validates: a malformed vector is the client's 400 here rather than an
+     * unindexable record later. The shared derived-index runtime reads the committed log; nothing
+     * is staged on the transaction.
+     */
+    prepareCommitted(primaryKey, vector, existingVector) {
+        // O(dims) on every write to the table, so skip it when the projection did not change — an
+        // unchanged vector was validated when it was first written.
+        if (derivedValuesEqual(vector, existingVector))
+            return;
+        this.validateVector(primaryKey, vector);
+    }
+    /** The runtime's projection guard: what fails here is delivered as unindexable, not applied. */
+    assertDerivedValue(vector, label) {
+        this.assertPlaneVector(vector, label);
+    }
+    validateVector(primaryKey, vector) {
+        if (!vector)
+            return;
+        this.assertPlaneVector(vector, `Vector for attribute "${String(primaryKey)}"`);
+    }
+    /**
+     * Everything reaching the plane — a committed record's projection or a query target — has to
+     * convert to the f32 it stores, and to a representable magnitude. What fails here would instead
+     * throw out of `Float32Array.from` or out of the crate, as an error neither the search path nor
+     * reconstruction can attribute to the record: a query would unlink a healthy file, and a record
+     * would abort every rebuild attempt at the same entry.
+     */
+    assertPlaneVector(vector, label) {
+        // A positive integer length, not merely a numeric one: a negative or fractional length skips
+        // the component loop and the emptiness check, and reaches Plane.create with it.
+        const length = vector?.length;
+        if (!Number.isInteger(length) || length < 1) {
+            throw new hdbError_ts_1.ClientError(`${label} must be an array of at least one number.`);
+        }
+        let sumOfSquares = 0;
+        for (let i = 0; i < vector.length; i++) {
+            const component = vector[i];
+            // The type check precedes Math.fround, which throws a TypeError on the BigInt a msgpackr
+            // or cbor-x decode produces for a large int64.
+            if (typeof component !== 'number' || !Number.isFinite(Math.fround(component))) {
+                throw new hdbError_ts_1.ClientError(`${label} has a component at index ${i} that is not a finite 32-bit float: ${String(component)}.`);
+            }
+            const asFloat32 = Math.fround(component);
+            sumOfSquares += asFloat32 * asFloat32;
+        }
+        // Components can each be f32-finite while their squares are not, which stores invMag 0 and
+        // makes every distance involving that node NaN.
+        if (!Number.isFinite(Math.fround(sumOfSquares))) {
+            throw new hdbError_ts_1.ClientError(`${label} has a magnitude too large to represent in 32-bit floats.`);
+        }
+        // The plane's dimensionality is fixed at create time by the first committed vector.
+        const dims = this.plane?.dims;
+        if (dims !== undefined && vector.length !== dims) {
+            throw new hdbError_ts_1.ClientError(`${label} has ${vector.length} components, but this index stores ${dims}.`);
+        }
+    }
+    applyDerivedValue(primaryKey, vector, version) {
+        this.validateVector(primaryKey, vector);
+        const safeKey = typeof primaryKey === 'number' ? [KEY_PREFIX, primaryKey] : primaryKey;
+        const pendingKey = pendingMappingKey(primaryKey);
+        const pendingMapping = this.pendingDerivedMappings.get(pendingKey);
+        const storedMapping = pendingMapping ?? this.indexStore.getSync(safeKey);
+        const oldNodeId = typeof storedMapping === 'number' ? storedMapping : storedMapping?.id;
+        // Neither a plane removal nor an insert is durable before the barrier, so a pending
+        // mapping keeps naming the last published node (previousId) for replay to remove.
+        const displacedNodeId = storedMapping?.previousId;
+        const durableNodeId = storedMapping?.pending ? displacedNodeId : oldNodeId;
+        if (storedMapping?.version != null && version != null && storedMapping.version > version)
+            return;
+        const nativeVector = vector ? Float32Array.from(vector) : undefined;
+        const signature = nativeVector
+            ? (0, node_crypto_1.createHash)('sha256')
+                .update(Buffer.from(nativeVector.buffer, nativeVector.byteOffset, nativeVector.byteLength))
+                .digest('base64url')
+            : undefined;
+        if (oldNodeId != null &&
+            signature &&
+            storedMapping.signature === signature &&
+            !pendingMapping &&
+            !storedMapping.pending) {
+            this.indexStore.putSync(safeKey, { id: oldNodeId, signature, version });
+            return;
+        }
+        let plane = vector ? this.getPlane(vector.length, true) : this.getPlane();
+        if (plane?.invalidated()) {
+            this.plane = undefined;
+            plane = vector ? this.getPlane(vector.length, true) : this.getPlane();
+        }
+        // The pre-commit check above ran before this worker had a plane to compare against, so it
+        // cannot have caught a mismatch. Reject before the removal below, or the record loses its
+        // old node on the way to failing.
+        if (vector && plane && vector.length !== plane.dims) {
+            throw new hdbError_ts_1.ClientError(`Vector for attribute "${String(primaryKey)}" has ${vector.length} components, but this index stores ${plane.dims}.`);
+        }
+        // in process a pending mapping's removals already happened and their ids may be reused
+        const removeDisplaced = displacedNodeId != null && !pendingMapping && displacedNodeId !== oldNodeId;
+        const removeOld = oldNodeId != null && !(pendingMapping && storedMapping.cleared);
+        if (removeDisplaced)
+            plane?.remove(displacedNodeId);
+        if (removeOld)
+            plane?.remove(oldNodeId);
+        if (!vector) {
+            const cleared = { id: oldNodeId, previousId: durableNodeId, version, pending: true, cleared: true };
+            if (oldNodeId != null || durableNodeId != null)
+                this.indexStore.putSync(safeKey, cleared);
+            else
+                this.indexStore.removeSync(safeKey);
+            this.pendingDerivedMappings.set(pendingKey, { ...cleared, primaryKey });
+            return;
+        }
+        if (!plane)
+            throw new hdbError_ts_1.ServerError('The native HNSW module is unavailable for a file-primary index', 503);
+        const nodeId = plane.insert(nativeVector, (0, msgpackr_1.pack)(primaryKey));
+        const mapping = { id: nodeId, previousId: durableNodeId, signature, version, pending: true };
+        this.indexStore.putSync(safeKey, mapping);
+        this.pendingDerivedMappings.set(pendingKey, { ...mapping, primaryKey });
+    }
+    async flushDerived(watermark) {
+        const plane = this.getPlane();
+        if (!plane) {
+            if (!(0, hnswPlaneBinding_ts_1.getPlaneBinding)())
+                throw new hdbError_ts_1.ServerError('The native HNSW module is unavailable', 503);
+            return this.publishDerivedMappings();
+        }
+        await plane.flushAsync(watermark);
+        this.publishDerivedMappings();
+    }
+    publishDerivedMappings() {
+        for (const { primaryKey, ...mapping } of this.pendingDerivedMappings.values()) {
+            const safeKey = typeof primaryKey === 'number' ? [KEY_PREFIX, primaryKey] : primaryKey;
+            if (mapping.cleared || mapping.id === undefined) {
+                this.indexStore.removeSync(safeKey);
+            }
+            else {
+                const published = { id: mapping.id, signature: mapping.signature, version: mapping.version };
+                this.indexStore.putSync(safeKey, published);
+            }
+        }
+        this.pendingDerivedMappings.clear();
     }
     index(primaryKey, vector, existingVector, options = {}) {
+        if (this.filePrimary) {
+            if (options.transaction)
+                return this.prepareCommitted(primaryKey, vector, existingVector);
+            // runIndexing invokes custom indexes without a transaction. The shared runtime owns the
+            // primary-record rebuild and the log replay; populating here would duplicate native
+            // construction and race the owner's generation reset.
+            return;
+        }
         // Reject non-finite components before touching the graph. NaN in particular poisons
         // bisectInsert (arr[mid].distance <= NaN is always false → returns 0, pinning the
         // candidate to rank 1 of every future search). Infinity causes analogous ordering
@@ -331,7 +1033,7 @@ class HierarchicalNavigableSmallWorld {
             const storedScale = q ? q.scale : undefined;
             let entryPoint = entryPointId && this.safeGetSync(entryPointId, options);
             if (entryPoint == null) {
-                const level = Math.floor(-Math.log(Math.random()) * this.mL);
+                const level = Math.min(Math.floor(-Math.log(this.random()) * this.mL), MAX_LEVEL);
                 const node = {
                     vector: storedVector,
                     scale: storedScale,
@@ -351,7 +1053,7 @@ class HierarchicalNavigableSmallWorld {
                 return;
             }
             // Generate random level for this new element
-            const level = oldNode.level ?? Math.min(Math.floor(-Math.log(Math.random()) * this.mL), MAX_LEVEL);
+            const level = oldNode.level ?? Math.min(Math.floor(-Math.log(this.random()) * this.mL), MAX_LEVEL);
             let currentLevel = entryPoint.level;
             if (level > currentLevel) {
                 // if we are at a higher level, make this the new entry point
@@ -500,14 +1202,15 @@ class HierarchicalNavigableSmallWorld {
                 }
             }
             // Store the new element
-            this.indexStore.put(nodeId, {
+            const storedNode = {
                 vector: storedVector,
                 scale: storedScale,
                 invMag,
                 level,
                 primaryKey,
                 ...connections,
-            }, options);
+            };
+            this.indexStore.put(nodeId, storedNode, options);
         }
         else {
             // removal of this node, but first make sure we have a valid entry point
@@ -818,8 +1521,10 @@ class HierarchicalNavigableSmallWorld {
             throw error;
         }
     }
-    /** O(1) node count — the shared id counter, else a single reverse seek to the largest node id. */
+    /** O(1) node count: the plane's id high-water, the shared id counter, or one reverse seek. */
     resolveNodeCount(options) {
+        if (this.filePrimary)
+            return this.getPlane()?.idHighWater() ?? 0;
         if (this.idIncrementer)
             return Number(Atomics.load(this.idIncrementer, 0));
         try {
@@ -1067,24 +1772,17 @@ class HierarchicalNavigableSmallWorld {
      * This the main entry from Harper's query functionality, where we actually search for an ordered list of nearest
      * neighbors, using the provided sort/order definition object and performing the multi-layer skip-list search.
      * This returns an iterable of the nearest neighbors to the provided target vector, with nearest ordered first.
-     * @param target
-     * @param value
-     * @param descending
-     * @param distance
-     * @param comparator
-     * @param context
+     *
+     * This is also the contract an index implemented outside this repo has to satisfy, so everything
+     * optional is named: a future capability (a paging cursor, a deadline, a recall target) is a new
+     * field on `options` rather than a positional argument that breaks every existing implementation.
+     * @param searchCondition the vector query: `target`, `comparator`, and the optional `value`,
+     *   `descending`, `distance`, `ef` and `filterExpansion` tuning knobs
+     * @param context the query context; its `transaction` is the nested RocksDB transaction reads use
+     * @param options optional, named: `filter` (predicate-aware traversal) and `minResults`
      */
-    search({ target, value, descending, distance, comparator, ef, filterExpansion, }, context, 
-    // Predicate-aware traversal (#1241). When provided, only nodes for which `filter(primaryKey)`
-    // returns true are admitted to the result list at layer 0; routing is unaffected. Composed by
-    // search.ts from companion AND conditions and caller-supplied vector/row filters. Must be
-    // synchronous and side-effect free. JS-API only (never from a REST query string).
-    filter, 
-    // offset + limit for a bounded query. A layer-0 search returns at most `ef` candidates, so a
-    // query asking for more rows than that used to come back short with no error — capped at 512
-    // (AUTO_EF_MAX) however large the limit was. Raising ef to cover the request keeps `limit`
-    // meaningful; the caller pays for what it asked for.
-    minResults) {
+    search({ target, value, descending, distance, comparator, ef, filterExpansion, maxIndexLagMilliseconds = hnswDerivedIndex_ts_1.DEFAULT_MAX_INDEX_LAG_MILLISECONDS, waitForIndexMilliseconds = 0, }, context, { filter, minResults, } = {}) {
+        const waiting = this.filePrimary && waitForIndexMilliseconds > 0;
         let limit; // only set for threshold comparators; 0 is a valid threshold (e.g. dotProduct)
         let limitInclusive = false; // true for `le`, false for `lt`
         switch (comparator) {
@@ -1120,7 +1818,18 @@ class HierarchicalNavigableSmallWorld {
             throw new hdbError_ts_1.ClientError('A target vector must be provided for an HNSW query');
         if (!Array.isArray(target))
             throw new hdbError_ts_1.ClientError('The target vector must be an array');
-        const options = context.transaction; // should have a nested RocksDB transaction
+        if (this.filePrimary &&
+            (typeof maxIndexLagMilliseconds !== 'number' ||
+                !Number.isFinite(maxIndexLagMilliseconds) ||
+                maxIndexLagMilliseconds < 0))
+            throw new hdbError_ts_1.ClientError('maxIndexLagMilliseconds must be a finite nonnegative number');
+        if (this.filePrimary &&
+            (typeof waitForIndexMilliseconds !== 'number' ||
+                !Number.isFinite(waitForIndexMilliseconds) ||
+                waitForIndexMilliseconds < 0 ||
+                waitForIndexMilliseconds > hnswDerivedIndex_ts_1.MAX_WAIT_FOR_INDEX_MILLISECONDS))
+            throw new hdbError_ts_1.ClientError(`waitForIndexMilliseconds must be a finite number between 0 and ${hnswDerivedIndex_ts_1.MAX_WAIT_FOR_INDEX_MILLISECONDS}`);
+        const txnOptions = context.transaction; // should have a nested RocksDB transaction
         // Resolve search ef: per-query ef wins; else use the schema-pinned value (from either ef option);
         // otherwise auto-scale with the graph size so recall holds as the table grows.
         let effectiveEf = this.efConstructionSearch;
@@ -1163,7 +1872,109 @@ class HierarchicalNavigableSmallWorld {
                 filterEvaluations: 0,
             }
             : undefined;
-        let entryPoint = this.getEntryPoint(options);
+        const rerankK = comparator === 'sort' && minResults !== undefined
+            ? Math.min(effectiveEf, Math.max(RERANK_MIN, minResults * RERANK_FACTOR))
+            : effectiveEf;
+        if (this.filePrimary && distanceFunction !== this.distance) {
+            throw new hdbError_ts_1.ClientError('A nativePlane index only supports its configured cosine distance');
+        }
+        const searchNative = (certified) => {
+            if (this.filePrimary && this.derivedReadiness() !== 'ready') {
+                throw new hdbError_ts_1.ServerError(`The native HNSW index is ${this.derivedReadiness() === 'unavailable' ? 'unavailable' : 'rebuilding'}`, 503);
+            }
+            // The plane traverses the index's own metric (cosine — the eligibility requirement), so a
+            // query overriding `distance` has to take the JS path: rescoreResults only corrects the
+            // reported distances of whatever candidates came back, not which candidates the beam kept.
+            if (this.planeEligible && distanceFunction === this.distance) {
+                const plane = this.getPlane(target.length, false);
+                // A file-primary index has no JS path to fall through to, so a target the plane cannot
+                // accept has to fail as the client error it is. Otherwise it throws out of
+                // Float32Array.from or the traversal, is read as plane corruption, and unlinks a healthy
+                // file — one malformed query costing a full reconstruction.
+                if (this.filePrimary && plane)
+                    this.assertPlaneVector(target, 'Search target');
+                // a non-file-primary query whose dimensionality differs from the graph's takes the JS
+                // path, which tolerates the mismatch, rather than disabling the healthy plane
+                if (plane && plane.dims === target.length && this.planeSearchReady(plane)) {
+                    const coverage = certified ?? (this.filePrimary ? this.queryCoverage(maxIndexLagMilliseconds) : undefined);
+                    try {
+                        const searched = this.searchPlane(plane, target, rerankK, effectiveEf, filter, filterState)
+                            .catch((error) => {
+                            if (error?.[NOT_A_PLANE_FAILURE])
+                                throw error;
+                            // There is no JS graph behind a file-primary index: it stays unavailable until
+                            // its audit-backed rebuild succeeds.
+                            this.disablePlane(error);
+                            throw new hdbError_ts_1.ServerError('The native HNSW index is rebuilding', 503);
+                        })
+                            .then((entries) => {
+                            if (coverage)
+                                Object.defineProperty(entries, 'indexCoverage', { value: coverage });
+                            return entries;
+                        });
+                        if (coverage)
+                            Object.defineProperty(searched, 'indexCoverage', { value: coverage });
+                        return searched;
+                    }
+                    catch (error) {
+                        // Handle a throw raised before the asynchronous native search returns its promise.
+                        this.disablePlane(error);
+                        throw new hdbError_ts_1.ServerError('The native HNSW index is rebuilding', 503);
+                    }
+                }
+            }
+            if (this.filePrimary) {
+                const state = this.derivedReadiness();
+                if (state === 'unavailable')
+                    throw new hdbError_ts_1.ServerError('The native HNSW index is unavailable', 503);
+                const planePath = this.planeFilePath();
+                if (state !== 'ready' || (planePath && (0, node_fs_1.existsSync)(planePath))) {
+                    throw new hdbError_ts_1.ServerError('The native HNSW index is rebuilding', 503);
+                }
+                // The absence of a file is not proof of an empty index — it is also the state just after
+                // any process removes an unopenable one. Only the surviving node mappings prove it.
+                if (this.hasNodeMappings()) {
+                    // A table taking no further writes never drains, so a query is the only thing left
+                    // that can notice the graph is gone and ask for it back.
+                    logger.error?.(`${this.indexStore.name} lost its native file while its node mappings survive`);
+                    this.derivedHost?.requestRebuild();
+                    throw new hdbError_ts_1.ServerError('The native HNSW index is rebuilding', 503);
+                }
+                const entries = withStats([], filterState);
+                Object.defineProperty(entries, 'indexCoverage', {
+                    value: certified ?? this.queryCoverage(maxIndexLagMilliseconds),
+                });
+                return entries;
+            }
+        };
+        if (waiting) {
+            context.signal?.throwIfAborted();
+            const host = this.derivedHost;
+            const state = host?.readiness().state;
+            if (state !== 'ready') {
+                throw new hdbError_ts_1.ServerError(`The native HNSW index is ${state === 'unavailable' ? 'unavailable' : 'rebuilding'}`, 503);
+            }
+            const searched = Promise.resolve(context.indexSearchStart).then(async () => {
+                context.signal?.throwIfAborted();
+                if (minResults === 0)
+                    return [];
+                const started = (0, derivedIndexRuntime_ts_1.derivedIndexTime)(this.indexStore.rootStore);
+                if (host.coverage(0)?.state !== 'current')
+                    await host.waitForCoverage(started, waitForIndexMilliseconds, context.signal);
+                context.signal?.throwIfAborted();
+                return searchNative({
+                    state: 'current',
+                    maxLagMilliseconds: maxIndexLagMilliseconds,
+                    lagUpperBoundMilliseconds: 0,
+                });
+            });
+            searched.catch(() => { });
+            return searched;
+        }
+        const native = searchNative();
+        if (native)
+            return native;
+        let entryPoint = this.getEntryPoint(txnOptions);
         if (!entryPoint)
             return withStats([], filterState);
         let entryPointId = entryPoint.id;
@@ -1176,7 +1987,7 @@ class HierarchicalNavigableSmallWorld {
         // nodes) rather than to ef. See DESIGN.md.
         for (let l = entryPoint.level; l >= 0; l--) {
             // Search for closest neighbors at current level
-            results = this.searchLayer(target, entryPointId, entryPoint, l === 0 ? effectiveEf : ROUTING_EF, l, options, distanceFunction, l === 0 ? filter : undefined, l === 0 ? filterState : undefined);
+            results = this.searchLayer(target, entryPointId, entryPoint, l === 0 ? effectiveEf : ROUTING_EF, l, txnOptions, distanceFunction, l === 0 ? filter : undefined, l === 0 ? filterState : undefined);
             if (results.length > 0) {
                 const neighbor = results[0]; // closest neighbor becomes new entry point
                 entryPoint = neighbor.node;
@@ -1185,6 +1996,8 @@ class HierarchicalNavigableSmallWorld {
         }
         if (limit !== undefined)
             results = results.filter((candidate) => limitInclusive ? candidate.distance <= limit : candidate.distance < limit);
+        if (results.length > rerankK)
+            results = results.slice(0, rerankK);
         return withStats(results.map((candidate) => ({
             // we return the result as an entry so we can provide distance as metadata
             key: candidate.node.primaryKey, // return value