@delali/narsil 0.2.4-next.52 → 0.2.4-next.55

package/dist/{chunk-YPYULM5E.mjs → chunk-TVT5V4E7.mjs}

@@ -1,9 +1,9 @@
-import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, DEFAULT_OFFSET } from './chunk-3GWP2TO6.mjs';
-import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn } from './chunk-WM7IS45M.mjs';
-import { flattenSchema } from './chunk-ZT5BRDFW.mjs';
+import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-D2S642IV.mjs';
+import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn } from './chunk-5T6AOR37.mjs';
+import { flattenSchema } from './chunk-L225SHNB.mjs';
 import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
 import { compareCodePoints, compareComparableKeys } from './chunk-2GWVQWGA.mjs';
-import { NarsilError, ErrorCodes } from './chunk-P4ELXVJQ.mjs';
+import { NarsilError, ErrorCodes } from './chunk-FRQVMFY7.mjs';
 
 // src/search/cursor-binding.ts
 var ABSENT_TAG = 0;
@@ -171,32 +171,37 @@ function now() {
   return Date.now();
 }
 function validateIndexName(name) {
-  if (!name || name.length === 0) {
-    throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
+  const given = name;
+  if (typeof given !== "string" || name.length === 0) {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index name must be a string of at least one character", {
+      received: given === null ? "null" : typeof given
+    });
   }
   if (name.length > MAX_INDEX_NAME_LENGTH) {
-    throw new NarsilError(
-      ErrorCodes.INDEX_NOT_FOUND,
-      `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
-      {
-        indexName: name,
-        length: name.length
-      }
-    );
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`, {
+      indexName: name,
+      length: name.length
+    });
   }
   if (!INDEX_NAME_PATTERN.test(name)) {
     throw new NarsilError(
-      ErrorCodes.INDEX_NOT_FOUND,
+      ErrorCodes.CONFIG_INVALID,
       `Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
       { indexName: name }
     );
   }
   if (name.includes("..")) {
-    throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name "${name}" must not contain ".."`, { indexName: name });
   }
 }
 function validateDocId(docId) {
-  if (!docId || docId.length === 0) {
+  const given = docId;
+  if (typeof given !== "string") {
+    throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must be a string", {
+      received: given === null ? "null" : Array.isArray(given) ? "array" : typeof given
+    });
+  }
+  if (docId.length === 0) {
     throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
   }
   if (docId.length > MAX_DOC_ID_LENGTH) {
@@ -237,6 +242,16 @@ function requirePositiveSafeInteger(field, value) {
     throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${field} must be a positive safe integer`, { field, value });
   }
 }
+function validateBootstrapModule(bootstrapModule) {
+  if (bootstrapModule === void 0) return;
+  const absolute = bootstrapModule.startsWith("/") || ABSOLUTE_WINDOWS_PATH_PATTERN.test(bootstrapModule) || MODULE_URL_SCHEME_PATTERN.test(bootstrapModule);
+  if (absolute) return;
+  throw new NarsilError(
+    ErrorCodes.CONFIG_INVALID,
+    `workers.bootstrapModule is "${bootstrapModule}", which a worker would resolve against Narsil's own directory, so pass an absolute path or a URL such as new URL('./bootstrap.js', import.meta.url).href`,
+    { bootstrapModule }
+  );
+}
 function validateWorkerConfig(workers, lifecycle) {
   if (workers === void 0) return;
   requirePositiveSafeInteger("workers.count", workers.count);
@@ -247,6 +262,7 @@ function validateWorkerConfig(workers, lifecycle) {
       mainCopyQueries: workers.mainCopyQueries
     });
   }
+  validateBootstrapModule(workers.bootstrapModule);
   const closeAfter = lifecycle?.idleTimeoutMs;
   if (workers.idleTimeoutMs !== void 0 && closeAfter !== void 0 && workers.idleTimeoutMs > closeAfter) {
     throw new NarsilError(
@@ -568,21 +584,19 @@ function findDensestWindow(positions, text, windowSize) {
 }
 function applySnippetTruncation(taggedText, originalText, positions, maxLen, preTag, postTag) {
   if (maxLen <= 0 || originalText.length <= maxLen) {
-    return { snippet: taggedText, adjustedPositions: positions };
+    return { snippet: taggedText, visiblePositions: positions };
   }
   const window = findDensestWindow(positions, originalText, maxLen);
   const snippetRaw = originalText.substring(window.start, window.end);
-  const adjustedPositions = [];
+  const visiblePositions = [];
+  const withinSnippet = [];
   for (const pos of positions) {
-    if (pos.start >= window.start && pos.end <= window.end) {
-      adjustedPositions.push({
-        start: pos.start - window.start,
-        end: pos.end - window.start
-      });
-    }
+    if (pos.start < window.start || pos.end > window.end) continue;
+    visiblePositions.push(pos);
+    withinSnippet.push({ start: pos.start - window.start, end: pos.end - window.start });
   }
   let snippet = snippetRaw;
-  const sortedDesc = [...adjustedPositions].sort((a, b) => b.start - a.start);
+  const sortedDesc = [...withinSnippet].sort((a, b) => b.start - a.start);
   for (const pos of sortedDesc) {
     snippet = snippet.substring(0, pos.start) + preTag + snippet.substring(pos.start, pos.end) + postTag + snippet.substring(pos.end);
   }
@@ -590,7 +604,7 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
   const needsTrailingEllipsis = window.end < originalText.length;
   if (needsLeadingEllipsis) snippet = `...${snippet}`;
   if (needsTrailingEllipsis) snippet = `${snippet}...`;
-  return { snippet, adjustedPositions };
+  return { snippet, visiblePositions };
 }
 function highlightField(text, queryTokens, language, options) {
   const preTag = options?.preTag ?? "<mark>";
@@ -637,7 +651,7 @@ function highlightField(text, queryTokens, language, options) {
     return { snippet: text, positions: [] };
   }
   if (maxSnippetLength > 0 && text.length > maxSnippetLength) {
-    const { snippet, adjustedPositions } = applySnippetTruncation(
+    const { snippet, visiblePositions } = applySnippetTruncation(
       text,
       text,
       mergedPositions,
@@ -645,7 +659,7 @@ function highlightField(text, queryTokens, language, options) {
       preTag,
       postTag
     );
-    return { snippet, positions: adjustedPositions };
+    return { snippet, positions: visiblePositions };
   }
   let result = text;
   const sortedDesc = [...mergedPositions].sort((a, b) => b.start - a.start);
@@ -656,7 +670,7 @@ function highlightField(text, queryTokens, language, options) {
 }
 
 // src/engine/query/highlight.ts
-function applyHighlights(hits, params, language, analysis) {
+function applyHighlights(hits, params, language, analysis, readStoredDocument) {
   if (!params.highlight) return;
   const queryTokenResult = tokenize(params.term ?? "", language, {
     stem: true,
@@ -682,9 +696,14 @@ function applyHighlights(hits, params, language, analysis) {
   }
   for (const hit of hits) {
     const highlights = {};
+    const projected = hit.document;
+    let stored;
     for (const field of params.highlight.fields) {
-      const doc = hit.document;
-      const fieldValue = doc[field];
+      let fieldValue = projected[field];
+      if (typeof fieldValue !== "string" && readStoredDocument !== void 0) {
+        stored ??= readStoredDocument(hit.id);
+        fieldValue = stored?.[field];
+      }
       if (typeof fieldValue === "string") {
         highlights[field] = highlightField(fieldValue, queryTokenResult.tokens, language, {
           preTag: params.highlight.preTag,
@@ -778,6 +797,32 @@ function vectorSearchersOf(context) {
 function resolveVectorIndex(context, fieldName) {
   return vectorSearchersOf(context).get(fieldName);
 }
+var SEARCH_MODES = /* @__PURE__ */ new Set(["fulltext", "vector", "hybrid"]);
+function requireKnownMode(params) {
+  if (params.mode === void 0 || SEARCH_MODES.has(params.mode)) return;
+  throw new NarsilError(
+    ErrorCodes.SEARCH_INVALID_MODE,
+    `A search takes the mode "fulltext", "vector", or "hybrid", while this query names "${String(params.mode)}"`,
+    { mode: String(params.mode), modes: [...SEARCH_MODES] }
+  );
+}
+function requireVectorSearchable(params, context, needsVector) {
+  if (!needsVector) return;
+  const field = params.vector?.field;
+  if (field === void 0) {
+    throw new NarsilError(
+      ErrorCodes.SEARCH_INVALID_MODE,
+      `A "${params.mode}" query needs a vector clause naming the field to search`,
+      { mode: params.mode }
+    );
+  }
+  if (vectorSearchersOf(context).has(field)) return;
+  throw new NarsilError(
+    ErrorCodes.SEARCH_INVALID_FIELD,
+    `Field "${field}" has no vector index, so a vector search cannot use it`,
+    { field, vectorFields: [...vectorSearchersOf(context).keys()] }
+  );
+}
 function clampAlpha(alpha) {
   if (alpha === void 0) return 0.5;
   if (!Number.isFinite(alpha)) return 0.5;
@@ -1065,6 +1110,7 @@ async function executeQuery(params, context) {
   const offset = clampOffset(params.offset);
   requireWithinResultWindow(limit, offset);
   const sortFields = normalizeSort(params.sort);
+  requireKnownMode(params);
   const sortSignature = sortSignatureOf(params.sort);
   const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
   const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
@@ -1078,6 +1124,7 @@ async function executeQuery(params, context) {
     );
   }
   requireSortableFields(params.sort, config.schema);
+  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
   let paginated;
   let nextCursor;
   let count;
@@ -1201,7 +1248,9 @@ async function executeQuery(params, context) {
     }
   }
   if (params.highlight) {
-    applyHighlights(paginated, params, language, manager.analysis);
+    const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
+    const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
+    applyHighlights(paginated, params, language, manager.analysis, readStoredDocument);
   }
   const elapsed = now() - startTime;
   return {
@@ -1217,10 +1266,12 @@ async function executeQuery(params, context) {
 async function executePreflight(params, context) {
   const { manager, language, config, workerSearch, indexName } = context;
   const startTime = now();
+  requireKnownMode(params);
   const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
   const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
   const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
   const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
+  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
   const requestedVectorField = params.vector?.field;
   const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
   let totalMatched;

package/dist/{chunk-T62YOSGS.mjs → chunk-U2AL3SON.mjs}

@@ -1,10 +1,8 @@
 import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
-import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-WFKIGEBY.mjs';
+import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-UTOFOJOR.mjs';
 import { compareCodePoints } from './chunk-2GWVQWGA.mjs';
-import { NarsilError, ErrorCodes } from './chunk-P4ELXVJQ.mjs';
-import { existsSync, readSync, closeSync, openSync } from 'fs';
-import { createRequire } from 'module';
-import { join, dirname } from 'path';
+import { NarsilError, ErrorCodes } from './chunk-FRQVMFY7.mjs';
+import { readSync, closeSync, openSync } from 'fs';
 
 // src/vector/constants.ts
 var WASM_PAGE_BYTES = 65536;
@@ -185,97 +183,6 @@ function arenaFloat32Distance(simd, byteA, byteB, dimension, metric, magnitudeA,
   return 1 - dot / (magnitudeA * magnitudeB);
 }
 
-// src/vector/native/backend.ts
-var nativeSearchCoreEnabled = true;
-function nativeSearchCoreIsEnabled() {
-  return nativeSearchCoreEnabled;
-}
-function stopUsingTheNativeSearchCore(error) {
-  if (!nativeSearchCoreEnabled) return;
-  nativeSearchCoreEnabled = false;
-  console.warn(
-    "The native search core raised an error, so this thread searches through WebAssembly from now on:",
-    error instanceof Error ? error.message : String(error)
-  );
-}
-function noteFallback(operation) {
-}
-var NATIVE_CORE_ABI_VERSION = 2;
-var BINARY_FILE_NAME = "narsil-core.node";
-var MUSL_DIRECTORY = "musl";
-var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([
-  "darwin-arm64",
-  "darwin-x64",
-  "linux-arm64",
-  "linux-x64",
-  "win32-arm64",
-  "win32-x64"
-]);
-var loaded;
-function usesMuslLibc() {
-  try {
-    const report = process.report.getReport();
-    return report.header?.glibcVersionRuntime === void 0;
-  } catch {
-    return false;
-  }
-}
-function publishedBinaryPath(require2) {
-  const target = `${process.platform}-${process.arch}`;
-  if (!PLATFORMS_WITH_A_BINARY.has(target)) return null;
-  try {
-    const manifest = require2.resolve(`@delali/narsil-native-${target}/package.json`);
-    const segments = process.platform === "linux" && usesMuslLibc() ? [MUSL_DIRECTORY, BINARY_FILE_NAME] : [BINARY_FILE_NAME];
-    return join(dirname(manifest), ...segments);
-  } catch {
-    return null;
-  }
-}
-function namedBinaryPath() {
-  const named = process.env.NARSIL_NATIVE_CORE_PATH;
-  return named !== void 0 && named.length > 0 ? named : null;
-}
-function refuseToSearchWithoutTheCore(reason) {
-  throw new NarsilError(ErrorCodes.CONFIG_INVALID, `NARSIL_REQUIRE_NATIVE_CORE is 1, and ${reason}`, {
-    platform: `${process.platform}-${process.arch}`
-  });
-}
-function reportMissingCore(reason, named) {
-  if (process.env.NARSIL_REQUIRE_NATIVE_CORE === "1") refuseToSearchWithoutTheCore(reason);
-  if (named) console.warn(`Narsil searches through WebAssembly in this process, because ${reason}.`);
-  return null;
-}
-function loadedCore(path) {
-  const require2 = createRequire(import.meta.url);
-  const core = require2(path);
-  if (typeof core.abiVersion !== "function" || core.abiVersion() !== NATIVE_CORE_ABI_VERSION) return null;
-  return core;
-}
-function loadNativeCore() {
-  if (loaded !== void 0) return loaded;
-  loaded = null;
-  const named = namedBinaryPath();
-  if (process.env.NARSIL_SEARCH_BACKEND === "wasm") {
-    return reportMissingCore("NARSIL_SEARCH_BACKEND asks for the WebAssembly search", false);
-  }
-  const path = named ?? publishedBinaryPath(createRequire(import.meta.url));
-  if (path === null || !existsSync(path)) {
-    return reportMissingCore(`this platform has no native search core at ${path ?? "any known path"}`, named !== null);
-  }
-  try {
-    loaded = loadedCore(path);
-  } catch (error) {
-    return reportMissingCore(
-      `loading the native search core at ${path} failed with "${error instanceof Error ? error.message : String(error)}"`,
-      named !== null
-    );
-  }
-  if (loaded === null) {
-    return reportMissingCore(`the native search core at ${path} reports another ABI version`, named !== null);
-  }
-  return loaded;
-}
-
 // src/vector/shared-buffers/growable.ts
 function sharedMemoryAvailable() {
   return typeof SharedArrayBuffer === "function";
@@ -1998,6 +1905,7 @@ function insertNode(state, ord, holdsGraphLock = false) {
   writeRecordBeforePlacement(state, ord);
   if (placeThroughTheCore(state, ord, holdsGraphLock)) return true;
   if (nodeExists(state, ord)) return false;
+  noteFallback("place");
   if (!holdsGraphLock) lockGraphShared(state.locks);
   try {
     clearTombstone(state, ord);
@@ -2068,6 +1976,7 @@ function markTombstone(state, ord) {
   if (!nodeExists(state, ord) || !reachTombstone(state, ord)) return;
   const field = nativeFieldFor(state);
   if (field !== null && nativeRemove(state, field, ord)) return;
+  noteFallback("remove");
   if (Atomics.compareExchange(state.tombstones, ord, 0, 1) === 0) {
     Atomics.add(state.header, GRAPH_TOMBSTONE_COUNT, 1);
   }
@@ -2083,6 +1992,7 @@ function compactTombstones(state) {
   if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
   const field = nativeFieldFor(state);
   if (field !== null && nativeCompact(state, field, state.buildMetric)) return;
+  noteFallback("compact");
   const tombstonedOrds = [];
   const slots = adjacencySlots(state.adjacency);
   for (let ord = 0; ord < slots; ord++) {
@@ -2236,6 +2146,7 @@ function traverse(state, query, k, searchMetric, options) {
       return { candidates, depth: useQuantized ? depth : 0, arenaQuery: null, qMag: 0, native };
     }
   }
+  noteFallback("search");
   const store = state.store;
   const arenaQuery = store.prepareQueryArena(query);
   const qMag = arenaQuery ? arenaQuery.magnitude : magnitude(query);
@@ -2361,6 +2272,7 @@ function collectHits(state, query, k, searchMetric, minSimilarity, options) {
   return hits;
 }
 function rescoreWithFullPrecision(state, candidates, query, qMag, arenaQuery, depth, metric, minSimilarity, filter) {
+  noteFallback("score");
   const rescored = [];
   const nearest = Math.min(candidates.size, depth);
   for (let i = 0; i < nearest; i++) {
@@ -2632,6 +2544,7 @@ function openSharedQuantizer(store, layout, metric) {
   function writeCodesOf(ordinals) {
     if (!isCalibrated() || ordinals.length === 0) return;
     if (nativeQuantise(store.handles, metric, ordinals)) return;
+    noteFallback("quantise");
     for (const ordinal of ordinals) writeRecordOf(ordinal);
   }
   function stageQuery(block, query) {
@@ -2728,6 +2641,7 @@ function openSharedQuantizer(store, layout, metric) {
     writeCodesOf,
     calibrateFrom(ordinals) {
       if (nativeCalibrate(store.handles, metric, ordinals)) return isCalibrated();
+      noteFallback("calibrate");
       const centroid = osqCentroid(storedVectors(ordinals), dimension, metric);
       if (centroid === null) return false;
       writeCentroid(centroid);
@@ -4189,7 +4103,11 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
       throw new Error(`Cannot insert HNSW node: vector for "${docId}" not found in VectorStore`);
     }
     if (entry.vector.length !== dimension) {
-      throw new Error(`Vector dimension mismatch: expected ${dimension}, got ${entry.vector.length}`);
+      throw new NarsilError(
+        ErrorCodes.VECTOR_DIMENSION_MISMATCH,
+        `Vector dimension mismatch: expected ${dimension}, got ${entry.vector.length}`,
+        { docId, expected: dimension, received: entry.vector.length }
+      );
     }
     insertNode(state, ord);
   }
@@ -4277,4 +4195,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
   };
 }
 
-export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, noteFallback, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore };
+export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore };

package/dist/chunk-UTOFOJOR.mjs

@@ -0,0 +1,135 @@
+import { NarsilError, ErrorCodes } from './chunk-FRQVMFY7.mjs';
+import { existsSync } from 'fs';
+import { createRequire } from 'module';
+import { join, dirname } from 'path';
+
+// src/vector/hnsw/constants.ts
+var MAX_LAYER_CAP = 32;
+var MAX_M = 512;
+var INITIAL_UPPER_SLOTS = 64;
+var INITIAL_LIST_CAPACITY = 64;
+var PLACEMENT_ROOM_REQUESTS = 8;
+var COMPACTION_TOMBSTONE_RATIO = 0.1;
+var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
+var DEFAULT_EF_SEARCH = 50;
+
+// src/vector/native/backend.ts
+var nativeSearchCoreEnabled = true;
+function nativeSearchCoreIsEnabled() {
+  return nativeSearchCoreEnabled;
+}
+function stopUsingTheNativeSearchCore(error) {
+  if (!nativeSearchCoreEnabled) return;
+  nativeSearchCoreEnabled = false;
+  console.warn(
+    "The native search core raised an error, so this thread searches through WebAssembly from now on:",
+    error instanceof Error ? error.message : String(error)
+  );
+}
+function noteFallback(operation) {
+}
+var NATIVE_CORE_ABI_VERSION = 2;
+var BINARY_FILE_NAME = "narsil-core.node";
+var MUSL_DIRECTORY = "musl";
+var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([
+  "darwin-arm64",
+  "darwin-x64",
+  "linux-arm64",
+  "linux-x64",
+  "win32-arm64",
+  "win32-x64"
+]);
+var loaded;
+function usesMuslLibc() {
+  try {
+    const report = process.report.getReport();
+    return report.header?.glibcVersionRuntime === void 0;
+  } catch {
+    return false;
+  }
+}
+function isUnavailable(value) {
+  return "missing" in value;
+}
+function binaryFileSegments() {
+  return process.platform === "linux" && usesMuslLibc() ? [MUSL_DIRECTORY, BINARY_FILE_NAME] : [BINARY_FILE_NAME];
+}
+function publishedCoreLocation(target) {
+  const packageName = `@delali/narsil-native-${target}`;
+  let manifest;
+  try {
+    manifest = createRequire(import.meta.url).resolve(`${packageName}/package.json`);
+  } catch {
+    return { missing: `this installation has no "${packageName}" package` };
+  }
+  const path = join(dirname(manifest), ...binaryFileSegments());
+  if (!existsSync(path)) return { missing: `the "${packageName}" package has no search core at ${path}` };
+  return { path };
+}
+function coreLocation() {
+  const named = process.env.NARSIL_NATIVE_CORE_PATH;
+  if (named !== void 0 && named.length > 0) {
+    if (!existsSync(named)) {
+      return { missing: `this process sets NARSIL_NATIVE_CORE_PATH to ${named}, where no file exists` };
+    }
+    return { path: named };
+  }
+  const target = `${process.platform}-${process.arch}`;
+  if (!PLATFORMS_WITH_A_BINARY.has(target)) {
+    return { missing: `the Narsil release includes no search core for ${target}` };
+  }
+  return publishedCoreLocation(target);
+}
+function refuseToSearchWithoutTheCore(reason) {
+  throw new NarsilError(ErrorCodes.CONFIG_INVALID, `NARSIL_REQUIRE_NATIVE_CORE is 1, yet ${reason}`, {
+    platform: `${process.platform}-${process.arch}`
+  });
+}
+function requestedBackend() {
+  const raw = process.env.NARSIL_SEARCH_BACKEND;
+  if (raw === void 0 || raw.trim().length === 0) return null;
+  const named = raw.trim().toLowerCase();
+  if (named === "native" || named === "wasm") return named;
+  throw new NarsilError(
+    ErrorCodes.CONFIG_INVALID,
+    `NARSIL_SEARCH_BACKEND takes either "native" or "wasm", while this environment sets it to "${raw}"`,
+    { value: raw, backends: ["native", "wasm"] }
+  );
+}
+function reportMissingCore(reason, announce) {
+  if (process.env.NARSIL_REQUIRE_NATIVE_CORE === "1") refuseToSearchWithoutTheCore(reason);
+  loaded = null;
+  if (announce) console.warn(`Narsil searches through WebAssembly on this thread, because ${reason}.`);
+  return null;
+}
+function coreAt(path) {
+  let core;
+  try {
+    core = createRequire(import.meta.url)(path);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { missing: `node raised "${message}" while loading the search core at ${path}` };
+  }
+  if (typeof core.abiVersion !== "function") return { missing: `the file at ${path} is no Narsil search core` };
+  const reported = core.abiVersion();
+  if (reported !== NATIVE_CORE_ABI_VERSION) {
+    return {
+      missing: `the search core at ${path} reports ABI version ${reported}, while this build of Narsil needs ${NATIVE_CORE_ABI_VERSION}`
+    };
+  }
+  return core;
+}
+function loadNativeCore() {
+  if (loaded !== void 0) return loaded;
+  if (requestedBackend() === "wasm") {
+    return reportMissingCore("this process sets NARSIL_SEARCH_BACKEND to the WebAssembly search", false);
+  }
+  const location = coreLocation();
+  if (isUnavailable(location)) return reportMissingCore(location.missing, true);
+  const core = coreAt(location.path);
+  if (isUnavailable(core)) return reportMissingCore(core.missing, true);
+  loaded = core;
+  return loaded;
+}
+
+export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, DEFAULT_EF_SEARCH, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, PLACEMENT_ROOM_REQUESTS, loadNativeCore, nativeSearchCoreIsEnabled, noteFallback, stopUsingTheNativeSearchCore };

package/dist/{chunk-2FO7KZ6K.mjs → chunk-XNQAIO74.mjs}

@@ -1,21 +1,59 @@
 import { detectRuntime } from './chunk-AV4IJKD2.mjs';
-import { cloneProjected, mergeFacets, visitTokens, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, projectionKeepsField, setNestedValue, fnv1a, bitsetHas, createBitSet, kWayMerge, boundedLevenshtein, bitsetSet, tokenize, bitsetFromSet, tokenizeIterator, bitsetAnd, bitsetOr, bitsetNot } from './chunk-WM7IS45M.mjs';
-import { flattenSchema, validateDocument, validateDocumentStrict, isQuantizationMode, isStorageMode, extractVectorFieldsFromSchema, isTextFieldType, VECTOR_PATTERN } from './chunk-ZT5BRDFW.mjs';
+import { cloneProjected, mergeFacets, visitTokens, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, projectionKeepsField, setNestedValue, fnv1a, bitsetHas, createBitSet, kWayMerge, boundedLevenshtein, bitsetSet, tokenize, bitsetFromSet, tokenizeIterator, bitsetAnd, bitsetOr, bitsetNot } from './chunk-5T6AOR37.mjs';
+import { flattenSchema, validateDocument, validateDocumentStrict, isQuantizationMode, isStorageMode, extractVectorFieldsFromSchema, isTextFieldType, VECTOR_PATTERN } from './chunk-L225SHNB.mjs';
 import { english } from './chunk-TGP5M5R7.mjs';
-import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, addToOrdinalFilter, ordinalFilterValues, isTombstoned, nativeScoresOf, toScore, removeFromOrdinalFilter, noteFallback, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, requireWholeNeighbourLists, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-T62YOSGS.mjs';
+import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, addToOrdinalFilter, ordinalFilterValues, isTombstoned, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, requireWholeNeighbourLists, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-U2AL3SON.mjs';
+import { noteFallback } from './chunk-UTOFOJOR.mjs';
 import { VALID_HNSW_METRICS } from './chunk-IFBKLBGB.mjs';
 import { compareCodePoints, toComparableSortValue, readSortField, compareComparableValues, SORT_VALUE_MAX_CODE_POINTS } from './chunk-2GWVQWGA.mjs';
-import { NarsilError, ErrorCodes } from './chunk-P4ELXVJQ.mjs';
+import { NarsilError, ErrorCodes } from './chunk-FRQVMFY7.mjs';
+import { totalmem } from 'os';
 import { getHeapStatistics } from 'v8';
 import { encode, decode } from '@msgpack/msgpack';
 
+var MEGABYTE_BYTES = 1048576;
+var IN_MEGABYTES = /^--max-old-space-size=(\d+)$/;
+var AS_PERCENTAGE = /^--max-old-space-size-percentage=(\d+(?:\.\d+)?)$/;
+function flagEntries() {
+  const fromNodeOptions = (process.env.NODE_OPTIONS ?? "").split(/\s+/).filter((entry) => entry.length > 0);
+  return [...process.execArgv, ...fromNodeOptions];
+}
+function memoryTheHostAllows() {
+  const constrained = process.constrainedMemory?.() ?? 0;
+  return constrained > 0 ? constrained : totalmem();
+}
+function readConfiguredLimit() {
+  let megabytes = null;
+  let percentage = null;
+  for (const entry of flagEntries()) {
+    const inMegabytes = IN_MEGABYTES.exec(entry);
+    if (inMegabytes !== null) {
+      megabytes = Number(inMegabytes[1]);
+      percentage = null;
+    }
+    const asPercentage = AS_PERCENTAGE.exec(entry);
+    if (asPercentage !== null) {
+      percentage = Number(asPercentage[1]);
+      megabytes = null;
+    }
+  }
+  const bytes = megabytes !== null && megabytes > 0 ? megabytes * MEGABYTE_BYTES : percentage !== null && percentage > 0 && percentage <= 100 ? Math.floor(memoryTheHostAllows() * percentage / 100) : null;
+  return bytes !== null && Number.isFinite(bytes) && bytes > 0 ? bytes : null;
+}
+var configuredLimit;
+function configuredLimitBytes() {
+  if (configuredLimit === void 0) configuredLimit = readConfiguredLimit();
+  return configuredLimit;
+}
 function readHeapStatistics() {
   try {
     const stats = getHeapStatistics();
     const usedBytes = stats.used_heap_size;
     const limitBytes = stats.heap_size_limit;
     if (!Number.isFinite(usedBytes) || !Number.isFinite(limitBytes) || limitBytes <= 0) return null;
-    return { usedBytes, limitBytes };
+    const available = stats.total_available_size;
+    const availableBytes = Number.isFinite(available) && available >= 0 ? available : null;
+    return { usedBytes, limitBytes, availableBytes, configuredLimitBytes: configuredLimitBytes() };
   } catch {
     return null;
   }
@@ -8421,8 +8459,9 @@ function completeGraph(state) {
 }
 function maintenanceStatus(state) {
   const storeSize = state.store.size;
-  const tombstoneRatio = storeSize > 0 ? state.tombstones.size / storeSize : 0;
   const graphCount = state.hnsw ? 1 : 0;
+  const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
+  const tombstoneRatio = graphNodes > 0 ? state.tombstones.size / graphNodes : 0;
   const estimatedCompactMs = Math.round(state.tombstones.size * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
   const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
   const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
@@ -8642,7 +8681,11 @@ function search(state, query, k, options) {
 }
 function searchWithFilter(state, query, k, options, filter) {
   if (query.length !== state.dimension) {
-    throw new Error(`Vector dimension mismatch: expected ${state.dimension}, got ${query.length}`);
+    throw new NarsilError(
+      ErrorCodes.VECTOR_DIMENSION_MISMATCH,
+      `Vector dimension mismatch: expected ${state.dimension}, got ${query.length}`,
+      { expected: state.dimension, received: query.length }
+    );
   }
   const currentLiveSize = liveSize(state);
   if (currentLiveSize === 0) return [];
@@ -11217,7 +11260,9 @@ function createDirectExecutor(options) {
       }
       case "memoryReport": {
         const usage = typeof process !== "undefined" && process.memoryUsage ? process.memoryUsage() : {};
-        return { ...usage, heapLimit: readHeapStatistics()?.limitBytes ?? null };
+        const heap = readHeapStatistics();
+        const heapLimit = heap === null ? null : heap.configuredLimitBytes ?? heap.limitBytes;
+        return { ...usage, heapLimit };
       }
       case "shutdown": {
         for (const [, entry] of indexes) {