@delali/narsil 0.2.4-next.52 → 0.2.4-next.55

package/dist/index.browser.mjs

@@ -1,6 +1,7 @@
 import { encode, decode } from '@msgpack/msgpack';
 
 // src/errors.ts
+var NARSIL_ERROR_MARKER = /* @__PURE__ */ Symbol.for("@delali/narsil.NarsilError");
 var ErrorCodes = {
   SCHEMA_INVALID_TYPE: "SCHEMA_INVALID_TYPE",
   SCHEMA_DEPTH_EXCEEDED: "SCHEMA_DEPTH_EXCEEDED",
@@ -18,6 +19,7 @@ var ErrorCodes = {
   WORKER_CRASHED: "WORKER_CRASHED",
   WORKER_BUSY: "WORKER_BUSY",
   WORKER_TIMEOUT: "WORKER_TIMEOUT",
+  WORKER_ACTION_FAILED: "WORKER_ACTION_FAILED",
   PERSISTENCE_SAVE_FAILED: "PERSISTENCE_SAVE_FAILED",
   PERSISTENCE_LOAD_FAILED: "PERSISTENCE_LOAD_FAILED",
   PERSISTENCE_DELETE_FAILED: "PERSISTENCE_DELETE_FAILED",
@@ -137,8 +139,12 @@ var NarsilError = class extends Error {
     this.name = "NarsilError";
     this.code = code;
     this.details = details ?? {};
+    Object.defineProperty(this, NARSIL_ERROR_MARKER, { value: true });
   }
 };
+function isNarsilError(value) {
+  return typeof value === "object" && value !== null && NARSIL_ERROR_MARKER in value;
+}
 function createNarsilError(code, message, details) {
   return new NarsilError(code, message, details);
 }
@@ -14325,7 +14331,11 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
       throw new Error(`Cannot insert HNSW node: vector for "${docId}" not found in VectorStore`);
     }
     if (entry.vector.length !== dimension) {
-      throw new Error(`Vector dimension mismatch: expected ${dimension}, got ${entry.vector.length}`);
+      throw new NarsilError(
+        ErrorCodes.VECTOR_DIMENSION_MISMATCH,
+        `Vector dimension mismatch: expected ${dimension}, got ${entry.vector.length}`,
+        { docId, expected: dimension, received: entry.vector.length }
+      );
     }
     insertNode(state, ord);
   }
@@ -16182,8 +16192,9 @@ function completeGraph(state) {
 }
 function maintenanceStatus(state) {
   const storeSize = state.store.size;
-  const tombstoneRatio = storeSize > 0 ? state.tombstones.size / storeSize : 0;
   const graphCount = state.hnsw ? 1 : 0;
+  const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
+  const tombstoneRatio = graphNodes > 0 ? state.tombstones.size / graphNodes : 0;
   const estimatedCompactMs = Math.round(state.tombstones.size * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
   const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
   const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
@@ -16402,7 +16413,11 @@ function search2(state, query, k, options) {
 }
 function searchWithFilter(state, query, k, options, filter) {
   if (query.length !== state.dimension) {
-    throw new Error(`Vector dimension mismatch: expected ${state.dimension}, got ${query.length}`);
+    throw new NarsilError(
+      ErrorCodes.VECTOR_DIMENSION_MISMATCH,
+      `Vector dimension mismatch: expected ${state.dimension}, got ${query.length}`,
+      { expected: state.dimension, received: query.length }
+    );
   }
   const currentLiveSize = liveSize(state);
   if (currentLiveSize === 0) return [];
@@ -19034,7 +19049,9 @@ function createDirectExecutor(options) {
       }
       case "memoryReport": {
         const usage = typeof process !== "undefined" && process.memoryUsage ? process.memoryUsage() : {};
-        return { ...usage, heapLimit: null };
+        const heap = readHeapStatistics();
+        const heapLimit = heap === null ? null : heap.configuredLimitBytes ?? heap.limitBytes;
+        return { ...usage, heapLimit };
       }
       case "shutdown": {
         for (const [, entry] of indexes) {
@@ -19910,7 +19927,7 @@ async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSIO
   if (header.envelopeFormatVersion > maxAcceptedVersion) {
     throw new NarsilError(
       ErrorCodes.ENVELOPE_VERSION_MISMATCH,
-      `This data was written by Narsil envelope format v${header.envelopeFormatVersion} and requires a newer version of Narsil. You are running ${VERSION}.`
+      `This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
     );
   }
   if (header.flags.encryptionEnabled) {
@@ -22443,6 +22460,8 @@ async function spawnWorker3() {
     if (typeof worker.unref === "function") {
       worker.unref();
     }
+    worker.on("error", () => discardWorker(worker));
+    worker.on("exit", () => discardWorker(worker));
     spawnedWorkerCount += 1;
     return worker;
   } catch {
@@ -24082,6 +24101,8 @@ var BATCH_CHUNK_SIZE = 1e3;
 var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
 var DEFAULT_OFFSET = 0;
 var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
+var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
+var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
 var MAX_INDEX_NAME_LENGTH = 255;
 var MAX_DOC_ID_LENGTH = 512;
 var MIN_SWEEP_INTERVAL_MS = 100;
@@ -24644,12 +24665,20 @@ function createInvalidationIntegration(deps) {
 }
 
 // src/engine/heap-pressure.ts
+function spentHeapFraction(heap) {
+  if (heap.configuredLimitBytes !== null) return heap.usedBytes / heap.configuredLimitBytes;
+  if (heap.availableBytes === null) return heap.usedBytes / heap.limitBytes;
+  return 1 - heap.availableBytes / heap.limitBytes;
+}
+function ceilingBytes(heap) {
+  return heap.configuredLimitBytes ?? heap.limitBytes;
+}
 function createHeapPressureNotifier(deps) {
   let warned = false;
   function check(indexName) {
     const heap = deps.readHeap();
     if (heap === null) return;
-    const fraction = heap.usedBytes / heap.limitBytes;
+    const fraction = spentHeapFraction(heap);
     if (warned) {
       if (fraction < HEAP_PRESSURE_REARM_FRACTION) warned = false;
       return;
@@ -24659,7 +24688,7 @@ function createHeapPressureNotifier(deps) {
     deps.emit({
       indexName,
       heapUsed: heap.usedBytes,
-      heapLimit: heap.limitBytes,
+      heapLimit: ceilingBytes(heap),
       estimatedMemoryBytes: deps.estimateIndexBytes(indexName)
     });
   }
@@ -25407,7 +25436,7 @@ function createWorkerExecutor(worker, config) {
     clearTimeout(entry.timeoutId);
     pending.delete(response.requestId);
     if (response.type === "error") {
-      entry.reject(new NarsilError(response.code, response.message));
+      entry.reject(new NarsilError(response.code, response.message, response.details));
     } else {
       entry.resolve(response.data);
     }
@@ -25963,6 +25992,7 @@ function scheduleRequestThreadPoolRestart(state) {
   if (typeof timer.unref === "function") timer.unref();
 }
 async function serveRequestsOnWorkers(state, listener) {
+  if (!state.workersEnabled) return 0;
   state.requestThreads = listener;
   state.keywordWorkerCount = resolveRequestThreadCount(state.config?.workers?.count);
   if (state.vectorCopyPolicy !== void 0) {
@@ -25985,6 +26015,7 @@ async function serveRequestsOnWorkers(state, listener) {
   return pool.workerCount;
 }
 function requestThreadCountOf(state) {
+  if (!state.workersEnabled) return 0;
   return resolveRequestThreadCount(state.config?.workers?.count);
 }
 function stopRequestThreads(state) {
@@ -26694,32 +26725,37 @@ function now() {
   return Date.now();
 }
 function validateIndexName(name) {
-  if (!name || name.length === 0) {
-    throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
+  const given = name;
+  if (typeof given !== "string" || name.length === 0) {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index name must be a string of at least one character", {
+      received: given === null ? "null" : typeof given
+    });
   }
   if (name.length > MAX_INDEX_NAME_LENGTH) {
-    throw new NarsilError(
-      ErrorCodes.INDEX_NOT_FOUND,
-      `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
-      {
-        indexName: name,
-        length: name.length
-      }
-    );
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`, {
+      indexName: name,
+      length: name.length
+    });
   }
   if (!INDEX_NAME_PATTERN.test(name)) {
     throw new NarsilError(
-      ErrorCodes.INDEX_NOT_FOUND,
+      ErrorCodes.CONFIG_INVALID,
       `Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
       { indexName: name }
     );
   }
   if (name.includes("..")) {
-    throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name "${name}" must not contain ".."`, { indexName: name });
   }
 }
 function validateDocId(docId) {
-  if (!docId || docId.length === 0) {
+  const given = docId;
+  if (typeof given !== "string") {
+    throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must be a string", {
+      received: given === null ? "null" : Array.isArray(given) ? "array" : typeof given
+    });
+  }
+  if (docId.length === 0) {
     throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
   }
   if (docId.length > MAX_DOC_ID_LENGTH) {
@@ -26760,6 +26796,16 @@ function requirePositiveSafeInteger(field, value) {
     throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${field} must be a positive safe integer`, { field, value });
   }
 }
+function validateBootstrapModule(bootstrapModule) {
+  if (bootstrapModule === void 0) return;
+  const absolute = bootstrapModule.startsWith("/") || ABSOLUTE_WINDOWS_PATH_PATTERN.test(bootstrapModule) || MODULE_URL_SCHEME_PATTERN.test(bootstrapModule);
+  if (absolute) return;
+  throw new NarsilError(
+    ErrorCodes.CONFIG_INVALID,
+    `workers.bootstrapModule is "${bootstrapModule}", which a worker would resolve against Narsil's own directory, so pass an absolute path or a URL such as new URL('./bootstrap.js', import.meta.url).href`,
+    { bootstrapModule }
+  );
+}
 function validateWorkerConfig(workers, lifecycle) {
   if (workers === void 0) return;
   requirePositiveSafeInteger("workers.count", workers.count);
@@ -26770,6 +26816,7 @@ function validateWorkerConfig(workers, lifecycle) {
       mainCopyQueries: workers.mainCopyQueries
     });
   }
+  validateBootstrapModule(workers.bootstrapModule);
   const closeAfter = lifecycle?.idleTimeoutMs;
   if (workers.idleTimeoutMs !== void 0 && closeAfter !== void 0 && workers.idleTimeoutMs > closeAfter) {
     throw new NarsilError(
@@ -27036,6 +27083,16 @@ function createEngineCore(config, hooks) {
 }
 
 // src/engine/index-lifecycle.ts
+async function runLifecycleHook(core, hookName, context) {
+  try {
+    await core.pluginRegistry.runHook(hookName, context);
+  } catch (err) {
+    console.warn(`${hookName} plugin hook error:`, err instanceof Error ? err.message : String(err));
+  }
+}
+async function announceIndexCreated(core, name, config) {
+  await runLifecycleHook(core, "onIndexCreate", { indexName: name, config });
+}
 async function createEngineIndex(core, config, name, indexConfig, indexUuid) {
   core.guardShutdown();
   validateIndexName(name);
@@ -27107,7 +27164,7 @@ async function createEngineIndex(core, config, name, indexConfig, indexUuid) {
   if (core.durability) {
     await core.durability.manager.persistMetadata(name);
   }
-  await core.pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
+  await runLifecycleHook(core, "onIndexCreate", { indexName: name, config: indexConfig });
 }
 async function dropEngineIndex(core, name) {
   core.guardShutdown();
@@ -27127,7 +27184,7 @@ async function dropEngineIndex(core, name) {
     core.watermarkNotifier.forget(name);
     core.analysisRebuild.clearStale(name);
   });
-  await core.pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
+  await runLifecycleHook(core, "onIndexDrop", { indexName: name, config: entry.config });
 }
 function registerEngineEmbeddingAdapter(core, name, adapter) {
   core.guardShutdown();
@@ -27549,7 +27606,15 @@ async function awaitWriteVisibility(ctx, indexName) {
 async function insertDocument(ctx, indexName, document2, docId, options) {
   ctx.guardShutdown();
   const entry = ctx.requireIndex(indexName);
-  const resolvedDocId = docId ?? providedDocId(document2) ?? ctx.idGenerator();
+  const embeddedDocId = providedDocId(document2);
+  if (docId !== void 0 && embeddedDocId !== void 0 && embeddedDocId !== docId) {
+    throw new NarsilError(
+      ErrorCodes.DOC_VALIDATION_FAILED,
+      `The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
+      { docId, documentId: embeddedDocId }
+    );
+  }
+  const resolvedDocId = docId ?? embeddedDocId ?? ctx.idGenerator();
   validateDocId(resolvedDocId);
   await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document: document2 });
   if (entry.config.required && entry.config.required.length > 0) {
@@ -27820,13 +27885,12 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
 // src/engine/mutations/insert-batch-documents.ts
 async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed) {
   const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
-  const chunkSize = hasAfterHook ? 1 : BATCH_CHUNK_SIZE;
   const succeeded = [];
   const buffered = /* @__PURE__ */ new Set();
   const touchedVectorFields = /* @__PURE__ */ new Set();
-  for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += chunkSize) {
+  for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
     if (ctx.abortController.signal.aborted) break;
-    const chunk = admitted.slice(chunkStart, chunkStart + chunkSize);
+    const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
     const applications = await applyInsertChunk(ctx, indexName, chunk, options);
     for (let i = 0; i < chunk.length; i++) {
       const doc = chunk[i];
@@ -27853,8 +27917,8 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
       }
       succeeded.push(doc.docId);
     }
-    const applied = chunkStart + chunkSize;
-    if (applied < admitted.length && applied % BATCH_CHUNK_SIZE === 0) {
+    const applied = chunkStart + BATCH_CHUNK_SIZE;
+    if (applied < admitted.length) {
       await new Promise((r) => setTimeout(r, 0));
     }
   }
@@ -28223,7 +28287,6 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
   const failed = [];
   const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
   const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
-  const documentByDocument = hasBeforeHook || hasAfterHook;
   const required = entry.config.required;
   const batchManager = ctx.requireManager(indexName);
   const batchVecIndexes = batchManager.getVectorIndexes();
@@ -28303,9 +28366,6 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
       } catch (err) {
         failed.push({ docId: batchDocId, error: asBatchInsertError(err) });
       }
-      if (documentByDocument && prepared.length > 0) {
-        await applyPrepared(prepared.splice(0, prepared.length));
-      }
     }
     if (prepared.length > 0) await applyPrepared(prepared);
     if (chunkEnd < documents.length) {
@@ -28420,12 +28480,10 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
   const succeeded = [];
   const failed = [];
   const manager = ctx.requireManager(indexName);
-  const hooked = ctx.pluginRegistry.hasHooks("beforeRemove") || ctx.pluginRegistry.hasHooks("afterRemove");
-  const chunkSize = hooked ? 1 : BATCH_CHUNK_SIZE;
   manager.beginBatchRemove();
   try {
-    for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += chunkSize) {
-      const chunkEnd = Math.min(chunkStart + chunkSize, docIds.length);
+    for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
+      const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
       const prepared = [];
       for (let i = chunkStart; i < chunkEnd; i++) {
         try {
@@ -28452,13 +28510,14 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
         if (!removal.buffered) await settleRemoval(ctx, indexName, removal.docId);
         succeeded.push(removal.docId);
       }
-      if (chunkEnd < docIds.length && chunkEnd % BATCH_CHUNK_SIZE === 0) {
+      if (chunkEnd < docIds.length) {
         await new Promise((r) => setTimeout(r, 0));
       }
     }
   } finally {
     manager.endBatchRemove();
   }
+  ctx.checkHeapPressure(indexName);
   if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
   return { succeeded, failed };
 }
@@ -28619,10 +28678,8 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
   const updateBatchManager = ctx.requireManager(indexName);
   const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
   const touchedVectorFields = /* @__PURE__ */ new Set();
-  const hooked = ctx.pluginRegistry.hasHooks("beforeUpdate") || ctx.pluginRegistry.hasHooks("afterUpdate");
-  const chunkSize = hooked ? 1 : BATCH_CHUNK_SIZE;
-  for (let chunkStart = 0; chunkStart < updates.length; chunkStart += chunkSize) {
-    const chunkEnd = Math.min(chunkStart + chunkSize, updates.length);
+  for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
+    const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
     const prepared = [];
     for (let i = chunkStart; i < chunkEnd; i++) {
       try {
@@ -28652,7 +28709,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
       }
       succeeded.push(update.docId);
     }
-    if (chunkEnd < updates.length && chunkEnd % BATCH_CHUNK_SIZE === 0) {
+    if (chunkEnd < updates.length) {
       await new Promise((r) => setTimeout(r, 0));
     }
   }
@@ -29287,6 +29344,7 @@ async function restoreFromSnapshot(indexName, data, deps) {
       await deps.durability.manager.persistMetadata(indexName);
       await deps.durability.manager.checkpoint(indexName);
     }
+    await deps.announceIndexCreated(indexName, indexConfig);
   } catch (err) {
     try {
       deps.executor.dropIndex(indexName);
@@ -29358,13 +29416,14 @@ function readProcessMemory() {
   }
   try {
     const usage = proc.memoryUsage();
+    const heap = readHeapStatistics();
     if (!Number.isFinite(usage.heapUsed) || !Number.isFinite(usage.heapTotal) || !Number.isFinite(usage.external) || !Number.isFinite(usage.rss)) {
       return null;
     }
     return {
       heapUsed: usage.heapUsed,
       heapTotal: usage.heapTotal,
-      heapLimit: readHeapStatistics()?.limitBytes ?? null,
+      heapLimit: heap === null ? null : heap.configuredLimitBytes ?? heap.limitBytes,
       external: usage.external,
       rss: usage.rss
     };
@@ -29832,21 +29891,19 @@ function findDensestWindow(positions, text, windowSize) {
 }
 function applySnippetTruncation(taggedText, originalText, positions, maxLen, preTag, postTag) {
   if (maxLen <= 0 || originalText.length <= maxLen) {
-    return { snippet: taggedText, adjustedPositions: positions };
+    return { snippet: taggedText, visiblePositions: positions };
   }
   const window2 = findDensestWindow(positions, originalText, maxLen);
   const snippetRaw = originalText.substring(window2.start, window2.end);
-  const adjustedPositions = [];
+  const visiblePositions = [];
+  const withinSnippet = [];
   for (const pos of positions) {
-    if (pos.start >= window2.start && pos.end <= window2.end) {
-      adjustedPositions.push({
-        start: pos.start - window2.start,
-        end: pos.end - window2.start
-      });
-    }
+    if (pos.start < window2.start || pos.end > window2.end) continue;
+    visiblePositions.push(pos);
+    withinSnippet.push({ start: pos.start - window2.start, end: pos.end - window2.start });
   }
   let snippet = snippetRaw;
-  const sortedDesc = [...adjustedPositions].sort((a, b) => b.start - a.start);
+  const sortedDesc = [...withinSnippet].sort((a, b) => b.start - a.start);
   for (const pos of sortedDesc) {
     snippet = snippet.substring(0, pos.start) + preTag + snippet.substring(pos.start, pos.end) + postTag + snippet.substring(pos.end);
   }
@@ -29854,7 +29911,7 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
   const needsTrailingEllipsis = window2.end < originalText.length;
   if (needsLeadingEllipsis) snippet = `...${snippet}`;
   if (needsTrailingEllipsis) snippet = `${snippet}...`;
-  return { snippet, adjustedPositions };
+  return { snippet, visiblePositions };
 }
 function highlightField(text, queryTokens, language, options) {
   const preTag = options?.preTag ?? "<mark>";
@@ -29901,7 +29958,7 @@ function highlightField(text, queryTokens, language, options) {
     return { snippet: text, positions: [] };
   }
   if (maxSnippetLength > 0 && text.length > maxSnippetLength) {
-    const { snippet, adjustedPositions } = applySnippetTruncation(
+    const { snippet, visiblePositions } = applySnippetTruncation(
       text,
       text,
       mergedPositions,
@@ -29909,7 +29966,7 @@ function highlightField(text, queryTokens, language, options) {
       preTag,
       postTag
     );
-    return { snippet, positions: adjustedPositions };
+    return { snippet, positions: visiblePositions };
   }
   let result = text;
   const sortedDesc = [...mergedPositions].sort((a, b) => b.start - a.start);
@@ -29920,7 +29977,7 @@ function highlightField(text, queryTokens, language, options) {
 }
 
 // src/engine/query/highlight.ts
-function applyHighlights(hits, params, language, analysis) {
+function applyHighlights(hits, params, language, analysis, readStoredDocument) {
   if (!params.highlight) return;
   const queryTokenResult = tokenize(params.term ?? "", language, {
     stem: true,
@@ -29946,9 +30003,14 @@ function applyHighlights(hits, params, language, analysis) {
   }
   for (const hit of hits) {
     const highlights = {};
+    const projected = hit.document;
+    let stored;
     for (const field of params.highlight.fields) {
-      const doc = hit.document;
-      const fieldValue = doc[field];
+      let fieldValue = projected[field];
+      if (typeof fieldValue !== "string" && readStoredDocument !== void 0) {
+        stored ??= readStoredDocument(hit.id);
+        fieldValue = stored?.[field];
+      }
       if (typeof fieldValue === "string") {
         highlights[field] = highlightField(fieldValue, queryTokenResult.tokens, language, {
           preTag: params.highlight.preTag,
@@ -30042,6 +30104,32 @@ function vectorSearchersOf(context) {
 function resolveVectorIndex(context, fieldName) {
   return vectorSearchersOf(context).get(fieldName);
 }
+var SEARCH_MODES = /* @__PURE__ */ new Set(["fulltext", "vector", "hybrid"]);
+function requireKnownMode(params) {
+  if (params.mode === void 0 || SEARCH_MODES.has(params.mode)) return;
+  throw new NarsilError(
+    ErrorCodes.SEARCH_INVALID_MODE,
+    `A search takes the mode "fulltext", "vector", or "hybrid", while this query names "${String(params.mode)}"`,
+    { mode: String(params.mode), modes: [...SEARCH_MODES] }
+  );
+}
+function requireVectorSearchable(params, context, needsVector) {
+  if (!needsVector) return;
+  const field = params.vector?.field;
+  if (field === void 0) {
+    throw new NarsilError(
+      ErrorCodes.SEARCH_INVALID_MODE,
+      `A "${params.mode}" query needs a vector clause naming the field to search`,
+      { mode: params.mode }
+    );
+  }
+  if (vectorSearchersOf(context).has(field)) return;
+  throw new NarsilError(
+    ErrorCodes.SEARCH_INVALID_FIELD,
+    `Field "${field}" has no vector index, so a vector search cannot use it`,
+    { field, vectorFields: [...vectorSearchersOf(context).keys()] }
+  );
+}
 function clampAlpha(alpha) {
   if (alpha === void 0) return 0.5;
   if (!Number.isFinite(alpha)) return 0.5;
@@ -30329,6 +30417,7 @@ async function executeQuery(params, context) {
   const offset = clampOffset(params.offset);
   requireWithinResultWindow(limit, offset);
   const sortFields = normalizeSort(params.sort);
+  requireKnownMode(params);
   const sortSignature = sortSignatureOf(params.sort);
   const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
   const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
@@ -30342,6 +30431,7 @@ async function executeQuery(params, context) {
     );
   }
   requireSortableFields(params.sort, config.schema);
+  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
   let paginated;
   let nextCursor;
   let count;
@@ -30465,7 +30555,9 @@ async function executeQuery(params, context) {
     }
   }
   if (params.highlight) {
-    applyHighlights(paginated, params, language, manager.analysis);
+    const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
+    const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
+    applyHighlights(paginated, params, language, manager.analysis, readStoredDocument);
   }
   const elapsed = now() - startTime;
   return {
@@ -30481,10 +30573,12 @@ async function executeQuery(params, context) {
 async function executePreflight(params, context) {
   const { manager, language, config, workerSearch, indexName } = context;
   const startTime = now();
+  requireKnownMode(params);
   const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
   const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
   const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
   const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
+  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
   const requestedVectorField = params.vector?.field;
   const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
   let totalMatched;
@@ -30631,13 +30725,14 @@ async function runEngineQuery(core, indexName, params, options) {
   try {
     const entry = core.requireIndex(indexName);
     const manager = core.requireManager(indexName);
+    const scoped = scopedParams(params, options);
+    await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
     const resolvedParams = await resolveVectorText(
-      scopedParams(params, options),
+      scoped,
       entry.embeddingAdapter,
       core.abortController.signal,
       entry.embeddingAdapterName
     );
-    await core.pluginRegistry.runHook("beforeSearch", { indexName, params: resolvedParams });
     const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
     const result = await executeQuery(resolvedParams, {
       manager,
@@ -30649,14 +30744,13 @@ async function runEngineQuery(core, indexName, params, options) {
       partitionIds: options?.partitionIds,
       cursorBinding: queryBindingOf(params)
     });
-    try {
-      await core.pluginRegistry.runHook("afterSearch", {
-        indexName,
-        params: resolvedParams,
-        results: result
-      });
-    } catch (err) {
-      console.warn("afterSearch plugin hook error:", err);
+    if (core.pluginRegistry.hasHooks("afterSearch")) {
+      const observed = { ...result, hits: [...result.hits] };
+      try {
+        await core.pluginRegistry.runHook("afterSearch", { indexName, params: resolvedParams, results: observed });
+      } catch (err) {
+        console.warn("afterSearch plugin hook error:", err);
+      }
     }
     if (core.analysisRebuild.isStale(indexName)) {
       result.analysisStale = true;
@@ -30672,8 +30766,10 @@ async function runEnginePreflight(core, indexName, params, options) {
   try {
     const entry = core.requireIndex(indexName);
     const manager = core.requireManager(indexName);
+    const scoped = scopedParams(params, options);
+    await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
     const resolvedParams = await resolveVectorText(
-      scopedParams(params, options),
+      scoped,
       entry.embeddingAdapter,
       core.abortController.signal,
       entry.embeddingAdapterName
@@ -30887,6 +30983,7 @@ function createNarsilFromCore(core, config) {
         indexRegistry,
         getVectorFieldPaths,
         dropIndex: narsil.dropIndex.bind(narsil),
+        announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
         requireManager,
         durability,
         filesystemDurability: core.filesystemDurability,
@@ -31043,4 +31140,4 @@ async function createNarsil(config) {
 // src/index.ts
 var VERSION = "0.1.0";
 
-export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache };
+export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache };

package/dist/index.d.ts

@@ -1,12 +1,12 @@
-import { b as StopWordOverride, C as CustomTokenizer } from './schema-OSWaaZzx.js';
-export { A as AnyDocument, B as BM25Params, E as EmbeddingFieldConfig, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, S as SchemaDefinition, c as ScoringMode, V as VectorIndexConfig, d as VectorQuantizationMode, e as VectorStorageMode, W as WriteOptions } from './schema-OSWaaZzx.js';
+import { b as StopWordOverride, C as CustomTokenizer } from './schema-mLpXZBEh.js';
+export { A as AnyDocument, B as BM25Params, E as EmbeddingFieldConfig, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, S as SchemaDefinition, c as ScoringMode, V as VectorIndexConfig, d as VectorQuantizationMode, e as VectorStorageMode, W as WriteOptions } from './schema-mLpXZBEh.js';
 import { L as LanguageModule } from './language-DB8KG1z5.js';
 export { T as TokenizerConfig } from './language-DB8KG1z5.js';
-export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, k as ComparisonFilter, D as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, l as FacetResult, m as FieldFilter, n as FilterExpression, G as GeoFilter, o as GeoPolygonFilter, p as GeoRadiusFilter, q as GroupConfig, r as GroupReducer, s as GroupResult, H as HighlightConfig, t as HighlightMatch, u as Hit, v as HybridConfig, I as IndexInfo, f as IndexStats, L as ListParams, g as ListResult, w as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, P as PartitionStatsResult, h as PreflightResult, x as PresenceFilter, y as ProcessMemoryReport, z as QueryCoverage, Q as QueryParams, e as QueryResult, J as ScoreComponents, K as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, O as SortField, R as SortSpec, T as StringFilter, i as SuggestParams, j as SuggestResult, U as TermMatchPolicy, V as VectorMaintenanceResult, W as VectorQueryConfig, X as WorkerCopyReport } from './memory-OmgkX_Z2.js';
-import { N as NarsilConfig } from './config-BcYZkdMw.js';
-export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-BcYZkdMw.js';
-import { a as Narsil } from './engine-zhms1qId.js';
-export { D as DocumentWriteOperations, I as IndexLifecycleOperations, N as NarsilEventMap } from './engine-zhms1qId.js';
+export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, l as ComparisonFilter, D as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, m as FacetResult, n as FieldFilter, o as FilterExpression, G as GeoFilter, p as GeoPolygonFilter, q as GeoRadiusFilter, r as GroupConfig, s as GroupReducer, t as GroupResult, H as HighlightConfig, u as HighlightMatch, v as Hit, w as HybridConfig, I as IndexInfo, f as IndexStats, L as ListParams, g as ListResult, x as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, P as PartitionStatsResult, h as PreflightResult, y as PresenceFilter, z as ProcessMemoryReport, J as QueryCoverage, Q as QueryParams, e as QueryResult, K as ScoreComponents, O as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, R as SortField, T as SortSpec, U as StringFilter, j as SuggestParams, k as SuggestResult, W as TermMatchPolicy, V as VectorMaintenanceResult, X as VectorQueryConfig, Y as WorkerCopyReport, i as isNarsilError } from './memory-xrXOZMbw.js';
+import { N as NarsilConfig } from './config-bXxnJrQc.js';
+export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-bXxnJrQc.js';
+import { a as Narsil } from './engine-Dj4I91Jv.js';
+export { D as DocumentWriteOperations, I as IndexLifecycleOperations, N as NarsilEventMap } from './engine-Dj4I91Jv.js';
 export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, b as PartitionStatistics, P as PersistenceAdapter } from './adapters-79OIa4xm.js';
 export { i as isSimdAvailable } from './simd-DW5AtJ6B.js';