@delali/narsil 0.3.1-next.73 → 0.3.1-next.75

package/dist/{chunk-ZULSEZCE.mjs → chunk-WL2GDSCV.mjs}

@@ -1,7 +1,9 @@
-import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, requireValidFilter, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex, fuzzyTermMatches } from './chunk-3YKWSO54.mjs';
-import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, DEFAULT_LIMIT, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_OFFSET, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isWordIndexedFieldType, isGeopointOrVectorType, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-AK3BLGCI.mjs';
+import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, requireValidFilter, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex } from './chunk-NAMGDK45.mjs';
+import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_MAX_PER_GROUP, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_OFFSET, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-V3B24XR7.mjs';
 import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
-import { holdsUnpairedSurrogate, compareCodePoints, compareComparableKeys } from './chunk-LJYTW7GT.mjs';
+import { tokenize, fuzzyTermMatches } from './chunk-UOJIV2NG.mjs';
+import { partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, flattenSchema, isWordIndexedFieldType, isGeopointOrVectorType } from './chunk-356UOTK2.mjs';
+import { holdsUnpairedSurrogate, compareCodePoints, compareComparableKeys, codePointCount } from './chunk-IBR6634K.mjs';
 import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
 
 // src/search/cursor-binding.ts
@@ -129,7 +131,7 @@ function boundVectorOf(vector) {
   return bound;
 }
 var FILTERS_SLOT = 2;
-var BOUND_SLOT_COUNT = 15;
+var BOUND_SLOT_COUNT = 16;
 function bindingOf(values) {
   const stream = { bytes: new Uint8Array(256), length: 0 };
   for (const value of values) {
@@ -148,6 +150,7 @@ function queryBindingOf(params) {
     params.tolerance,
     params.prefixLength,
     params.prefix,
+    params.partialWords,
     params.exact,
     params.pinned,
     params.mode,
@@ -316,9 +319,18 @@ function requireKnownMode(params) {
     { mode: String(params.mode), modes: [...SEARCH_MODES] }
   );
 }
+function requireBooleanPartialWords(params) {
+  const setting = params.partialWords;
+  if (setting === void 0 || typeof setting === "boolean") return;
+  throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The "partialWords" query option takes true or false', {
+    option: "partialWords",
+    received: setting === null ? "null" : typeof setting
+  });
+}
 function requireValidQueryOptions(params, config) {
   const strict = config.strict === true;
   requireValidTypoTolerance(params);
+  requireBooleanPartialWords(params);
   requireSortableFields(params.sort, config.schema, strict);
   requireValidQueryFilter(params.filters, config.schema);
   requireUsableQueryFields(params, config.schema, strict);
@@ -424,6 +436,16 @@ function validatePartitionConfig(partitions) {
     });
   }
 }
+function requireSurfaceFormsForPartialFields(schema, surfaceForms) {
+  if (surfaceForms !== false) return;
+  const partialFields = partialFieldsOf(schema);
+  if (partialFields.length === 0) return;
+  throw new NarsilError(
+    ErrorCodes.CONFIG_INVALID,
+    `Field "${partialFields[0]}" declares "partial", which compares words as the documents wrote them, so its index needs surfaceForms left on`,
+    { fields: [...partialFields], surfaceForms: false }
+  );
+}
 function validatePatternValueLimit(limit) {
   if (limit === void 0) return;
   if (typeof limit !== "number" || !Number.isInteger(limit) || limit < MIN_PATTERN_VALUE_LIMIT || limit > MAX_PATTERN_VALUE_LIMIT) {
@@ -813,6 +835,31 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
   if (needsTrailingEllipsis) snippet = `${snippet}...`;
   return { snippet, visiblePositions };
 }
+function containsPartialWord(spelling, words) {
+  for (const word of words) {
+    if (spelling.length > word.length && spelling.includes(word)) return true;
+  }
+  return false;
+}
+function partialWordMatcher(text, language, partial) {
+  const indexed = tokenize(text, language, {
+    stem: true,
+    removeStopWords: true,
+    collectSurfaces: true,
+    customTokenizer: partial.customTokenizer,
+    stopWordOverride: partial.stopWordOverride
+  });
+  const positions = /* @__PURE__ */ new Set();
+  const spellings = /* @__PURE__ */ new Set();
+  for (let at = 0; at < indexed.tokens.length; at++) {
+    const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
+    if (!containsPartialWord(spelling, partial.words)) continue;
+    positions.add(indexed.tokens[at].position);
+    spellings.add(spelling);
+  }
+  if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
+  return (position) => positions.has(position);
+}
 function highlightField(text, queryTokens, language, options) {
   const preTag = options?.preTag ?? "<mark>";
   const postTag = options?.postTag ?? "</mark>";
@@ -849,11 +896,13 @@ function highlightField(text, queryTokens, language, options) {
     matchByFieldToken.set(fieldToken, matches);
     return matches;
   };
+  const matchesInside = options?.partialWords === void 0 ? null : partialWordMatcher(text, language, options.partialWords);
   for (let i = 0; i < fieldResult.tokens.length; i++) {
     if (i >= charOffsets.length) break;
     const fieldToken = fieldResult.tokens[i].token;
     const prefixMatch = prefixToken !== void 0 && fieldToken.startsWith(prefixToken);
-    if (prefixMatch || matchesQuery(fieldToken)) {
+    const insideMatch = matchesInside?.(fieldResult.tokens[i].position, fieldToken) === true;
+    if (prefixMatch || insideMatch || matchesQuery(fieldToken)) {
       matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
     }
   }
@@ -885,7 +934,39 @@ function highlightField(text, queryTokens, language, options) {
 }
 
 // src/engine/query/highlight.ts
-function applyHighlights(hits, params, language, analysis, readStoredDocument) {
+function writtenWordsOf(params, language, analysis) {
+  return tokenize(params.term ?? "", language, {
+    stem: false,
+    removeStopWords: true,
+    customTokenizer: analysis.customTokenizer,
+    stopWordOverride: analysis.stopWords
+  }).tokens;
+}
+function partialWordTagging(params, schema, queryTokens, written, analysis) {
+  if (params.partialWords !== true || params.exact === true) return null;
+  const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
+  if (fields.length === 0) return null;
+  const writtenAt = /* @__PURE__ */ new Map();
+  for (const word of written) {
+    if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
+  }
+  const seenTokens = /* @__PURE__ */ new Set();
+  const words = [];
+  for (const queryToken of queryTokens) {
+    if (seenTokens.has(queryToken.token)) continue;
+    seenTokens.add(queryToken.token);
+    const word = writtenAt.get(queryToken.position);
+    if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
+    words.push(word);
+    if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
+  }
+  if (words.length === 0) return null;
+  return {
+    fields: new Set(fields),
+    highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
+  };
+}
+function applyHighlights(hits, params, language, analysis, schema, readStoredDocument) {
   if (!params.highlight) return;
   const queryTokenResult = tokenize(params.term ?? "", language, {
     stem: true,
@@ -893,17 +974,15 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
     customTokenizer: analysis.customTokenizer,
     stopWordOverride: analysis.stopWords
   });
+  const queryTokens = queryTokenResult.tokens;
+  const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
+  const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
+  const written = usesPrefix || usesPartialWords ? writtenWordsOf(params, language, analysis) : [];
   let prefixToken;
   let completedToken;
-  if (params.prefix === true && params.exact !== true && queryTokenResult.tokens.length > 0) {
-    const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
-    const unstemmed = tokenize(params.term ?? "", language, {
-      stem: false,
-      removeStopWords: true,
-      customTokenizer: analysis.customTokenizer,
-      stopWordOverride: analysis.stopWords
-    });
-    for (const t of unstemmed.tokens) {
+  if (usesPrefix) {
+    const lastToken = queryTokens[queryTokens.length - 1];
+    for (const t of written) {
       if (t.position === lastToken.position) {
         prefixToken = t.token;
         completedToken = lastToken.token;
@@ -911,6 +990,7 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
       }
     }
   }
+  const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
   for (const hit of hits) {
     const highlights = {};
     const projected = hit.document;
@@ -922,14 +1002,15 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
         fieldValue = stored?.[field];
       }
       if (typeof fieldValue === "string") {
-        highlights[field] = highlightField(fieldValue, queryTokenResult.tokens, language, {
+        highlights[field] = highlightField(fieldValue, queryTokens, language, {
           preTag: params.highlight.preTag,
           postTag: params.highlight.postTag,
           maxSnippetLength: params.highlight.maxSnippetLength,
           prefixToken,
           completedToken,
           tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
-          prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
+          prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
+          partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0
         });
       }
     }
@@ -1439,7 +1520,7 @@ async function executeQuery(params, callerContext) {
   if (params.highlight) {
     const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
     const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
-    applyHighlights(paginated, params, language, manager.analysis, readStoredDocument);
+    applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument);
   }
   const elapsed = now() - startTime;
   return {
@@ -1557,4 +1638,4 @@ function pickDisplaySurface(surfaceOccurrences) {
   return best;
 }
 
-export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, resolveHybridFusion, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
+export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };