@delali/narsil 0.3.1-next.73 → 0.3.1-next.74

package/dist/chunk-I23G3IZC.mjs

@@ -0,0 +1,593 @@
+import { isWordIndexedBase, flattenSchema } from './chunk-Z3DQPOJQ.mjs';
+import { createNarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
+
+// src/core/tokenizer/constants.ts
+var TOKEN_CACHE_SIZE_FLOOR = 5e4;
+var TOKEN_CACHE_SIZE_CEILING = 2e6;
+var TOKEN_CACHE_BYTES_PER_ENTRY = 200;
+var NODE_TOKEN_CACHE_ENTRIES = 1e6;
+var UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES = 2e5;
+var CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION = 0.05;
+var SMALL_DEVICE_MEMORY_GB = 1;
+var MEDIUM_DEVICE_MEMORY_GB = 4;
+var SMALL_DEVICE_TOKEN_CACHE_ENTRIES = 1e5;
+var MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES = 25e4;
+var LARGE_DEVICE_TOKEN_CACHE_ENTRIES = 5e5;
+var DEFAULT_MIN_TOKEN_LENGTH = 1;
+
+// src/core/tokenizer/normalize.ts
+var DOTTED_CAPITAL_I = "\u0130";
+var COMBINING_DOT_ABOVE = "\u0307";
+var ARMENIAN_MARKS = /[\u055B-\u055F]/;
+var FOLDED_FORMS = /\u0307|[\u0130\u055B-\u055F\u2018\u2019\u02BC\u02BB]/g;
+var WIDTH_FORMS = /[\uFF01-\uFF5E\uFF61-\uFF9F]/;
+var WIDTH_FORMS_GLOBAL = /[\uFF01-\uFF5E\uFF61-\uFF9F]/g;
+var FULLWIDTH_ASCII_END = 65374;
+var FULLWIDTH_ASCII_OFFSET = 65248;
+var HALFWIDTH_KANA_START = 65377;
+var HALFWIDTH_VOICED_MARK = 65438;
+var HALFWIDTH_SEMI_VOICED_MARK = 65439;
+var COMBINING_VOICED_MARK = "\u3099";
+var COMBINING_SEMI_VOICED_MARK = "\u309A";
+var HALFWIDTH_KANA_TARGETS = "\u3002\u300C\u300D\u3001\u30FB\u30F2\u30A1\u30A3\u30A5\u30A7\u30A9\u30E3\u30E5\u30E7\u30C3\u30FC\u30A2\u30A4\u30A6\u30A8\u30AA\u30AB\u30AD\u30AF\u30B1\u30B3\u30B5\u30B7\u30B9\u30BB\u30BD\u30BF\u30C1\u30C4\u30C6\u30C8\u30CA\u30CB\u30CC\u30CD\u30CE\u30CF\u30D2\u30D5\u30D8\u30DB\u30DE\u30DF\u30E0\u30E1\u30E2\u30E4\u30E6\u30E8\u30E9\u30EA\u30EB\u30EC\u30ED\u30EF\u30F3";
+function foldForm(match) {
+  if (match === DOTTED_CAPITAL_I) return "i";
+  if (match === COMBINING_DOT_ABOVE || ARMENIAN_MARKS.test(match)) return "";
+  return "'";
+}
+function foldWidth(match) {
+  const codePoint = match.charCodeAt(0);
+  if (codePoint <= FULLWIDTH_ASCII_END) return String.fromCharCode(codePoint - FULLWIDTH_ASCII_OFFSET);
+  if (codePoint === HALFWIDTH_VOICED_MARK) return COMBINING_VOICED_MARK;
+  if (codePoint === HALFWIDTH_SEMI_VOICED_MARK) return COMBINING_SEMI_VOICED_MARK;
+  return HALFWIDTH_KANA_TARGETS[codePoint - HALFWIDTH_KANA_START];
+}
+function isAscii(text) {
+  for (let i = 0; i < text.length; i++) {
+    if (text.charCodeAt(i) > 127) return false;
+  }
+  return true;
+}
+function foldWidthForms(text) {
+  if (!WIDTH_FORMS.test(text)) return text;
+  return text.replace(WIDTH_FORMS_GLOBAL, foldWidth);
+}
+function normalizeForSplitting(text) {
+  if (isAscii(text)) return text.toLowerCase();
+  return foldWidthForms(text).normalize("NFC").replace(FOLDED_FORMS, foldForm).toLowerCase();
+}
+function stripDiacritics(text) {
+  return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+}
+
+// src/core/tokenizer/cache.ts
+function clampCacheSize(entries) {
+  return Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(entries, TOKEN_CACHE_SIZE_CEILING));
+}
+function computeDefaultCacheSize() {
+  try {
+    if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
+      const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
+      if (constrainedMemory > 0 && constrainedMemory < Number.MAX_SAFE_INTEGER) {
+        const budget = constrainedMemory * CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION;
+        return clampCacheSize(Math.floor(budget / TOKEN_CACHE_BYTES_PER_ENTRY));
+      }
+      return clampCacheSize(NODE_TOKEN_CACHE_ENTRIES);
+    }
+    if (typeof navigator !== "undefined") {
+      const mem = navigator.deviceMemory;
+      if (typeof mem === "number" && mem > 0) {
+        let entries;
+        if (mem <= SMALL_DEVICE_MEMORY_GB) entries = SMALL_DEVICE_TOKEN_CACHE_ENTRIES;
+        else if (mem <= MEDIUM_DEVICE_MEMORY_GB) entries = MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES;
+        else entries = LARGE_DEVICE_TOKEN_CACHE_ENTRIES;
+        return clampCacheSize(entries);
+      }
+    }
+  } catch {
+    return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
+  }
+  return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
+}
+var normalizationCache = /* @__PURE__ */ new Map();
+var normalizationCacheSize = 0;
+var maxCacheSize = computeDefaultCacheSize();
+var cachedLangName = "";
+var cachedFlags = "";
+var cachedBucket = null;
+function configureNormalizationCache(maxSize) {
+  if (!Number.isFinite(maxSize)) {
+    throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must be a finite number", {
+      received: maxSize
+    });
+  }
+  if (maxSize < 0) {
+    throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must not be negative", {
+      received: maxSize
+    });
+  }
+  if (maxSize === 0) {
+    throw createNarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      "tokenizerCacheSize must be greater than zero; the normalization cache cannot be disabled",
+      {
+        received: maxSize
+      }
+    );
+  }
+  maxCacheSize = Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(Math.floor(maxSize), TOKEN_CACHE_SIZE_CEILING));
+  if (normalizationCacheSize > maxCacheSize) {
+    evictOldestEntries(normalizationCacheSize - maxCacheSize);
+  }
+}
+function getCacheBucket(language, flags) {
+  const langKey = `${language.name}@${language.revision}`;
+  if (cachedBucket && langKey === cachedLangName && flags === cachedFlags) return cachedBucket;
+  const key = `${langKey}:${flags}`;
+  let bucket = normalizationCache.get(key);
+  if (!bucket) {
+    bucket = /* @__PURE__ */ new Map();
+    normalizationCache.set(key, bucket);
+  }
+  cachedLangName = langKey;
+  cachedFlags = flags;
+  cachedBucket = bucket;
+  return bucket;
+}
+function evictOldestEntries(count) {
+  let remaining = count;
+  for (const [key, bucket] of normalizationCache) {
+    if (remaining <= 0) break;
+    if (bucket.size <= remaining) {
+      remaining -= bucket.size;
+      normalizationCacheSize -= bucket.size;
+      normalizationCache.delete(key);
+      if (bucket === cachedBucket) cachedBucket = null;
+    } else {
+      let deleted = 0;
+      for (const raw of bucket.keys()) {
+        if (deleted >= remaining) break;
+        bucket.delete(raw);
+        deleted++;
+      }
+      normalizationCacheSize -= deleted;
+      remaining = 0;
+    }
+  }
+}
+function transformToken(raw, language, stem, removeDiacritics) {
+  const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
+  let bucket = getCacheBucket(language, flags);
+  const cached = bucket.get(raw);
+  if (cached !== void 0) return cached;
+  let normalized = raw;
+  if (language.normalizer) {
+    normalized = language.normalizer(normalized);
+  }
+  if (removeDiacritics) {
+    normalized = stripDiacritics(normalized);
+  }
+  if (stem && language.stemmer) {
+    normalized = language.stemmer(normalized);
+  }
+  if (normalizationCacheSize >= maxCacheSize) {
+    evictOldestEntries(Math.max(1, maxCacheSize >>> 2));
+    bucket = getCacheBucket(language, flags);
+  }
+  bucket.set(raw, normalized);
+  normalizationCacheSize++;
+  return normalized;
+}
+function clearNormalizationCache() {
+  normalizationCache.clear();
+  normalizationCacheSize = 0;
+  cachedBucket = null;
+}
+function resetNormalizationCache() {
+  clearNormalizationCache();
+  maxCacheSize = computeDefaultCacheSize();
+}
+function getNormalizationCacheSize() {
+  return normalizationCacheSize;
+}
+
+// src/core/tokenizer/ngrams.ts
+var OTHER_SCRIPT = 0;
+var HAN_SCRIPT = 1;
+var HIRAGANA_SCRIPT = 2;
+var KATAKANA_SCRIPT = 3;
+var HANGUL_SCRIPT = 4;
+var THAI_SCRIPT = 5;
+var LAO_SCRIPT = 6;
+var KHMER_SCRIPT = 7;
+var MYANMAR_SCRIPT = 8;
+var PROLONGED_SOUND_MARK = 12540;
+var IDEOGRAPHIC_ITERATION_MARK = 12293;
+var LOWEST_NGRAM_CODE_POINT = 3584;
+var LOWEST_MARK_CODE_POINT = 768;
+var THAI_BLOCK_END = 3711;
+var LAO_BLOCK_END = 3839;
+var MYANMAR_BLOCK_START = 4096;
+var MYANMAR_BLOCK_END = 4255;
+var KHMER_BLOCK_START = 6016;
+var KHMER_BLOCK_END = 6655;
+var SOUTHEAST_ASIAN_BLOCKS_END = 6656;
+var HIRAGANA_BLOCK_START = 12352;
+var HIRAGANA_BLOCK_END = 12447;
+var COMBINING_VOICED_MARK2 = 12441;
+var COMBINING_SEMI_VOICED_MARK2 = 12442;
+var KATAKANA_BLOCK_START = 12448;
+var KATAKANA_BLOCK_END = 12543;
+var HAN_EXTENSION_A_START = 13312;
+var HAN_EXTENSION_A_END = 19903;
+var HAN_UNIFIED_START = 19968;
+var HAN_UNIFIED_END = 40959;
+var HANGUL_SYLLABLE_START = 44032;
+var HANGUL_SYLLABLE_END = 55203;
+var HAN_COMPATIBILITY_START = 63744;
+var HAN_COMPATIBILITY_END = 64255;
+var SUPPLEMENTARY_PLANE_START = 65536;
+var MARK = /\p{M}/uy;
+var HAN = /\p{Script=Han}/uy;
+var HIRAGANA = /\p{Script=Hiragana}/uy;
+var KATAKANA = /\p{Script=Katakana}/uy;
+var HANGUL = /\p{Script=Hangul}/uy;
+var THAI = /\p{Script=Thai}/uy;
+var LAO = /\p{Script=Lao}/uy;
+var KHMER = /\p{Script=Khmer}/uy;
+var MYANMAR = /\p{Script=Myanmar}/uy;
+function matchesAt(pattern, part, index) {
+  pattern.lastIndex = index;
+  return pattern.test(part);
+}
+function isMark(part, index, codePoint) {
+  if (codePoint < LOWEST_MARK_CODE_POINT) return false;
+  if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return false;
+  if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) {
+    return codePoint === COMBINING_VOICED_MARK2 || codePoint === COMBINING_SEMI_VOICED_MARK2;
+  }
+  if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return false;
+  if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return false;
+  return matchesAt(MARK, part, index);
+}
+function classifyOutsideCommonBlocks(part, index) {
+  if (matchesAt(HAN, part, index)) return HAN_SCRIPT;
+  if (matchesAt(HIRAGANA, part, index)) return HIRAGANA_SCRIPT;
+  if (matchesAt(KATAKANA, part, index)) return KATAKANA_SCRIPT;
+  if (matchesAt(HANGUL, part, index)) return HANGUL_SCRIPT;
+  if (matchesAt(THAI, part, index)) return THAI_SCRIPT;
+  if (matchesAt(LAO, part, index)) return LAO_SCRIPT;
+  if (matchesAt(KHMER, part, index)) return KHMER_SCRIPT;
+  if (matchesAt(MYANMAR, part, index)) return MYANMAR_SCRIPT;
+  return OTHER_SCRIPT;
+}
+function classifySoutheastAsian(part, index, codePoint) {
+  if (codePoint <= THAI_BLOCK_END) return matchesAt(THAI, part, index) ? THAI_SCRIPT : OTHER_SCRIPT;
+  if (codePoint <= LAO_BLOCK_END) return matchesAt(LAO, part, index) ? LAO_SCRIPT : OTHER_SCRIPT;
+  if (codePoint >= MYANMAR_BLOCK_START && codePoint <= MYANMAR_BLOCK_END) {
+    return matchesAt(MYANMAR, part, index) ? MYANMAR_SCRIPT : OTHER_SCRIPT;
+  }
+  if (codePoint >= KHMER_BLOCK_START && codePoint <= KHMER_BLOCK_END) {
+    return matchesAt(KHMER, part, index) ? KHMER_SCRIPT : OTHER_SCRIPT;
+  }
+  return classifyOutsideCommonBlocks(part, index);
+}
+function classify(part, index, codePoint, current) {
+  if (codePoint < LOWEST_NGRAM_CODE_POINT) return OTHER_SCRIPT;
+  if (codePoint < SOUTHEAST_ASIAN_BLOCKS_END) return classifySoutheastAsian(part, index, codePoint);
+  if (codePoint === PROLONGED_SOUND_MARK) {
+    return current === HIRAGANA_SCRIPT ? HIRAGANA_SCRIPT : KATAKANA_SCRIPT;
+  }
+  if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= HIRAGANA_BLOCK_END) return HIRAGANA_SCRIPT;
+  if (codePoint >= KATAKANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) return KATAKANA_SCRIPT;
+  if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return HAN_SCRIPT;
+  if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return HAN_SCRIPT;
+  if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return HANGUL_SCRIPT;
+  if (codePoint >= HAN_COMPATIBILITY_START && codePoint <= HAN_COMPATIBILITY_END) return HAN_SCRIPT;
+  if (codePoint === IDEOGRAPHIC_ITERATION_MARK) return HAN_SCRIPT;
+  return classifyOutsideCommonBlocks(part, index);
+}
+function emitRun(part, offsets, kind, size, out) {
+  const count = offsets.length - 1;
+  if (count < 1) return;
+  if (kind === OTHER_SCRIPT || count <= size) {
+    out.push(part.slice(offsets[0], offsets[count]));
+    return;
+  }
+  for (let start = 0; start + size <= count; start++) {
+    out.push(part.slice(offsets[start], offsets[start + size]));
+  }
+}
+function expandNgrams(part, size, out) {
+  const length = part.length;
+  if (length === 0) return;
+  const offsets = [];
+  let kind = OTHER_SCRIPT;
+  let index = 0;
+  while (index < length) {
+    const codePoint = part.codePointAt(index);
+    if (codePoint === void 0) break;
+    const width = codePoint >= SUPPLEMENTARY_PLANE_START ? 2 : 1;
+    if (offsets.length > 0 && isMark(part, index, codePoint)) {
+      index += width;
+      continue;
+    }
+    const next = classify(part, index, codePoint, kind);
+    if (offsets.length > 0 && next !== kind) {
+      offsets.push(index);
+      emitRun(part, offsets, kind, size, out);
+      offsets.length = 0;
+    }
+    kind = next;
+    offsets.push(index);
+    index += width;
+  }
+  offsets.push(length);
+  emitRun(part, offsets, kind, size, out);
+}
+
+// src/core/tokenizer/tokenize.ts
+function producesSurfaceForms(language, options) {
+  if (!options?.collectSurfaces || options.customTokenizer) return false;
+  return (options.stem ?? true) && language.stemmer !== null;
+}
+var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
+function resolveStopWords(language, override) {
+  if (!override) return language.stopWords;
+  if (override instanceof Set) return override;
+  return override(language.stopWords);
+}
+function splitText(text, language) {
+  const pattern = language.tokenizer?.splitPattern ?? DEFAULT_SPLIT_PATTERN;
+  const parts = text.split(pattern);
+  const ngramSize = language.tokenizer?.ngramSize;
+  if (ngramSize === void 0 || ngramSize < 1) return parts;
+  const size = Math.floor(ngramSize);
+  const expanded = [];
+  for (const part of parts) {
+    expandNgrams(part, size, expanded);
+  }
+  return expanded;
+}
+var CHAR_APOSTROPHE = 39;
+var CHAR_S = 115;
+function stripPossessive(token) {
+  const len = token.length;
+  if (len >= 2 && token.charCodeAt(len - 1) === CHAR_S && token.charCodeAt(len - 2) === CHAR_APOSTROPHE) {
+    return token.slice(0, -2);
+  }
+  if (len >= 1 && token.charCodeAt(len - 1) === CHAR_APOSTROPHE) {
+    return token.slice(0, -1);
+  }
+  return token;
+}
+function visitTokens(text, language, options, visit) {
+  const {
+    stem = true,
+    removeStopWords = true,
+    removeDiacritics = false,
+    collectSurfaces = false,
+    customTokenizer,
+    stopWordOverride
+  } = options ?? {};
+  if (customTokenizer) {
+    for (const entry of customTokenizer.tokenize(text)) visit(entry.token, entry.position, void 0, entry.token);
+    return;
+  }
+  const rawParts = splitText(normalizeForSplitting(text), language);
+  const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
+  const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
+  const wantSurfaces = collectSurfaces && stem && language.stemmer !== null;
+  const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : null;
+  const stripPossessives = language.tokenizer?.stripPossessive ?? false;
+  for (let position = 0; position < rawParts.length; position++) {
+    const part = rawParts[position];
+    const candidate = stripPossessives ? stripPossessive(part) : part;
+    if (candidate.length < minLength || stopWords?.has(candidate)) continue;
+    const processed = transformToken(candidate, language, stem, effectiveDiacritics);
+    if (processed.length === 0) continue;
+    if (!wantSurfaces) {
+      visit(processed, position, void 0, part);
+      continue;
+    }
+    const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
+    visit(processed, position, surface === processed ? void 0 : surface, part);
+  }
+}
+function tokenize(text, language, options) {
+  if (options?.customTokenizer) {
+    const customResult = options.customTokenizer.tokenize(text);
+    return { tokens: customResult, originalTokens: customResult.map((entry) => entry.token) };
+  }
+  const tokens = [];
+  const originalTokens = [];
+  const surfaces = producesSurfaceForms(language, options) ? [] : void 0;
+  visitTokens(text, language, options, (token, position, surface, original) => {
+    tokens.push({ token, position });
+    originalTokens.push(original);
+    surfaces?.push(surface);
+  });
+  return { tokens, originalTokens, surfaces };
+}
+function* tokenizeIterator(text, language, options) {
+  const {
+    stem = true,
+    removeStopWords = true,
+    removeDiacritics = false,
+    collectSurfaces = false,
+    customTokenizer,
+    stopWordOverride
+  } = options ?? {};
+  if (customTokenizer) {
+    const customResult = customTokenizer.tokenize(text);
+    for (const entry of customResult) {
+      yield entry;
+    }
+    return;
+  }
+  const normalized = normalizeForSplitting(text);
+  const rawParts = splitText(normalized, language);
+  const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
+  const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
+  const stemsTokens = stem && language.stemmer !== null;
+  const wantSurfaces = collectSurfaces && stemsTokens;
+  const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
+  const stripPossessives = language.tokenizer?.stripPossessive ?? false;
+  let position = 0;
+  for (const part of rawParts) {
+    const candidate = stripPossessives ? stripPossessive(part) : part;
+    if (candidate.length < minLength) {
+      position++;
+      continue;
+    }
+    if (stopWords.has(candidate)) {
+      position++;
+      continue;
+    }
+    const processed = transformToken(candidate, language, stem, effectiveDiacritics);
+    if (processed.length > 0) {
+      if (wantSurfaces) {
+        const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
+        yield { token: processed, position, surface: surface === processed ? void 0 : surface };
+      } else {
+        yield { token: processed, position };
+      }
+    }
+    position++;
+  }
+}
+
+// src/core/fuzzy.ts
+var FIRST_SURROGATE = 55296;
+var LAST_SURROGATE = 57343;
+var LAST_BASIC_PLANE_CODE_POINT = 65535;
+function hasSurrogate(text) {
+  for (let index = 0; index < text.length; index++) {
+    const unit = text.charCodeAt(index);
+    if (unit >= FIRST_SURROGATE && unit <= LAST_SURROGATE) return true;
+  }
+  return false;
+}
+function fuzzyPrefixOf(queryTerm, prefixLength) {
+  let end = 0;
+  for (let counted = 0; counted < prefixLength && end < queryTerm.length; counted++) {
+    const codePoint = queryTerm.codePointAt(end);
+    end += codePoint !== void 0 && codePoint > LAST_BASIC_PLANE_CODE_POINT ? 2 : 1;
+  }
+  return queryTerm.slice(0, end);
+}
+function fuzzyTermMatches(queryTerm, indexTerm, tolerance, prefixLength) {
+  if (queryTerm === indexTerm) return true;
+  if (tolerance <= 0) return false;
+  if (!indexTerm.startsWith(fuzzyPrefixOf(queryTerm, prefixLength))) return false;
+  return boundedLevenshtein(queryTerm, indexTerm, tolerance).withinTolerance;
+}
+function boundedLevenshtein(a, b, tolerance) {
+  if (tolerance < 0) return { distance: -1, withinTolerance: false };
+  if (a === b) return { distance: 0, withinTolerance: true };
+  if (hasSurrogate(a) || hasSurrogate(b)) return bandedDistance(Array.from(a), Array.from(b), tolerance);
+  return bandedDistance(a, b, tolerance);
+}
+function bandedDistance(a, b, tolerance) {
+  const m = a.length;
+  const n = b.length;
+  const beyond = tolerance + 1;
+  if (Math.abs(m - n) > tolerance) return { distance: beyond, withinTolerance: false };
+  let previous = new Uint32Array(n + 1).fill(beyond);
+  let current = new Uint32Array(n + 1).fill(beyond);
+  const firstRowEnd = Math.min(n, tolerance);
+  for (let j = 0; j <= firstRowEnd; j++) previous[j] = j;
+  for (let i = 1; i <= m; i++) {
+    const low = Math.max(1, i - tolerance);
+    const high = Math.min(n, i + tolerance);
+    current[low - 1] = low === 1 ? i : beyond;
+    let rowMin = current[low - 1];
+    const character = a[i - 1];
+    for (let j = low; j <= high; j++) {
+      const substitution = previous[j - 1] + (character === b[j - 1] ? 0 : 1);
+      const cell = Math.min(previous[j] + 1, current[j - 1] + 1, substitution, beyond);
+      current[j] = cell;
+      if (cell < rowMin) rowMin = cell;
+    }
+    if (rowMin > tolerance) return { distance: beyond, withinTolerance: false };
+    const finished = previous;
+    previous = current;
+    current = finished;
+  }
+  const distance = previous[n];
+  return { distance, withinTolerance: distance <= tolerance };
+}
+
+// src/core/partition/utils.ts
+function getOrCreateFieldNameIndex(table, fieldName) {
+  const existing = table.indexMap.get(fieldName);
+  if (existing !== void 0) return existing;
+  const idx = table.names.length;
+  table.names.push(fieldName);
+  table.indexMap.set(fieldName, idx);
+  return idx;
+}
+var VECTOR_PATTERN = /^vector\[(\d+)]$/;
+function getNestedValue(obj, path) {
+  const parts = path.split(".");
+  let current = obj;
+  for (let i = 0; i < parts.length; i++) {
+    if (current === null || current === void 0 || typeof current !== "object") return void 0;
+    if (!Object.hasOwn(current, parts[i])) return void 0;
+    current = current[parts[i]];
+  }
+  return current;
+}
+function stringArraysEqual(a, b) {
+  if (a === b) return true;
+  if (!a || !b) return false;
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+function textFieldsChanged(oldDoc, newDoc, fields) {
+  for (const { path, base } of fields) {
+    if (!isWordIndexedBase(base)) continue;
+    const oldVal = getNestedValue(oldDoc, path);
+    const newVal = getNestedValue(newDoc, path);
+    if (base === "string") {
+      if (oldVal !== newVal) return true;
+    } else {
+      if (!stringArraysEqual(oldVal, newVal)) return true;
+    }
+  }
+  return false;
+}
+function parseVectorDimension(fieldType) {
+  const match = VECTOR_PATTERN.exec(fieldType);
+  return match ? Number.parseInt(match[1], 10) : null;
+}
+function tokenizeOptions(options) {
+  return {
+    stem: true,
+    removeStopWords: true,
+    collectSurfaces: options?.collectSurfaces === true,
+    stopWordOverride: options?.stopWordOverride,
+    customTokenizer: options?.customTokenizer
+  };
+}
+function getFlatSchema(state, schema) {
+  if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
+  state.flatSchemaCache = flattenSchema(schema);
+  state.lastSchemaRef = schema;
+  return state.flatSchemaCache;
+}
+function getFieldValueForDoc(docStore, docId, fieldPath) {
+  const stored = docStore.get(docId);
+  if (!stored) return void 0;
+  return getNestedValue(stored.fields, fieldPath);
+}
+function getFieldValueByInternalId(docStore, internalId, fieldPath) {
+  const externalId = docStore.getExternalId(internalId);
+  if (externalId === void 0) return void 0;
+  return getFieldValueForDoc(docStore, externalId, fieldPath);
+}
+
+export { boundedLevenshtein, clearNormalizationCache, configureNormalizationCache, fuzzyPrefixOf, fuzzyTermMatches, getFieldValueByInternalId, getFieldValueForDoc, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, resetNormalizationCache, textFieldsChanged, tokenize, tokenizeIterator, tokenizeOptions, visitTokens };

package/dist/chunk-KEHK2O6P.mjs

@@ -0,0 +1,13 @@
+// src/vector/hnsw/constants.ts
+var MAX_LAYER_CAP = 32;
+var MAX_M = 512;
+var MIN_M = 2;
+var EARLIER_RELEASE_MIN_M = 1;
+var INITIAL_UPPER_SLOTS = 64;
+var INITIAL_LIST_CAPACITY = 64;
+var PLACEMENT_ROOM_REQUESTS = 8;
+var COMPACTION_TOMBSTONE_RATIO = 0.1;
+var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
+var DEFAULT_EF_SEARCH = 50;
+
+export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, DEFAULT_EF_SEARCH, EARLIER_RELEASE_MIN_M, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, MIN_M, PLACEMENT_ROOM_REQUESTS };

package/dist/{chunk-AFKH4ZK3.mjs → chunk-LX4JERXI.mjs}

@@ -1,4 +1,4 @@
-import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-AK3BLGCI.mjs';
+import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-Z3DQPOJQ.mjs';
 import { ErrorCodes, NarsilError } from './chunk-I64LXKO3.mjs';
 
 // src/narsil/internals.ts

package/dist/{chunk-LPBWCHBE.mjs → chunk-NOZUQM7J.mjs}

@@ -1,5 +1,5 @@
-import { MAX_SORT_FIELDS } from './chunk-AK3BLGCI.mjs';
-import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-PU43WDY4.mjs';
+import { MAX_SORT_FIELDS } from './chunk-3DDUNI6N.mjs';
+import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-2E35UM3G.mjs';
 import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-I64LXKO3.mjs';
 import { mkdtemp, open, rm } from 'fs/promises';
 import { tmpdir } from 'os';

package/dist/{chunk-WVDFG45M.mjs → chunk-PO53FV4Q.mjs}

@@ -1,5 +1,6 @@
 import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
-import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-PU43WDY4.mjs';
+import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-2E35UM3G.mjs';
+import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-KEHK2O6P.mjs';
 import { compareCodePoints } from './chunk-LJYTW7GT.mjs';
 import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
 import { readSync, closeSync, openSync } from 'fs';