@delali/narsil 0.1.14 → 0.2.0

package/dist/{chunk-HMO6OMZ3.mjs → chunk-S6IAXOBA.mjs}

@@ -1,7 +1,7 @@
+import { english } from './chunk-TGP5M5R7.mjs';
 import { createVectorStore, createArenaSimd, createHNSWIndex, magnitude, createBoundedMaxHeap, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-S7RJJMHI.mjs';
-import { english } from './chunk-TW6BXTOL.mjs';
-import { NarsilError, ErrorCodes, createNarsilError } from './chunk-ALB35PRQ.mjs';
-import { decode, encode } from '@msgpack/msgpack';
+import { NarsilError, ErrorCodes, createNarsilError } from './chunk-M2EDG55Y.mjs';
+import { encode, decode } from '@msgpack/msgpack';
 
 // src/languages/registry.ts
 var languages = /* @__PURE__ */ new Map();
@@ -17,157 +17,68 @@ function getLanguage(name) {
   return lang;
 }
 
-// src/search/facets.ts
-function mergeFacets(partitionFacets) {
-  const merged = /* @__PURE__ */ new Map();
-  for (const partition of partitionFacets) {
-    for (const [field, facetResult] of Object.entries(partition)) {
-      let fieldMap = merged.get(field);
-      if (!fieldMap) {
-        fieldMap = /* @__PURE__ */ new Map();
-        merged.set(field, fieldMap);
-      }
-      for (const [value, count] of Object.entries(facetResult.values)) {
-        fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
-      }
-    }
-  }
-  const result = {};
-  for (const [field, valueMap] of merged) {
-    const values = {};
-    for (const [value, count] of valueMap) {
-      values[value] = count;
-    }
-    result[field] = { values, count: valueMap.size };
-  }
-  return result;
-}
-
-// src/core/bitset.ts
-function createBitSet(capacity) {
-  return new Uint32Array(capacity + 31 >>> 5 || 1);
-}
-function bitsetSet(bs, index) {
-  const word = index >>> 5;
-  if (word < bs.length) {
-    bs[word] |= 1 << (index & 31);
-  }
-}
-function bitsetHas(bs, index) {
-  const word = index >>> 5;
-  if (word >= bs.length) return false;
-  return (bs[word] & 1 << (index & 31)) !== 0;
-}
-function bitsetAnd(a, b) {
-  const len = Math.min(a.length, b.length);
-  const result = new Uint32Array(len);
-  for (let i = 0; i < len; i++) {
-    result[i] = a[i] & b[i];
-  }
-  return result;
-}
-function bitsetOr(a, b) {
-  const longer = a.length >= b.length ? a : b;
-  const shorter = a.length >= b.length ? b : a;
-  const result = new Uint32Array(longer.length);
-  for (let i = 0; i < shorter.length; i++) {
-    result[i] = longer[i] | shorter[i];
-  }
-  for (let i = shorter.length; i < longer.length; i++) {
-    result[i] = longer[i];
-  }
-  return result;
+// src/core/tokenizer/normalize.ts
+var DOTTED_CAPITAL_I = "\u0130";
+var COMBINING_DOT_ABOVE = "\u0307";
+var ARMENIAN_MARKS = /[\u055B-\u055F]/;
+var FOLDED_FORMS = /\u0307|[\u0130\u055B-\u055F\u2018\u2019\u02BC\u02BB]/g;
+var WIDTH_FORMS = /[\uFF01-\uFF5E\uFF61-\uFF9F]/;
+var WIDTH_FORMS_GLOBAL = /[\uFF01-\uFF5E\uFF61-\uFF9F]/g;
+var FULLWIDTH_ASCII_END = 65374;
+var FULLWIDTH_ASCII_OFFSET = 65248;
+var HALFWIDTH_KANA_START = 65377;
+var HALFWIDTH_VOICED_MARK = 65438;
+var HALFWIDTH_SEMI_VOICED_MARK = 65439;
+var COMBINING_VOICED_MARK = "\u3099";
+var COMBINING_SEMI_VOICED_MARK = "\u309A";
+var HALFWIDTH_KANA_TARGETS = "\u3002\u300C\u300D\u3001\u30FB\u30F2\u30A1\u30A3\u30A5\u30A7\u30A9\u30E3\u30E5\u30E7\u30C3\u30FC\u30A2\u30A4\u30A6\u30A8\u30AA\u30AB\u30AD\u30AF\u30B1\u30B3\u30B5\u30B7\u30B9\u30BB\u30BD\u30BF\u30C1\u30C4\u30C6\u30C8\u30CA\u30CB\u30CC\u30CD\u30CE\u30CF\u30D2\u30D5\u30D8\u30DB\u30DE\u30DF\u30E0\u30E1\u30E2\u30E4\u30E6\u30E8\u30E9\u30EA\u30EB\u30EC\u30ED\u30EF\u30F3";
+function foldForm(match) {
+  if (match === DOTTED_CAPITAL_I) return "i";
+  if (match === COMBINING_DOT_ABOVE || ARMENIAN_MARKS.test(match)) return "";
+  return "'";
+}
+function foldWidth(match) {
+  const codePoint = match.charCodeAt(0);
+  if (codePoint <= FULLWIDTH_ASCII_END) return String.fromCharCode(codePoint - FULLWIDTH_ASCII_OFFSET);
+  if (codePoint === HALFWIDTH_VOICED_MARK) return COMBINING_VOICED_MARK;
+  if (codePoint === HALFWIDTH_SEMI_VOICED_MARK) return COMBINING_SEMI_VOICED_MARK;
+  return HALFWIDTH_KANA_TARGETS[codePoint - HALFWIDTH_KANA_START];
 }
-function bitsetNot(bs, capacity) {
-  const result = new Uint32Array(bs.length);
-  const fullWords = capacity >>> 5;
-  for (let i = 0; i < fullWords; i++) {
-    result[i] = ~bs[i];
-  }
-  const remainder = capacity & 31;
-  if (remainder > 0 && fullWords < bs.length) {
-    result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
-  }
-  return result;
-}
-function bitsetIsEmpty(bs) {
-  for (let i = 0; i < bs.length; i++) {
-    if (bs[i] !== 0) return false;
+function isAscii(text) {
+  for (let i = 0; i < text.length; i++) {
+    if (text.charCodeAt(i) > 127) return false;
   }
   return true;
 }
-function bitsetFromSet(set, capacity) {
-  const bs = createBitSet(capacity);
-  for (const id of set) {
-    bs[id >>> 5] |= 1 << (id & 31);
-  }
-  return bs;
+function foldWidthForms(text) {
+  if (!WIDTH_FORMS.test(text)) return text;
+  return text.replace(WIDTH_FORMS_GLOBAL, foldWidth);
 }
-
-// src/core/fuzzy.ts
-function boundedLevenshtein(a, b, tolerance) {
-  if (tolerance < 0) return { distance: -1, withinTolerance: false };
-  if (a === b) return { distance: 0, withinTolerance: true };
-  const m = a.length;
-  const n = b.length;
-  if (m === 0) {
-    return n <= tolerance ? { distance: n, withinTolerance: true } : { distance: n, withinTolerance: false };
-  }
-  if (n === 0) {
-    return m <= tolerance ? { distance: m, withinTolerance: true } : { distance: m, withinTolerance: false };
-  }
-  if (a.startsWith(b)) {
-    const diff2 = m - n;
-    return diff2 <= tolerance ? { distance: diff2, withinTolerance: true } : { distance: diff2, withinTolerance: false };
-  }
-  if (b.startsWith(a)) {
-    return { distance: 0, withinTolerance: true };
-  }
-  const diff = Math.abs(m - n);
-  if (diff > tolerance) return { distance: -1, withinTolerance: false };
-  const matrix = [];
-  for (let i = 0; i <= m; i++) {
-    matrix[i] = [i];
-    for (let j = 1; j <= n; j++) {
-      matrix[i][j] = i === 0 ? j : 0;
-    }
-  }
-  for (let i = 1; i <= m; i++) {
-    let rowMin = Infinity;
-    for (let j = 1; j <= n; j++) {
-      if (a[i - 1] === b[j - 1]) {
-        matrix[i][j] = matrix[i - 1][j - 1];
-      } else {
-        matrix[i][j] = Math.min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + 1);
-      }
-      rowMin = Math.min(rowMin, matrix[i][j]);
-    }
-    if (rowMin > tolerance) {
-      return { distance: -1, withinTolerance: false };
-    }
-  }
-  const finalDistance = matrix[m][n];
-  return {
-    distance: finalDistance,
-    withinTolerance: finalDistance <= tolerance
-  };
+function normalizeForSplitting(text) {
+  if (isAscii(text)) return text.toLowerCase();
+  return foldWidthForms(text).normalize("NFC").replace(FOLDED_FORMS, foldForm).toLowerCase();
+}
+function stripDiacritics(text) {
+  return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
 }
 
-// src/core/tokenizer.ts
-var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{N}_'-]+/u;
-var DEFAULT_MIN_TOKEN_LENGTH = 1;
+// src/core/tokenizer/cache.ts
 var CACHE_SIZE_FLOOR = 5e4;
 var CACHE_SIZE_CEILING = 2e6;
 var BYTES_PER_ENTRY = 200;
+var NODE_CACHE_ENTRIES = 1e6;
+var UNKNOWN_MEMORY_CACHE_ENTRIES = 2e5;
+function clampCacheSize(entries) {
+  return Math.max(CACHE_SIZE_FLOOR, Math.min(entries, CACHE_SIZE_CEILING));
+}
 function computeDefaultCacheSize() {
   try {
     if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
       const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
       if (constrainedMemory > 0) {
-        const budget = Math.floor(constrainedMemory * 0.05 / BYTES_PER_ENTRY);
-        return Math.max(CACHE_SIZE_FLOOR, Math.min(budget, CACHE_SIZE_CEILING));
+        return clampCacheSize(Math.floor(constrainedMemory * 0.05 / BYTES_PER_ENTRY));
       }
-      return Math.max(CACHE_SIZE_FLOOR, Math.min(1e6, CACHE_SIZE_CEILING));
+      return clampCacheSize(NODE_CACHE_ENTRIES);
     }
     if (typeof navigator !== "undefined") {
       const mem = navigator.deviceMemory;
@@ -176,17 +87,20 @@ function computeDefaultCacheSize() {
         if (mem <= 1) entries = 1e5;
         else if (mem <= 4) entries = 25e4;
         else entries = 5e5;
-        return Math.max(CACHE_SIZE_FLOOR, Math.min(entries, CACHE_SIZE_CEILING));
+        return clampCacheSize(entries);
       }
-      return Math.max(CACHE_SIZE_FLOOR, Math.min(2e5, CACHE_SIZE_CEILING));
     }
   } catch {
+    return clampCacheSize(UNKNOWN_MEMORY_CACHE_ENTRIES);
   }
-  return Math.max(CACHE_SIZE_FLOOR, Math.min(2e5, CACHE_SIZE_CEILING));
+  return clampCacheSize(UNKNOWN_MEMORY_CACHE_ENTRIES);
 }
 var normalizationCache = /* @__PURE__ */ new Map();
 var normalizationCacheSize = 0;
 var maxCacheSize = computeDefaultCacheSize();
+var cachedLangName = "";
+var cachedFlags = "";
+var cachedBucket = null;
 function configureNormalizationCache(maxSize) {
   if (!Number.isFinite(maxSize)) {
     throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must be a finite number", {
@@ -212,26 +126,16 @@ function configureNormalizationCache(maxSize) {
     evictOldestEntries(normalizationCacheSize - maxCacheSize);
   }
 }
-var cachedLangName = "";
-var cachedFlags = "";
-var cachedBucket = null;
-function stripDiacritics(text) {
-  return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
-}
-function resolveStopWords(language, override) {
-  if (!override) return language.stopWords;
-  if (override instanceof Set) return override;
-  return override(language.stopWords);
-}
 function getCacheBucket(language, flags) {
-  if (cachedBucket && language.name === cachedLangName && flags === cachedFlags) return cachedBucket;
-  const key = `${language.name}:${flags}`;
+  const langKey = `${language.name}@${language.revision}`;
+  if (cachedBucket && langKey === cachedLangName && flags === cachedFlags) return cachedBucket;
+  const key = `${langKey}:${flags}`;
   let bucket = normalizationCache.get(key);
   if (!bucket) {
     bucket = /* @__PURE__ */ new Map();
     normalizationCache.set(key, bucket);
   }
-  cachedLangName = language.name;
+  cachedLangName = langKey;
   cachedFlags = flags;
   cachedBucket = bucket;
   return bucket;
@@ -263,6 +167,9 @@ function transformToken(raw, language, stem, removeDiacritics) {
   const cached = bucket.get(raw);
   if (cached !== void 0) return cached;
   let normalized = raw;
+  if (language.normalizer) {
+    normalized = language.normalizer(normalized);
+  }
   if (removeDiacritics) {
     normalized = stripDiacritics(normalized);
   }
@@ -277,9 +184,173 @@ function transformToken(raw, language, stem, removeDiacritics) {
   normalizationCacheSize++;
   return normalized;
 }
+function clearNormalizationCache() {
+  normalizationCache.clear();
+  normalizationCacheSize = 0;
+  cachedBucket = null;
+}
+function resetNormalizationCache() {
+  clearNormalizationCache();
+  maxCacheSize = computeDefaultCacheSize();
+}
+function getNormalizationCacheSize() {
+  return normalizationCacheSize;
+}
+
+// src/core/tokenizer/ngrams.ts
+var OTHER_SCRIPT = 0;
+var HAN_SCRIPT = 1;
+var HIRAGANA_SCRIPT = 2;
+var KATAKANA_SCRIPT = 3;
+var HANGUL_SCRIPT = 4;
+var THAI_SCRIPT = 5;
+var LAO_SCRIPT = 6;
+var KHMER_SCRIPT = 7;
+var MYANMAR_SCRIPT = 8;
+var PROLONGED_SOUND_MARK = 12540;
+var IDEOGRAPHIC_ITERATION_MARK = 12293;
+var LOWEST_NGRAM_CODE_POINT = 3584;
+var LOWEST_MARK_CODE_POINT = 768;
+var THAI_BLOCK_END = 3711;
+var LAO_BLOCK_END = 3839;
+var MYANMAR_BLOCK_START = 4096;
+var MYANMAR_BLOCK_END = 4255;
+var KHMER_BLOCK_START = 6016;
+var KHMER_BLOCK_END = 6655;
+var SOUTHEAST_ASIAN_BLOCKS_END = 6656;
+var HIRAGANA_BLOCK_START = 12352;
+var HIRAGANA_BLOCK_END = 12447;
+var COMBINING_VOICED_MARK2 = 12441;
+var COMBINING_SEMI_VOICED_MARK2 = 12442;
+var KATAKANA_BLOCK_START = 12448;
+var KATAKANA_BLOCK_END = 12543;
+var HAN_EXTENSION_A_START = 13312;
+var HAN_EXTENSION_A_END = 19903;
+var HAN_UNIFIED_START = 19968;
+var HAN_UNIFIED_END = 40959;
+var HANGUL_SYLLABLE_START = 44032;
+var HANGUL_SYLLABLE_END = 55203;
+var HAN_COMPATIBILITY_START = 63744;
+var HAN_COMPATIBILITY_END = 64255;
+var SUPPLEMENTARY_PLANE_START = 65536;
+var MARK = /\p{M}/uy;
+var HAN = /\p{Script=Han}/uy;
+var HIRAGANA = /\p{Script=Hiragana}/uy;
+var KATAKANA = /\p{Script=Katakana}/uy;
+var HANGUL = /\p{Script=Hangul}/uy;
+var THAI = /\p{Script=Thai}/uy;
+var LAO = /\p{Script=Lao}/uy;
+var KHMER = /\p{Script=Khmer}/uy;
+var MYANMAR = /\p{Script=Myanmar}/uy;
+function matchesAt(pattern, part, index) {
+  pattern.lastIndex = index;
+  return pattern.test(part);
+}
+function isMark(part, index, codePoint) {
+  if (codePoint < LOWEST_MARK_CODE_POINT) return false;
+  if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return false;
+  if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) {
+    return codePoint === COMBINING_VOICED_MARK2 || codePoint === COMBINING_SEMI_VOICED_MARK2;
+  }
+  if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return false;
+  if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return false;
+  return matchesAt(MARK, part, index);
+}
+function classifyOutsideCommonBlocks(part, index) {
+  if (matchesAt(HAN, part, index)) return HAN_SCRIPT;
+  if (matchesAt(HIRAGANA, part, index)) return HIRAGANA_SCRIPT;
+  if (matchesAt(KATAKANA, part, index)) return KATAKANA_SCRIPT;
+  if (matchesAt(HANGUL, part, index)) return HANGUL_SCRIPT;
+  if (matchesAt(THAI, part, index)) return THAI_SCRIPT;
+  if (matchesAt(LAO, part, index)) return LAO_SCRIPT;
+  if (matchesAt(KHMER, part, index)) return KHMER_SCRIPT;
+  if (matchesAt(MYANMAR, part, index)) return MYANMAR_SCRIPT;
+  return OTHER_SCRIPT;
+}
+function classifySoutheastAsian(part, index, codePoint) {
+  if (codePoint <= THAI_BLOCK_END) return matchesAt(THAI, part, index) ? THAI_SCRIPT : OTHER_SCRIPT;
+  if (codePoint <= LAO_BLOCK_END) return matchesAt(LAO, part, index) ? LAO_SCRIPT : OTHER_SCRIPT;
+  if (codePoint >= MYANMAR_BLOCK_START && codePoint <= MYANMAR_BLOCK_END) {
+    return matchesAt(MYANMAR, part, index) ? MYANMAR_SCRIPT : OTHER_SCRIPT;
+  }
+  if (codePoint >= KHMER_BLOCK_START && codePoint <= KHMER_BLOCK_END) {
+    return matchesAt(KHMER, part, index) ? KHMER_SCRIPT : OTHER_SCRIPT;
+  }
+  return classifyOutsideCommonBlocks(part, index);
+}
+function classify(part, index, codePoint, current) {
+  if (codePoint < LOWEST_NGRAM_CODE_POINT) return OTHER_SCRIPT;
+  if (codePoint < SOUTHEAST_ASIAN_BLOCKS_END) return classifySoutheastAsian(part, index, codePoint);
+  if (codePoint === PROLONGED_SOUND_MARK) {
+    return current === HIRAGANA_SCRIPT ? HIRAGANA_SCRIPT : KATAKANA_SCRIPT;
+  }
+  if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= HIRAGANA_BLOCK_END) return HIRAGANA_SCRIPT;
+  if (codePoint >= KATAKANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) return KATAKANA_SCRIPT;
+  if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return HAN_SCRIPT;
+  if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return HAN_SCRIPT;
+  if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return HANGUL_SCRIPT;
+  if (codePoint >= HAN_COMPATIBILITY_START && codePoint <= HAN_COMPATIBILITY_END) return HAN_SCRIPT;
+  if (codePoint === IDEOGRAPHIC_ITERATION_MARK) return HAN_SCRIPT;
+  return classifyOutsideCommonBlocks(part, index);
+}
+function emitRun(part, offsets, kind, size, out) {
+  const count = offsets.length - 1;
+  if (count < 1) return;
+  if (kind === OTHER_SCRIPT || count <= size) {
+    out.push(part.slice(offsets[0], offsets[count]));
+    return;
+  }
+  for (let start = 0; start + size <= count; start++) {
+    out.push(part.slice(offsets[start], offsets[start + size]));
+  }
+}
+function expandNgrams(part, size, out) {
+  const length = part.length;
+  if (length === 0) return;
+  const offsets = [];
+  let kind = OTHER_SCRIPT;
+  let index = 0;
+  while (index < length) {
+    const codePoint = part.codePointAt(index);
+    if (codePoint === void 0) break;
+    const width = codePoint >= SUPPLEMENTARY_PLANE_START ? 2 : 1;
+    if (offsets.length > 0 && isMark(part, index, codePoint)) {
+      index += width;
+      continue;
+    }
+    const next = classify(part, index, codePoint, kind);
+    if (offsets.length > 0 && next !== kind) {
+      offsets.push(index);
+      emitRun(part, offsets, kind, size, out);
+      offsets.length = 0;
+    }
+    kind = next;
+    offsets.push(index);
+    index += width;
+  }
+  offsets.push(length);
+  emitRun(part, offsets, kind, size, out);
+}
+
+// src/core/tokenizer/tokenize.ts
+var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
+var DEFAULT_MIN_TOKEN_LENGTH = 1;
+function resolveStopWords(language, override) {
+  if (!override) return language.stopWords;
+  if (override instanceof Set) return override;
+  return override(language.stopWords);
+}
 function splitText(text, language) {
   const pattern = language.tokenizer?.splitPattern ?? DEFAULT_SPLIT_PATTERN;
-  return text.split(pattern);
+  const parts = text.split(pattern);
+  const ngramSize = language.tokenizer?.ngramSize;
+  if (ngramSize === void 0 || ngramSize < 1) return parts;
+  const size = Math.floor(ngramSize);
+  const expanded = [];
+  for (const part of parts) {
+    expandNgrams(part, size, expanded);
+  }
+  return expanded;
 }
 var CHAR_APOSTROPHE = 39;
 var CHAR_S = 115;
@@ -293,12 +364,6 @@ function stripPossessive(token) {
   }
   return token;
 }
-function isAscii(text) {
-  for (let i = 0; i < text.length; i++) {
-    if (text.charCodeAt(i) > 127) return false;
-  }
-  return true;
-}
 function tokenize(text, language, options) {
   const {
     stem = true,
@@ -316,11 +381,11 @@ function tokenize(text, language, options) {
       originalTokens: originals
     };
   }
-  const normalized = isAscii(text) ? text.toLowerCase() : text.normalize("NFC").toLowerCase();
+  const normalized = normalizeForSplitting(text);
   const rawParts = splitText(normalized, language);
   const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
   const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
-  const stemsTokens = stem && language.stemmer !== void 0;
+  const stemsTokens = stem && language.stemmer !== null;
   const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
   const stripPossessives = language.tokenizer?.stripPossessive ?? false;
   const tokens = [];
@@ -348,66 +413,287 @@ function tokenize(text, language, options) {
     }
     position++;
   }
-  return { tokens, originalTokens, surfaces };
-}
-function* tokenizeIterator(text, language, options) {
-  const {
-    stem = true,
-    removeStopWords = true,
-    removeDiacritics = false,
-    collectSurfaces = false,
-    customTokenizer,
-    stopWordOverride
-  } = options ?? {};
-  if (customTokenizer) {
-    const customResult = customTokenizer.tokenize(text);
-    for (const entry of customResult) {
-      yield entry;
+  return { tokens, originalTokens, surfaces };
+}
+function* tokenizeIterator(text, language, options) {
+  const {
+    stem = true,
+    removeStopWords = true,
+    removeDiacritics = false,
+    collectSurfaces = false,
+    customTokenizer,
+    stopWordOverride
+  } = options ?? {};
+  if (customTokenizer) {
+    const customResult = customTokenizer.tokenize(text);
+    for (const entry of customResult) {
+      yield entry;
+    }
+    return;
+  }
+  const normalized = normalizeForSplitting(text);
+  const rawParts = splitText(normalized, language);
+  const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
+  const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
+  const stemsTokens = stem && language.stemmer !== null;
+  const wantSurfaces = collectSurfaces && stemsTokens;
+  const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
+  const stripPossessives = language.tokenizer?.stripPossessive ?? false;
+  let position = 0;
+  for (const part of rawParts) {
+    const candidate = stripPossessives ? stripPossessive(part) : part;
+    if (candidate.length < minLength) {
+      position++;
+      continue;
+    }
+    if (stopWords.has(candidate)) {
+      position++;
+      continue;
+    }
+    const processed = transformToken(candidate, language, stem, effectiveDiacritics);
+    if (processed.length > 0) {
+      if (wantSurfaces) {
+        const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
+        yield { token: processed, position, surface: surface === processed ? void 0 : surface };
+      } else {
+        yield { token: processed, position };
+      }
+    }
+    position++;
+  }
+}
+
+// src/partitioning/distributed-scoring.ts
+function analysedQueryTokens(term, language, analysis) {
+  const { tokens } = tokenize(term, language, {
+    stem: true,
+    removeStopWords: true,
+    customTokenizer: analysis.customTokenizer,
+    stopWordOverride: analysis.stopWords
+  });
+  const unique = /* @__PURE__ */ new Set();
+  for (const { token } of tokens) {
+    unique.add(token);
+  }
+  return unique;
+}
+function copyNumericRecord(source) {
+  const copy = /* @__PURE__ */ Object.create(null);
+  for (const [key, value] of Object.entries(source)) {
+    if (typeof value === "number" && Number.isFinite(value)) {
+      copy[key] = value;
+    }
+  }
+  return copy;
+}
+function sanitizeGlobalStats(stats) {
+  return {
+    totalDocuments: typeof stats.totalDocuments === "number" && Number.isFinite(stats.totalDocuments) ? stats.totalDocuments : 0,
+    docFrequencies: copyNumericRecord(stats.docFrequencies),
+    totalFieldLengths: copyNumericRecord(stats.totalFieldLengths),
+    averageFieldLengths: copyNumericRecord(stats.averageFieldLengths)
+  };
+}
+function pruneStatsToQueryTerms(stats, term, language, analysis) {
+  const docFrequencies = /* @__PURE__ */ Object.create(null);
+  for (const token of analysedQueryTokens(term, language, analysis)) {
+    if (Object.hasOwn(stats.docFrequencies, token)) {
+      docFrequencies[token] = stats.docFrequencies[token];
+    }
+  }
+  return {
+    totalDocuments: stats.totalDocuments,
+    docFrequencies,
+    totalFieldLengths: stats.totalFieldLengths,
+    averageFieldLengths: stats.averageFieldLengths
+  };
+}
+function collectQueryTermStats(manager, term, language, analysis) {
+  const queryTokens = analysedQueryTokens(term, language, analysis);
+  let totalDocuments = 0;
+  const docFrequencies = /* @__PURE__ */ Object.create(null);
+  const totalFieldLengths = /* @__PURE__ */ Object.create(null);
+  for (const partition of manager.getAllPartitions()) {
+    const stats = partition.stats;
+    totalDocuments += stats.totalDocuments;
+    for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
+      totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
+    }
+    for (const token of queryTokens) {
+      if (Object.hasOwn(stats.docFrequencies, token)) {
+        docFrequencies[token] = (docFrequencies[token] ?? 0) + stats.docFrequencies[token];
+      }
+    }
+  }
+  const averageFieldLengths = /* @__PURE__ */ Object.create(null);
+  if (totalDocuments > 0) {
+    for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
+      averageFieldLengths[field] = totalLength / totalDocuments;
+    }
+  }
+  return { totalDocuments, docFrequencies, totalFieldLengths, averageFieldLengths };
+}
+function mergePartitionStats(statsArray) {
+  let totalDocuments = 0;
+  const docFrequencies = /* @__PURE__ */ Object.create(null);
+  const totalFieldLengths = /* @__PURE__ */ Object.create(null);
+  for (const stats of statsArray) {
+    totalDocuments += stats.totalDocuments;
+    for (const [term, freq] of Object.entries(stats.docFrequencies)) {
+      docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
+    }
+    for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
+      totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
+    }
+  }
+  const averageFieldLengths = /* @__PURE__ */ Object.create(null);
+  if (totalDocuments > 0) {
+    for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
+      averageFieldLengths[field] = totalLength / totalDocuments;
+    }
+  }
+  return {
+    totalDocuments,
+    docFrequencies,
+    totalFieldLengths,
+    averageFieldLengths
+  };
+}
+
+// src/search/facets.ts
+function mergeFacets(partitionFacets) {
+  const merged = /* @__PURE__ */ new Map();
+  for (const partition of partitionFacets) {
+    for (const [field, facetResult] of Object.entries(partition)) {
+      let fieldMap = merged.get(field);
+      if (!fieldMap) {
+        fieldMap = /* @__PURE__ */ new Map();
+        merged.set(field, fieldMap);
+      }
+      for (const [value, count] of Object.entries(facetResult.values)) {
+        fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
+      }
+    }
+  }
+  const result = {};
+  for (const [field, valueMap] of merged) {
+    const values = {};
+    for (const [value, count] of valueMap) {
+      values[value] = count;
+    }
+    result[field] = { values, count: valueMap.size };
+  }
+  return result;
+}
+
+// src/core/bitset.ts
+function createBitSet(capacity) {
+  return new Uint32Array(capacity + 31 >>> 5 || 1);
+}
+function bitsetSet(bs, index) {
+  const word = index >>> 5;
+  if (word < bs.length) {
+    bs[word] |= 1 << (index & 31);
+  }
+}
+function bitsetHas(bs, index) {
+  const word = index >>> 5;
+  if (word >= bs.length) return false;
+  return (bs[word] & 1 << (index & 31)) !== 0;
+}
+function bitsetAnd(a, b) {
+  const len = Math.min(a.length, b.length);
+  const result = new Uint32Array(len);
+  for (let i = 0; i < len; i++) {
+    result[i] = a[i] & b[i];
+  }
+  return result;
+}
+function bitsetOr(a, b) {
+  const longer = a.length >= b.length ? a : b;
+  const shorter = a.length >= b.length ? b : a;
+  const result = new Uint32Array(longer.length);
+  for (let i = 0; i < shorter.length; i++) {
+    result[i] = longer[i] | shorter[i];
+  }
+  for (let i = shorter.length; i < longer.length; i++) {
+    result[i] = longer[i];
+  }
+  return result;
+}
+function bitsetNot(bs, capacity) {
+  const result = new Uint32Array(bs.length);
+  const fullWords = capacity >>> 5;
+  for (let i = 0; i < fullWords; i++) {
+    result[i] = ~bs[i];
+  }
+  const remainder = capacity & 31;
+  if (remainder > 0 && fullWords < bs.length) {
+    result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
+  }
+  return result;
+}
+function bitsetIsEmpty(bs) {
+  for (let i = 0; i < bs.length; i++) {
+    if (bs[i] !== 0) return false;
+  }
+  return true;
+}
+function bitsetFromSet(set, capacity) {
+  const bs = createBitSet(capacity);
+  for (const id of set) {
+    bs[id >>> 5] |= 1 << (id & 31);
+  }
+  return bs;
+}
+
+// src/core/fuzzy.ts
+function boundedLevenshtein(a, b, tolerance) {
+  if (tolerance < 0) return { distance: -1, withinTolerance: false };
+  if (a === b) return { distance: 0, withinTolerance: true };
+  const m = a.length;
+  const n = b.length;
+  if (m === 0) {
+    return n <= tolerance ? { distance: n, withinTolerance: true } : { distance: n, withinTolerance: false };
+  }
+  if (n === 0) {
+    return m <= tolerance ? { distance: m, withinTolerance: true } : { distance: m, withinTolerance: false };
+  }
+  if (a.startsWith(b)) {
+    const diff2 = m - n;
+    return diff2 <= tolerance ? { distance: diff2, withinTolerance: true } : { distance: diff2, withinTolerance: false };
+  }
+  if (b.startsWith(a)) {
+    return { distance: 0, withinTolerance: true };
+  }
+  const diff = Math.abs(m - n);
+  if (diff > tolerance) return { distance: -1, withinTolerance: false };
+  const matrix = [];
+  for (let i = 0; i <= m; i++) {
+    matrix[i] = [i];
+    for (let j = 1; j <= n; j++) {
+      matrix[i][j] = i === 0 ? j : 0;
     }
-    return;
   }
-  const normalized = isAscii(text) ? text.toLowerCase() : text.normalize("NFC").toLowerCase();
-  const rawParts = splitText(normalized, language);
-  const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
-  const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
-  const stemsTokens = stem && language.stemmer !== void 0;
-  const wantSurfaces = collectSurfaces && stemsTokens;
-  const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
-  const stripPossessives = language.tokenizer?.stripPossessive ?? false;
-  let position = 0;
-  for (const part of rawParts) {
-    const candidate = stripPossessives ? stripPossessive(part) : part;
-    if (candidate.length < minLength) {
-      position++;
-      continue;
-    }
-    if (stopWords.has(candidate)) {
-      position++;
-      continue;
-    }
-    const processed = transformToken(candidate, language, stem, effectiveDiacritics);
-    if (processed.length > 0) {
-      if (wantSurfaces) {
-        const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
-        yield { token: processed, position, surface: surface === processed ? void 0 : surface };
+  for (let i = 1; i <= m; i++) {
+    let rowMin = Infinity;
+    for (let j = 1; j <= n; j++) {
+      if (a[i - 1] === b[j - 1]) {
+        matrix[i][j] = matrix[i - 1][j - 1];
       } else {
-        yield { token: processed, position };
+        matrix[i][j] = Math.min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + 1);
       }
+      rowMin = Math.min(rowMin, matrix[i][j]);
+    }
+    if (rowMin > tolerance) {
+      return { distance: -1, withinTolerance: false };
     }
-    position++;
   }
-}
-function clearNormalizationCache() {
-  normalizationCache.clear();
-  normalizationCacheSize = 0;
-  cachedBucket = null;
-}
-function resetNormalizationCache() {
-  clearNormalizationCache();
-  maxCacheSize = computeDefaultCacheSize();
-}
-function getNormalizationCacheSize() {
-  return normalizationCacheSize;
+  const finalDistance = matrix[m][n];
+  return {
+    distance: finalDistance,
+    withinTolerance: finalDistance <= tolerance
+  };
 }
 
 // src/schema/validator/shared.ts
@@ -979,49 +1265,6 @@ function queryTermSatisfied(queryToken, indexTokens, tolerance, exact) {
   return false;
 }
 
-// src/partitioning/distributed-scoring.ts
-function collectGlobalStats(manager) {
-  const aggregate = manager.getAggregateStats();
-  const averageFieldLengths = {};
-  if (aggregate.totalDocuments > 0) {
-    for (const [field, totalLength] of Object.entries(aggregate.totalFieldLengths)) {
-      averageFieldLengths[field] = totalLength / aggregate.totalDocuments;
-    }
-  }
-  return {
-    totalDocuments: aggregate.totalDocuments,
-    docFrequencies: aggregate.docFrequencies,
-    totalFieldLengths: aggregate.totalFieldLengths,
-    averageFieldLengths
-  };
-}
-function mergePartitionStats(statsArray) {
-  let totalDocuments = 0;
-  const docFrequencies = {};
-  const totalFieldLengths = {};
-  for (const stats of statsArray) {
-    totalDocuments += stats.totalDocuments;
-    for (const [term, freq] of Object.entries(stats.docFrequencies)) {
-      docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
-    }
-    for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
-      totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
-    }
-  }
-  const averageFieldLengths = {};
-  if (totalDocuments > 0) {
-    for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
-      averageFieldLengths[field] = totalLength / totalDocuments;
-    }
-  }
-  return {
-    totalDocuments,
-    docFrequencies,
-    totalFieldLengths,
-    averageFieldLengths
-  };
-}
-
 // src/partitioning/fan-out.ts
 async function fanOutQuery(manager, params, language, schema, config, searchOptions) {
   const partitions = config.partitionIds ? config.partitionIds.map((id) => manager.partitionAt(id)).filter((p) => p !== void 0) : manager.getAllPartitions();
@@ -1031,7 +1274,12 @@ async function fanOutQuery(manager, params, language, schema, config, searchOpti
   let globalStats;
   const effectiveMode = resolveEffectiveMode(config);
   if (effectiveMode === "dfs") {
-    globalStats = collectGlobalStats(manager);
+    if (params.term !== void 0 && params.term.trim().length > 0) {
+      globalStats = collectQueryTermStats(manager, params.term, language, {
+        stopWords: searchOptions?.stopWords,
+        customTokenizer: searchOptions?.customTokenizer
+      });
+    }
   } else if (effectiveMode === "broadcast") {
     globalStats = config.globalStats;
   }
@@ -1212,6 +1460,232 @@ function heapPop(heap) {
   }
   return top;
 }
+
+// src/analysis/registry.ts
+var tokenizers = /* @__PURE__ */ new Map();
+var stopWordSets = /* @__PURE__ */ new Map();
+function requireName(name, kind) {
+  if (typeof name !== "string" || name.trim().length === 0) {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `A registered ${kind} needs a non-empty name`, { name });
+  }
+}
+function registerTokenizer(name, tokenizer) {
+  requireName(name, "tokenizer");
+  if (typeof tokenizer?.tokenize !== "function") {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Tokenizer "${name}" needs a tokenize method`, { tokenizer: name });
+  }
+  tokenizers.set(name, tokenizer);
+}
+function registerStopWords(name, stopWords) {
+  requireName(name, "stop word set");
+  if (!(stopWords instanceof Set) && typeof stopWords !== "function") {
+    throw new NarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      `Stop word set "${name}" needs a Set of words or a function returning one`,
+      { stopWords: name }
+    );
+  }
+  stopWordSets.set(name, stopWords);
+}
+function hasTokenizer(name) {
+  return tokenizers.has(name);
+}
+function hasStopWords(name) {
+  return stopWordSets.has(name);
+}
+function getTokenizer(name) {
+  const tokenizer = tokenizers.get(name);
+  if (!tokenizer) {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Tokenizer "${name}" is not registered on this engine`, {
+      tokenizer: name,
+      available: [...tokenizers.keys()]
+    });
+  }
+  return tokenizer;
+}
+function getStopWords(name) {
+  const stopWords = stopWordSets.get(name);
+  if (!stopWords) {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Stop word set "${name}" is not registered on this engine`, {
+      stopWords: name,
+      available: [...stopWordSets.keys()]
+    });
+  }
+  return stopWords;
+}
+function resolveIndexAnalysis(config) {
+  return {
+    stopWords: typeof config.stopWords === "string" ? getStopWords(config.stopWords) : config.stopWords,
+    customTokenizer: typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer
+  };
+}
+function metadataToWire(meta) {
+  const wire = {
+    index_name: meta.indexName,
+    schema: meta.schema,
+    language: meta.language,
+    partition_count: meta.partitionCount,
+    bm25_params: meta.bm25Params,
+    created_at: meta.createdAt,
+    engine_version: meta.engineVersion
+  };
+  if (meta.vectorFields) {
+    wire.vector_fields = meta.vectorFields;
+  }
+  if (meta.embedding) {
+    wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
+  }
+  if (meta.surfaceForms === true) {
+    wire.surface_forms_enabled = true;
+  }
+  if (meta.analysisRevision !== void 0) {
+    wire.analysis_revision = meta.analysisRevision;
+  }
+  if (meta.tokenizer !== void 0) {
+    wire.tokenizer = meta.tokenizer;
+  }
+  if (meta.stopWords !== void 0) {
+    wire.stop_words = meta.stopWords;
+  }
+  if (meta.stopWordList !== void 0) {
+    wire.stop_word_list = meta.stopWordList;
+  }
+  if (meta.partitionLimits !== void 0) {
+    wire.partition_limits = {
+      ...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
+      ...meta.partitionLimits.maxPartitions !== void 0 ? { max_partitions: meta.partitionLimits.maxPartitions } : {},
+      ...meta.partitionLimits.watermark !== void 0 ? { watermark: meta.partitionLimits.watermark } : {}
+    };
+  }
+  if (meta.defaultScoring !== void 0) {
+    wire.default_scoring = meta.defaultScoring;
+  }
+  if (meta.trackPositions !== void 0) {
+    wire.track_positions = meta.trackPositions;
+  }
+  if (meta.strict !== void 0) {
+    wire.strict = meta.strict;
+  }
+  if (meta.required !== void 0) {
+    wire.required = meta.required;
+  }
+  if (meta.vectorPromotion !== void 0) {
+    const promotion = meta.vectorPromotion;
+    wire.vector_promotion = {
+      ...promotion.threshold !== void 0 ? { threshold: promotion.threshold } : {},
+      ...promotion.filterThreshold !== void 0 ? { filter_threshold: promotion.filterThreshold } : {},
+      ...promotion.hnswConfig !== void 0 ? {
+        hnsw_config: {
+          ...promotion.hnswConfig.m !== void 0 ? { m: promotion.hnswConfig.m } : {},
+          ...promotion.hnswConfig.efConstruction !== void 0 ? { ef_construction: promotion.hnswConfig.efConstruction } : {},
+          ...promotion.hnswConfig.metric !== void 0 ? { metric: promotion.hnswConfig.metric } : {}
+        }
+      } : {},
+      ...promotion.quantization !== void 0 ? { quantization: promotion.quantization } : {}
+    };
+  }
+  return wire;
+}
+function wireToMetadata(raw) {
+  const meta = {
+    indexName: raw.index_name,
+    schema: raw.schema ?? {},
+    language: raw.language ?? "english",
+    partitionCount: raw.partition_count ?? 1,
+    bm25Params: raw.bm25_params ?? { k1: 1.2, b: 0.75 },
+    createdAt: raw.created_at ?? 0,
+    engineVersion: raw.engine_version ?? "0.0.0"
+  };
+  if (raw.vector_fields) {
+    meta.vectorFields = raw.vector_fields;
+  }
+  if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
+    meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
+  }
+  if (raw.surface_forms_enabled === true) {
+    meta.surfaceForms = true;
+  }
+  if (typeof raw.analysis_revision === "string") {
+    meta.analysisRevision = raw.analysis_revision;
+  }
+  if (typeof raw.tokenizer === "string") {
+    meta.tokenizer = raw.tokenizer;
+  }
+  if (typeof raw.stop_words === "string") {
+    meta.stopWords = raw.stop_words;
+  }
+  if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
+    meta.stopWordList = raw.stop_word_list;
+  }
+  if (raw.partition_limits && typeof raw.partition_limits === "object") {
+    const limits = {
+      ...isPositiveInteger2(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
+      ...isPositiveInteger2(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
+      ...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
+    };
+    if (Object.keys(limits).length > 0) {
+      meta.partitionLimits = limits;
+    }
+  }
+  if (raw.default_scoring === "local" || raw.default_scoring === "dfs" || raw.default_scoring === "broadcast") {
+    meta.defaultScoring = raw.default_scoring;
+  }
+  if (typeof raw.track_positions === "boolean") {
+    meta.trackPositions = raw.track_positions;
+  }
+  if (typeof raw.strict === "boolean") {
+    meta.strict = raw.strict;
+  }
+  if (Array.isArray(raw.required) && raw.required.every((field) => typeof field === "string")) {
+    meta.required = raw.required;
+  }
+  if (raw.vector_promotion && typeof raw.vector_promotion === "object") {
+    const promotion = raw.vector_promotion;
+    const hnsw = promotion.hnsw_config && typeof promotion.hnsw_config === "object" ? promotion.hnsw_config : void 0;
+    const metric = hnsw?.metric;
+    const restored = {};
+    const hnswConfig = {};
+    if (isPositiveInteger2(hnsw?.m)) {
+      hnswConfig.m = hnsw?.m;
+    }
+    if (isPositiveInteger2(hnsw?.ef_construction)) {
+      hnswConfig.efConstruction = hnsw?.ef_construction;
+    }
+    if (metric === "cosine" || metric === "dotProduct" || metric === "euclidean") {
+      hnswConfig.metric = metric;
+    }
+    if (isPositiveInteger2(promotion.threshold)) {
+      restored.threshold = promotion.threshold;
+    }
+    if (isPositiveInteger2(promotion.filter_threshold)) {
+      restored.filterThreshold = promotion.filter_threshold;
+    }
+    if (Object.keys(hnswConfig).length > 0) {
+      restored.hnswConfig = hnswConfig;
+    }
+    if (promotion.quantization === "sq8" || promotion.quantization === "none") {
+      restored.quantization = promotion.quantization;
+    }
+    if (Object.keys(restored).length > 0) {
+      meta.vectorPromotion = restored;
+    }
+  }
+  return meta;
+}
+function isPositiveInteger2(value) {
+  return typeof value === "number" && Number.isInteger(value) && value >= 1;
+}
+function isWatermarkFraction(value) {
+  return typeof value === "number" && value > 0 && value <= 1;
+}
+function serializeMetadata(meta) {
+  const wire = metadataToWire(meta);
+  return encode(wire);
+}
+function deserializeMetadata(data) {
+  const raw = decode(data);
+  return wireToMetadata(raw);
+}
 function sanitizeSurfaceForms(raw) {
   if (raw === void 0 || raw === null || typeof raw !== "object" || Array.isArray(raw)) return void 0;
   const result = /* @__PURE__ */ Object.create(null);
@@ -1306,68 +1780,18 @@ function wireToPartition(raw) {
       geopoint
     },
     surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
-    vectorData,
-    statistics: {
-      totalDocuments: raw.statistics?.total_documents ?? 0,
-      totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
-      averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
-      docFrequencies: raw.statistics?.doc_frequencies ?? {}
-    }
-  };
-}
-function deserializePayloadV1(data) {
-  const raw = decode(data);
-  return wireToPartition(raw);
-}
-function metadataToWire(meta) {
-  const wire = {
-    index_name: meta.indexName,
-    schema: meta.schema,
-    language: meta.language,
-    partition_count: meta.partitionCount,
-    bm25_params: meta.bm25Params,
-    created_at: meta.createdAt,
-    engine_version: meta.engineVersion
-  };
-  if (meta.vectorFields) {
-    wire.vector_fields = meta.vectorFields;
-  }
-  if (meta.embedding) {
-    wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
-  }
-  if (meta.surfaceForms === true) {
-    wire.surface_forms_enabled = true;
-  }
-  return wire;
-}
-function wireToMetadata(raw) {
-  const meta = {
-    indexName: raw.index_name,
-    schema: raw.schema ?? {},
-    language: raw.language ?? "english",
-    partitionCount: raw.partition_count ?? 1,
-    bm25Params: raw.bm25_params ?? { k1: 1.2, b: 0.75 },
-    createdAt: raw.created_at ?? 0,
-    engineVersion: raw.engine_version ?? "0.0.0"
+    vectorData,
+    statistics: {
+      totalDocuments: raw.statistics?.total_documents ?? 0,
+      totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
+      averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
+      docFrequencies: raw.statistics?.doc_frequencies ?? {}
+    }
   };
-  if (raw.vector_fields) {
-    meta.vectorFields = raw.vector_fields;
-  }
-  if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
-    meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
-  }
-  if (raw.surface_forms_enabled === true) {
-    meta.surfaceForms = true;
-  }
-  return meta;
-}
-function serializeMetadata(meta) {
-  const wire = metadataToWire(meta);
-  return encode(wire);
 }
-function deserializeMetadata(data) {
+function deserializePayloadV1(data) {
   const raw = decode(data);
-  return wireToMetadata(raw);
+  return wireToPartition(raw);
 }
 
 // src/serialization/payload-v2.ts
@@ -1571,10 +1995,77 @@ function createDocumentStore() {
   };
 }
 
-// src/core/inverted-index.ts
+// src/core/posting-list.ts
 var INITIAL_CAPACITY = 8;
 var MAX_TERM_FREQUENCY = 65535;
 var COMPACTION_THRESHOLD = 0.3;
+function createPostingList() {
+  return {
+    length: 0,
+    docIds: [],
+    termFrequencies: new Uint16Array(INITIAL_CAPACITY),
+    fieldNameIndices: new Uint8Array(INITIAL_CAPACITY),
+    positions: null,
+    docIdSet: /* @__PURE__ */ new Set(),
+    deletedDocs: /* @__PURE__ */ new Set(),
+    totalTermFrequency: 0
+  };
+}
+function growTypedArrays(list) {
+  const newCap = list.termFrequencies.length * 2;
+  const newTF = new Uint16Array(newCap);
+  newTF.set(list.termFrequencies);
+  list.termFrequencies = newTF;
+  const newFNI = new Uint8Array(newCap);
+  newFNI.set(list.fieldNameIndices);
+  list.fieldNameIndices = newFNI;
+}
+function compactList(list) {
+  if (list.deletedDocs.size === 0) return;
+  let writeIdx = 0;
+  for (let i = 0; i < list.length; i++) {
+    if (!list.deletedDocs.has(list.docIds[i])) {
+      if (writeIdx !== i) {
+        list.docIds[writeIdx] = list.docIds[i];
+        list.termFrequencies[writeIdx] = list.termFrequencies[i];
+        list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
+        if (list.positions) {
+          list.positions[writeIdx] = list.positions[i];
+        }
+      }
+      writeIdx++;
+    } else {
+      list.totalTermFrequency -= list.termFrequencies[i];
+    }
+  }
+  list.docIds.length = writeIdx;
+  if (list.positions) list.positions.length = writeIdx;
+  list.length = writeIdx;
+  list.deletedDocs.clear();
+}
+function compactDocEntries(list, internalId) {
+  let writeIdx = 0;
+  for (let i = 0; i < list.length; i++) {
+    if (list.docIds[i] !== internalId) {
+      if (writeIdx !== i) {
+        list.docIds[writeIdx] = list.docIds[i];
+        list.termFrequencies[writeIdx] = list.termFrequencies[i];
+        list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
+        if (list.positions) {
+          list.positions[writeIdx] = list.positions[i];
+        }
+      }
+      writeIdx++;
+    } else {
+      list.totalTermFrequency -= list.termFrequencies[i];
+    }
+  }
+  list.docIds.length = writeIdx;
+  if (list.positions) list.positions.length = writeIdx;
+  list.length = writeIdx;
+}
+
+// src/core/inverted-index.ts
 function createInvertedIndex(fieldNameTable) {
   const index = /* @__PURE__ */ new Map();
   const charBuckets = /* @__PURE__ */ new Map();
@@ -1619,74 +2110,12 @@ function createInvertedIndex(fieldNameTable) {
   function getOrCreateList(token) {
     let list = index.get(token);
     if (!list) {
-      list = {
-        length: 0,
-        docIds: [],
-        termFrequencies: new Uint16Array(INITIAL_CAPACITY),
-        fieldNameIndices: new Uint8Array(INITIAL_CAPACITY),
-        positions: null,
-        docIdSet: /* @__PURE__ */ new Set(),
-        deletedDocs: /* @__PURE__ */ new Set(),
-        totalTermFrequency: 0
-      };
+      list = createPostingList();
       index.set(token, list);
       trackToken(token);
     }
     return list;
   }
-  function growTypedArrays(list) {
-    const newCap = list.termFrequencies.length * 2;
-    const newTF = new Uint16Array(newCap);
-    newTF.set(list.termFrequencies);
-    list.termFrequencies = newTF;
-    const newFNI = new Uint8Array(newCap);
-    newFNI.set(list.fieldNameIndices);
-    list.fieldNameIndices = newFNI;
-  }
-  function compactList(list) {
-    if (list.deletedDocs.size === 0) return;
-    let writeIdx = 0;
-    for (let i = 0; i < list.length; i++) {
-      if (!list.deletedDocs.has(list.docIds[i])) {
-        if (writeIdx !== i) {
-          list.docIds[writeIdx] = list.docIds[i];
-          list.termFrequencies[writeIdx] = list.termFrequencies[i];
-          list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
-          if (list.positions) {
-            list.positions[writeIdx] = list.positions[i];
-          }
-        }
-        writeIdx++;
-      } else {
-        list.totalTermFrequency -= list.termFrequencies[i];
-      }
-    }
-    list.docIds.length = writeIdx;
-    if (list.positions) list.positions.length = writeIdx;
-    list.length = writeIdx;
-    list.deletedDocs.clear();
-  }
-  function compactDocEntries(list, internalId) {
-    let writeIdx = 0;
-    for (let i = 0; i < list.length; i++) {
-      if (list.docIds[i] !== internalId) {
-        if (writeIdx !== i) {
-          list.docIds[writeIdx] = list.docIds[i];
-          list.termFrequencies[writeIdx] = list.termFrequencies[i];
-          list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
-          if (list.positions) {
-            list.positions[writeIdx] = list.positions[i];
-          }
-        }
-        writeIdx++;
-      } else {
-        list.totalTermFrequency -= list.termFrequencies[i];
-      }
-    }
-    list.docIds.length = writeIdx;
-    if (list.positions) list.positions.length = writeIdx;
-    list.length = writeIdx;
-  }
   return {
     insert(token, internalId, termFrequency, fieldNameIndex, positions) {
       const list = getOrCreateList(token);
@@ -3489,6 +3918,64 @@ function applyPartitionFiltersBitset(state, filters, schema) {
   return evaluateFilters(filters, context);
 }
 
+// src/core/partition/rebuild.ts
+var EMPTY_STATS = {
+  totalDocuments: 0,
+  totalFieldLengths: {},
+  averageFieldLengths: {},
+  docFrequencies: {}
+};
+function textFieldsOf(flatSchema) {
+  return Object.entries(flatSchema).filter(([, fieldType]) => fieldType === "string" || fieldType === "string[]");
+}
+function rebuildTextIndex(state, schema, language, options) {
+  const textFields = textFieldsOf(getFlatSchema(state, schema));
+  state.invertedIdx.clear();
+  state.surfaceRegistry.clear();
+  state.stats.deserialize(EMPTY_STATS);
+  state.invertedIdx.beginBatch();
+  try {
+    for (const [docId, stored] of state.docStore.all()) {
+      const internalId = state.docStore.getInternalId(docId);
+      if (internalId === void 0) continue;
+      const fields = stored.fields;
+      const fieldLengths = {};
+      const tokensByField = {};
+      for (const [fieldPath, fieldType] of textFields) {
+        const value = getNestedValue(fields, fieldPath);
+        if (value === void 0 || value === null) continue;
+        if (fieldType === "string") {
+          indexStringField(
+            state,
+            internalId,
+            fieldPath,
+            value,
+            language,
+            options,
+            fieldLengths,
+            tokensByField
+          );
+        } else {
+          indexStringArrayField(
+            state,
+            internalId,
+            fieldPath,
+            value,
+            language,
+            options,
+            fieldLengths,
+            tokensByField
+          );
+        }
+      }
+      state.docStore.storeRef(docId, stored.fields, fieldLengths);
+      state.stats.addDocument(fieldLengths, tokensByField);
+    }
+  } finally {
+    state.invertedIdx.endBatch();
+  }
+}
+
 // src/core/scorer.ts
 var DEFAULT_K1 = 1.2;
 var DEFAULT_B = 0.75;
@@ -3511,8 +3998,7 @@ function computeBM25WithGlobalStats(termFrequency, globalDocFrequency, globalTot
   return computeBM25(termFrequency, globalDocFrequency, globalTotalDocs, fieldLength, globalAvgFieldLength, params);
 }
 
-// src/core/partition/search.ts
-var DEFAULT_MAX_RESULTS = 1e3;
+// src/core/partition/scoring.ts
 var EMPTY_COMPONENTS = Object.freeze({});
 function accumulateTermScore(docScores, internalId, termScore, collect, fieldName, token, termFrequency, fieldLength, idf) {
   const existing = docScores.get(internalId);
@@ -3541,6 +4027,61 @@ function accumulateTermScore(docScores, internalId, termScore, collect, fieldNam
     });
   }
 }
+function topKFromMap(docScores, k, resolver) {
+  if (k <= 0) return [];
+  const heap = [];
+  for (const [internalId, data] of docScores) {
+    if (heap.length < k) {
+      heap.push({ internalId, score: data.score });
+      if (heap.length === k) buildMinHeap(heap);
+    } else if (data.score > heap[0].score) {
+      heap[0] = { internalId, score: data.score };
+      siftDown(heap, 0);
+    }
+  }
+  heap.sort((a, b) => b.score - a.score);
+  const result = [];
+  for (let i = 0; i < heap.length; i++) {
+    const data = docScores.get(heap[i].internalId);
+    if (!data) continue;
+    const externalId = resolver.toExternal(heap[i].internalId);
+    if (externalId === void 0) continue;
+    result.push({
+      docId: externalId,
+      score: data.score,
+      termFrequencies: data.termFrequencies,
+      fieldLengths: data.fieldLengths,
+      idf: data.idf
+    });
+  }
+  return result;
+}
+function buildMinHeap(heap) {
+  for (let i = (heap.length >> 1) - 1; i >= 0; i--) {
+    siftDown(heap, i);
+  }
+}
+function siftDown(heap, idx) {
+  const len = heap.length;
+  while (true) {
+    let smallest = idx;
+    const left = 2 * idx + 1;
+    const right = 2 * idx + 2;
+    if (left < len && heap[left].score < heap[smallest].score) smallest = left;
+    if (right < len && heap[right].score < heap[smallest].score) smallest = right;
+    if (smallest === idx) break;
+    const tmp = heap[idx];
+    heap[idx] = heap[smallest];
+    heap[smallest] = tmp;
+    idx = smallest;
+  }
+}
+
+// src/core/partition/search.ts
+var DEFAULT_MAX_RESULTS = 1e3;
+function globalDocFreqFor(docFreqs, term, fallback) {
+  return Object.hasOwn(docFreqs, term) ? docFreqs[term] : fallback;
+}
 function searchFulltext(state, params) {
   const {
     queryTokens,
@@ -3587,7 +4128,7 @@ function searchFulltext(state, params) {
       seen.add(term);
       const postingList = state.invertedIdx.lookup(term);
       if (!postingList) continue;
-      const docFreq = globalStats ? globalDocFreqs[term] ?? postingList.docIdSet.size : postingList.docIdSet.size;
+      const docFreq = globalStats ? globalDocFreqFor(globalDocFreqs, term, postingList.docIdSet.size) : postingList.docIdSet.size;
       found.push({ token: term, postingList, docFreq });
     }
     if (found.length === 0) return [];
@@ -3702,7 +4243,7 @@ function searchFulltext(state, params) {
       let totalPostings = 0;
       const matches = [];
       for (const m of rawMatches) {
-        const docFreq = globalStats ? globalDocFreqs[m.token] ?? m.postingList.docIdSet.size : m.postingList.docIdSet.size;
+        const docFreq = globalStats ? globalDocFreqFor(globalDocFreqs, m.token, m.postingList.docIdSet.size) : m.postingList.docIdSet.size;
         const idf = computeIDF(docFreq, totalDocs);
         totalPostings += m.postingList.length;
         matches.push({ token: m.token, docFreq, idf, postingList: m.postingList });
@@ -3773,7 +4314,7 @@ function searchFulltext(state, params) {
         return postingList ? [{ token: qt.token, postingList }] : [];
       })() : state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength);
       for (const match of matchingPostings) {
-        const docFreq = globalStats ? globalDocFreqs[match.token] ?? match.postingList.docIdSet.size : match.postingList.docIdSet.size;
+        const docFreq = globalStats ? globalDocFreqFor(globalDocFreqs, match.token, match.postingList.docIdSet.size) : match.postingList.docIdSet.size;
         const idf = computeIDF(docFreq, totalDocs);
         const list = match.postingList;
         const hasDeleted = list.deletedDocs.size > 0;
@@ -3816,55 +4357,6 @@ function searchFulltext(state, params) {
   const scored = topKFromMap(docScores, Math.min(k, totalMatched), resolver);
   return { scored, totalMatched };
 }
-function topKFromMap(docScores, k, resolver) {
-  if (k <= 0) return [];
-  const heap = [];
-  for (const [internalId, data] of docScores) {
-    if (heap.length < k) {
-      heap.push({ internalId, score: data.score });
-      if (heap.length === k) buildMinHeap(heap);
-    } else if (data.score > heap[0].score) {
-      heap[0] = { internalId, score: data.score };
-      siftDown(heap, 0);
-    }
-  }
-  heap.sort((a, b) => b.score - a.score);
-  const result = [];
-  for (let i = 0; i < heap.length; i++) {
-    const data = docScores.get(heap[i].internalId);
-    if (!data) continue;
-    const externalId = resolver.toExternal(heap[i].internalId);
-    if (externalId === void 0) continue;
-    result.push({
-      docId: externalId,
-      score: data.score,
-      termFrequencies: data.termFrequencies,
-      fieldLengths: data.fieldLengths,
-      idf: data.idf
-    });
-  }
-  return result;
-}
-function buildMinHeap(heap) {
-  for (let i = (heap.length >> 1) - 1; i >= 0; i--) {
-    siftDown(heap, i);
-  }
-}
-function siftDown(heap, idx) {
-  const len = heap.length;
-  while (true) {
-    let smallest = idx;
-    const left = 2 * idx + 1;
-    const right = 2 * idx + 2;
-    if (left < len && heap[left].score < heap[smallest].score) smallest = left;
-    if (right < len && heap[right].score < heap[smallest].score) smallest = right;
-    if (smallest === idx) break;
-    const tmp = heap[idx];
-    heap[idx] = heap[smallest];
-    heap[smallest] = tmp;
-    idx = smallest;
-  }
-}
 
 // src/core/partition/serialization.ts
 function serializePartition(state, partitionId, indexName, totalPartitions, language, schema) {
@@ -4337,6 +4829,9 @@ function createPartitionIndex(partitionId, trackPositions = true) {
         state.docStore.store(docId, document, stored.fieldLengths);
       }
     },
+    rebuildTextIndex(schema, language, options) {
+      rebuildTextIndex(state, schema, language, options);
+    },
     get(docId) {
       const stored = state.docStore.get(docId);
       if (!stored) return void 0;
@@ -4436,6 +4931,7 @@ function setNestedValue(obj, path, value) {
 }
 function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
   const count = initialPartitionCount ?? 1;
+  const analysis = resolveIndexAnalysis(config);
   const trackPositions = config.trackPositions ?? true;
   const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
   let partitions = [];
@@ -4454,14 +4950,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
   }
   function resolveInsertOptions(options) {
     const applyStrict = config.strict === true;
-    const applyAnalyzer = config.stopWords !== void 0 || config.tokenizer !== void 0;
+    const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
     const applySurfaces = config.surfaceForms === true;
     if (!applyStrict && !applyAnalyzer && !applySurfaces) return options;
     const resolved = { ...options };
     if (applyStrict) resolved.strict = true;
     if (applyAnalyzer) {
-      resolved.stopWordOverride = options?.stopWordOverride ?? config.stopWords;
-      resolved.customTokenizer = options?.customTokenizer ?? config.tokenizer;
+      resolved.stopWordOverride = options?.stopWordOverride ?? analysis.stopWords;
+      resolved.customTokenizer = options?.customTokenizer ?? analysis.customTokenizer;
     }
     if (applySurfaces) resolved.collectSurfaces = true;
     return resolved;
@@ -4500,6 +4996,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
     get config() {
       return config;
     },
+    get analysis() {
+      return analysis;
+    },
     getPartition(partitionId) {
       validatePartitionId(partitionId);
       return partitions[partitionId];
@@ -4532,24 +5031,44 @@ function createPartitionManager(indexName, config, language, router, initialPart
       partitions.splice(partitionId, 1);
       rebuildDocPartitionMap();
     },
-    insert(docId, document, options) {
-      const currentMaxDocs = config.partitions?.maxDocsPerPartition;
-      if (currentMaxDocs !== void 0) {
-        const totalCapacity = currentMaxDocs * partitions.length;
-        if (docPartitionMap.size >= totalCapacity) {
-          throw new NarsilError(
-            ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
-            `Index "${indexName}" has reached its capacity of ${totalCapacity} documents (${currentMaxDocs} per partition \xD7 ${partitions.length} partitions)`,
-            {
-              indexName,
-              currentCount: docPartitionMap.size,
-              totalCapacity,
-              maxDocsPerPartition: currentMaxDocs,
-              partitionCount: partitions.length
-            }
-          );
+    trimPartitions(newCount) {
+      if (newCount < 1) {
+        throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Cannot trim index "${indexName}" below one partition`, {
+          indexName,
+          requestedCount: newCount
+        });
+      }
+      if (newCount >= partitions.length) {
+        return;
+      }
+      for (const [docId, pid] of docPartitionMap) {
+        if (pid >= newCount) {
+          docPartitionMap.delete(docId);
         }
       }
+      partitions.length = newCount;
+    },
+    assertCapacity(pendingWrites = 0, partitionCountCap) {
+      const currentMaxDocs = config.partitions?.maxDocsPerPartition;
+      if (currentMaxDocs === void 0) return;
+      const effectivePartitionCount = partitionCountCap === void 0 ? partitions.length : Math.min(partitions.length, partitionCountCap);
+      const totalCapacity = currentMaxDocs * effectivePartitionCount;
+      if (docPartitionMap.size + pendingWrites >= totalCapacity) {
+        throw new NarsilError(
+          ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
+          `Index "${indexName}" has reached its capacity of ${totalCapacity} documents (${currentMaxDocs} per partition \xD7 ${effectivePartitionCount} partitions)`,
+          {
+            indexName,
+            currentCount: docPartitionMap.size + pendingWrites,
+            totalCapacity,
+            maxDocsPerPartition: currentMaxDocs,
+            partitionCount: effectivePartitionCount
+          }
+        );
+      }
+    },
+    insert(docId, document, options) {
+      manager.assertCapacity();
       const pid = router.route(docId, partitions.length);
       const insertOpts = resolveInsertOptions(options);
       partitions[pid].insert(docId, document, config.schema, language, insertOpts);
@@ -4563,6 +5082,10 @@ function createPartitionManager(indexName, config, language, router, initialPart
       partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
       docPartitionMap.delete(docId);
     },
+    rebuildTextIndex(partitionId) {
+      validatePartitionId(partitionId);
+      partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
+    },
     beginBatchRemove() {
       for (let i = 0; i < partitions.length; i++) {
         partitions[i].beginBatch();
@@ -4620,8 +5143,8 @@ function createPartitionManager(indexName, config, language, router, initialPart
     },
     getAggregateStats() {
       let totalDocuments = 0;
-      const docFrequencies = {};
-      const totalFieldLengths = {};
+      const docFrequencies = /* @__PURE__ */ Object.create(null);
+      const totalFieldLengths = /* @__PURE__ */ Object.create(null);
       for (let i = 0; i < partitions.length; i++) {
         const stats = partitions[i].stats;
         totalDocuments += stats.totalDocuments;
@@ -4706,8 +5229,33 @@ function createPartitionRouter() {
   return { route, routeBatch };
 }
 
-// src/vector/scalar-quantization.ts
+// src/vector/scalar-quantization-calibration.ts
 var PADDING_FACTOR = 0.01;
+function computeCalibrationBounds(vectors, dimensions) {
+  let globalMin = Number.POSITIVE_INFINITY;
+  let globalMax = Number.NEGATIVE_INFINITY;
+  let count = 0;
+  for (const vec of vectors) {
+    for (let d = 0; d < dimensions; d++) {
+      const val = vec[d];
+      if (val < globalMin) globalMin = val;
+      if (val > globalMax) globalMax = val;
+    }
+    count++;
+  }
+  if (count === 0) return null;
+  const range = globalMax - globalMin;
+  const pad = range * PADDING_FACTOR;
+  globalMin -= pad;
+  globalMax += pad;
+  if (globalMin === globalMax) {
+    globalMin -= 1e-3;
+    globalMax += 1e-3;
+  }
+  return { alpha: (globalMax - globalMin) / 255, offset: globalMin };
+}
+
+// src/vector/scalar-quantization.ts
 var INITIAL_CAPACITY2 = 16;
 var PAGE_BYTES = 65536;
 function createScalarQuantizer(dimensions, ordinalSource) {
@@ -4811,28 +5359,10 @@ function createScalarQuantizer(dimensions, ordinalSource) {
     docToOrd.set(docId, ord);
   }
   function calibrateFromVectors(vectors) {
-    let globalMin = Number.POSITIVE_INFINITY;
-    let globalMax = Number.NEGATIVE_INFINITY;
-    let count = 0;
-    for (const vec of vectors) {
-      for (let d = 0; d < dimensions; d++) {
-        const val = vec[d];
-        if (val < globalMin) globalMin = val;
-        if (val > globalMax) globalMax = val;
-      }
-      count++;
-    }
-    if (count === 0) return;
-    const range = globalMax - globalMin;
-    const pad = range * PADDING_FACTOR;
-    globalMin -= pad;
-    globalMax += pad;
-    if (globalMin === globalMax) {
-      globalMin -= 1e-3;
-      globalMax += 1e-3;
-    }
-    alpha = (globalMax - globalMin) / 255;
-    offset = globalMin;
+    const bounds = computeCalibrationBounds(vectors, dimensions);
+    if (bounds === null) return;
+    alpha = bounds.alpha;
+    offset = bounds.offset;
     updateDerivedConstants();
     calibrated = true;
   }
@@ -5044,20 +5574,6 @@ function createScalarQuantizer(dimensions, ordinalSource) {
     }
   };
 }
-function deserializeScalarQuantizer(data, dimensions, ordinalSource) {
-  const quantizer = createScalarQuantizer(dimensions, ordinalSource);
-  if (data.alpha === 0 && data.offset === 0 && Object.keys(data.quantizedVectors).length === 0) {
-    return quantizer;
-  }
-  quantizer.restoreCalibration(data.alpha, data.offset);
-  for (const [docId, values] of Object.entries(data.quantizedVectors)) {
-    const quantized = new Uint8Array(values);
-    const sum = data.vectorSums[docId] ?? 0;
-    const sumSq = data.vectorSumSqs[docId] ?? 0;
-    quantizer.restoreEntry(docId, quantized, sum, sumSq);
-  }
-  return quantizer;
-}
 
 // src/workers/node-worker.ts
 async function spawnNodeWorker(entryPoint) {
@@ -5499,6 +6015,22 @@ function estimateMemoryBytes(state) {
   return Math.round(bytes);
 }
 
+// src/vector/scalar-quantization-restore.ts
+function deserializeScalarQuantizer(data, dimensions, ordinalSource) {
+  const quantizer = createScalarQuantizer(dimensions, ordinalSource);
+  if (data.alpha === 0 && data.offset === 0 && Object.keys(data.quantizedVectors).length === 0) {
+    return quantizer;
+  }
+  quantizer.restoreCalibration(data.alpha, data.offset);
+  for (const [docId, values] of Object.entries(data.quantizedVectors)) {
+    const quantized = new Uint8Array(values);
+    const sum = data.vectorSums[docId] ?? 0;
+    const sumSq = data.vectorSumSqs[docId] ?? 0;
+    quantizer.restoreEntry(docId, quantized, sum, sumSq);
+  }
+  return quantizer;
+}
+
 // src/vector/vector-index/persistence.ts
 function serialize(state) {
   const vectors = [];
@@ -5846,7 +6378,17 @@ function createDirectExecutor() {
       vectorIndexes.set(fieldPath, createVectorIndex(fieldPath, dim, config.vectorPromotion));
     }
     const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
-    indexes.set(indexName, { manager, config, language, vectorIndexes });
+    indexes.set(indexName, {
+      manager,
+      config,
+      language,
+      searchOptions: {
+        bm25Params: config.bm25,
+        stopWords: manager.analysis.stopWords,
+        customTokenizer: manager.analysis.customTokenizer
+      },
+      vectorIndexes
+    });
   }
   function dropIndex(indexName) {
     const entry = requireIndex(indexName);
@@ -5863,6 +6405,15 @@ function createDirectExecutor() {
   }
   async function execute(action) {
     switch (action.type) {
+      case "bootstrap": {
+        if (typeof action.moduleUrl !== "string" || action.moduleUrl.trim().length === 0) {
+          throw new NarsilError(ErrorCodes.CONFIG_INVALID, "A bootstrap module needs a non-empty module URL", {
+            moduleUrl: action.moduleUrl
+          });
+        }
+        await import(action.moduleUrl);
+        return void 0;
+      }
       case "createIndex": {
         const language = getLanguage(action.config.language ?? "english");
         createIndex(action.indexName, action.config, language);
@@ -5889,17 +6440,30 @@ function createDirectExecutor() {
       }
       case "query": {
         const entry = requireIndex(action.indexName);
-        const result = await fanOutQuery(entry.manager, action.params, entry.language, entry.config.schema, {
-          scoringMode: entry.config.defaultScoring ?? "local",
-          partitionIds: action.partitionIds
-        });
+        const result = await fanOutQuery(
+          entry.manager,
+          action.params,
+          entry.language,
+          entry.config.schema,
+          {
+            scoringMode: action.params.scoring ?? entry.config.defaultScoring ?? "local",
+            partitionIds: action.partitionIds,
+            ...action.globalStats !== void 0 ? { globalStats: sanitizeGlobalStats(action.globalStats) } : {}
+          },
+          entry.searchOptions
+        );
         return result;
       }
       case "preflight": {
         const entry = requireIndex(action.indexName);
-        const result = await fanOutQuery(entry.manager, action.params, entry.language, entry.config.schema, {
-          scoringMode: entry.config.defaultScoring ?? "local"
-        });
+        const result = await fanOutQuery(
+          entry.manager,
+          action.params,
+          entry.language,
+          entry.config.schema,
+          { scoringMode: entry.config.defaultScoring ?? "local" },
+          entry.searchOptions
+        );
         return { count: result.totalMatched };
       }
       case "get": {
@@ -5945,6 +6509,9 @@ function createDirectExecutor() {
       }
       case "deserialize": {
         const entry = requireIndex(action.indexName);
+        while (entry.manager.partitionCount <= action.partitionId) {
+          entry.manager.addPartition();
+        }
         entry.manager.deserializePartition(action.partitionId, action.data);
         return void 0;
       }
@@ -5976,4 +6543,4 @@ function createDirectExecutor() {
   };
 }
 
-export { MAX_NESTING_DEPTH, clearNormalizationCache, configureNormalizationCache, createDirectExecutor, createPartitionIndex, createPartitionManager, createPartitionRouter, createVectorIndex, deserializeMetadata, deserializePayloadV1, deserializePayloadV2, detectRuntime, extractVectorFieldsFromSchema, fanOutQuery, flattenSchema, fnv1a, getLanguage, getNestedValue, getNormalizationCacheSize, isNodeMainThread, kWayMerge, mergePartitionStats, registerLanguage, resetNormalizationCache, serializeMetadata, spawnNodeWorker, tokenize, validateRequiredFields, validateSchema, validateVectorPromotion };
+export { MAX_NESTING_DEPTH, clearNormalizationCache, configureNormalizationCache, createDirectExecutor, createPartitionIndex, createPartitionManager, createPartitionRouter, createVectorIndex, deserializeMetadata, deserializePayloadV1, deserializePayloadV2, detectRuntime, extractVectorFieldsFromSchema, fanOutQuery, flattenSchema, fnv1a, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNodeMainThread, kWayMerge, mergePartitionStats, pruneStatsToQueryTerms, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache, resolveIndexAnalysis, serializeMetadata, spawnNodeWorker, tokenize, validateRequiredFields, validateSchema, validateVectorPromotion };