@delali/narsil 0.1.0 → 0.1.2

package/dist/{chunk-5HXMVMSM.mjs → chunk-LIRVYUQ5.mjs}

@@ -1,5 +1,7 @@
+import { createHNSWIndex, magnitude, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-GVN2W4VA.mjs';
 import { english } from './chunk-WYC635GI.mjs';
-import { NarsilError, ErrorCodes } from './chunk-JKIBUPAY.mjs';
+import { NarsilError, ErrorCodes } from './chunk-7GKPWGCN.mjs';
+import { decode, encode } from '@msgpack/msgpack';
 
 // src/languages/registry.ts
 var languages = /* @__PURE__ */ new Map();
@@ -95,6 +97,9 @@ var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{N}_'-]+/u;
 var DEFAULT_MIN_TOKEN_LENGTH = 1;
 var normalizationCache = /* @__PURE__ */ new Map();
 var MAX_CACHE_SIZE = 65536;
+var cachedLangName = "";
+var cachedFlags = "";
+var cachedPrefix = "";
 function stripDiacritics(text) {
   return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
 }
@@ -103,9 +108,17 @@ function resolveStopWords(language, override) {
   if (override instanceof Set) return override;
   return override(language.stopWords);
 }
+function getCachePrefix(language, flags) {
+  if (language.name === cachedLangName && flags === cachedFlags) return cachedPrefix;
+  cachedLangName = language.name;
+  cachedFlags = flags;
+  cachedPrefix = `${language.name}:${flags}:`;
+  return cachedPrefix;
+}
 function transformToken(raw, language, stem, removeDiacritics) {
   const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
-  const cacheKey = `${language.name}:${flags}:${raw}`;
+  const prefix = getCachePrefix(language, flags);
+  const cacheKey = prefix + raw;
   const cached = normalizationCache.get(cacheKey);
   if (cached !== void 0) return cached;
   let normalized = raw;
@@ -115,20 +128,23 @@ function transformToken(raw, language, stem, removeDiacritics) {
   if (stem && language.stemmer) {
     normalized = language.stemmer(normalized);
   }
-  cacheAndReturn(cacheKey, normalized);
-  return normalized;
-}
-function cacheAndReturn(key, value) {
   if (normalizationCache.size >= MAX_CACHE_SIZE) {
     const firstKey = normalizationCache.keys().next().value;
     normalizationCache.delete(firstKey);
   }
-  normalizationCache.set(key, value);
+  normalizationCache.set(cacheKey, normalized);
+  return normalized;
 }
 function splitText(text, language) {
   const pattern = language.tokenizer?.splitPattern ?? DEFAULT_SPLIT_PATTERN;
   return text.split(pattern);
 }
+function isAscii(text) {
+  for (let i = 0; i < text.length; i++) {
+    if (text.charCodeAt(i) > 127) return false;
+  }
+  return true;
+}
 function tokenize(text, language, options) {
   const {
     stem = true,
@@ -144,7 +160,7 @@ function tokenize(text, language, options) {
       originalTokens: customResult.map((t) => t.token)
     };
   }
-  const normalized = text.normalize("NFC").toLowerCase();
+  const normalized = isAscii(text) ? text.toLowerCase() : text.normalize("NFC").toLowerCase();
   const rawParts = splitText(normalized, language);
   const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
   const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
@@ -440,6 +456,31 @@ function validateDocument(document, schema) {
   }
   validateDocumentFields(document, schema, "");
 }
+function collectExtraFields(doc, schema, prefix, extras) {
+  for (const key of Object.keys(doc)) {
+    if (key === "id") continue;
+    const path = prefix ? `${prefix}.${key}` : key;
+    if (!(key in schema)) {
+      extras.push(path);
+      continue;
+    }
+    const schemaType = schema[key];
+    if (isPlainObject(schemaType) && isPlainObject(doc[key])) {
+      collectExtraFields(doc[key], schemaType, path, extras);
+    }
+  }
+}
+function validateDocumentStrict(document, schema) {
+  const extras = [];
+  collectExtraFields(document, schema, "", extras);
+  if (extras.length > 0) {
+    throw new NarsilError(
+      ErrorCodes.DOC_VALIDATION_FAILED,
+      `Document contains fields not defined in schema: ${extras.join(", ")}`,
+      { extraFields: extras }
+    );
+  }
+}
 function flattenRecursive(schema, prefix, result) {
   for (const [field, type] of Object.entries(schema)) {
     const path = prefix ? `${prefix}.${field}` : field;
@@ -478,6 +519,9 @@ function fulltextSearch(partition, params, language, schema, options) {
     return { scored: [], totalMatched: 0 };
   }
   const queryTokens = deduplicateTokens(queryTokenResult.tokens);
+  const hasPostFilters = params.filters !== void 0 || params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any";
+  const requestedLimit = params.limit !== void 0 || params.offset !== void 0 ? (params.limit ?? 10) + (params.offset ?? 0) + 1 : void 0;
+  const maxResults = requestedLimit !== void 0 && !hasPostFilters ? requestedLimit : void 0;
   const rawResult = partition.searchFulltext({
     queryTokens,
     fields: params.fields,
@@ -486,7 +530,9 @@ function fulltextSearch(partition, params, language, schema, options) {
     prefixLength: params.prefixLength ?? 2,
     exact: params.exact ?? false,
     bm25Params: options?.bm25Params,
-    globalStats: options?.globalStats
+    globalStats: options?.globalStats,
+    maxResults,
+    termMatch: params.termMatch
   });
   let scored = rawResult.scored;
   const termMatch = params.termMatch ?? "any";
@@ -501,7 +547,8 @@ function fulltextSearch(partition, params, language, schema, options) {
     const matchingDocIds = partition.applyFilters(params.filters, schema);
     scored = scored.filter((doc) => matchingDocIds.has(doc.docId));
   }
-  return { scored, totalMatched: scored.length };
+  const totalMatched = hasPostFilters ? scored.length : rawResult.totalMatched;
+  return { scored, totalMatched };
 }
 function validateSearchFields(fields, flatSchema) {
   for (const field of fields) {
@@ -581,7 +628,8 @@ function hybridSearch(partition, params, language, schema, options) {
     value: vectorConfig.value,
     k: params.limit ?? 10,
     similarity: vectorConfig.similarity,
-    metric: vectorConfig.metric
+    metric: vectorConfig.metric,
+    efSearch: vectorConfig.efSearch
   });
   const normalizedText = normalizeScores(textResult.scored);
   const normalizedVector = normalizeScores(vectorResult.scored);
@@ -636,7 +684,8 @@ function runVectorOnly(partition, params) {
     value: vectorConfig.value,
     k: params.limit ?? 10,
     similarity: vectorConfig.similarity,
-    metric: vectorConfig.metric
+    metric: vectorConfig.metric,
+    efSearch: vectorConfig.efSearch
   });
   return result;
 }
@@ -680,8 +729,8 @@ function collectGlobalStats(manager) {
 }
 
 // src/partitioning/fan-out.ts
-function fanOutQuery(manager, params, language, schema, config, searchOptions) {
-  const partitions = manager.getAllPartitions();
+async function fanOutQuery(manager, params, language, schema, config, searchOptions) {
+  const partitions = config.partitionIds ? config.partitionIds.map((id) => manager.partitionAt(id)).filter((p) => p !== void 0) : manager.getAllPartitions();
   if (partitions.length === 0) {
     return { scored: [], totalMatched: 0 };
   }
@@ -693,7 +742,16 @@ function fanOutQuery(manager, params, language, schema, config, searchOptions) {
     globalStats = config.globalStats;
   }
   const options = buildSearchOptions(searchOptions, globalStats);
-  const outcomes = dispatchToAllPartitions(partitions, params, language, schema, options);
+  if (partitions.length === 1 && !config.dispatcher) {
+    const result = dispatchSinglePartition(partitions[0], params, language, schema, options);
+    let facets2;
+    if (params.facets) {
+      const matchingDocIds = new Set(result.scored.map((doc) => doc.docId));
+      facets2 = partitions[0].computeFacets(matchingDocIds, params.facets, schema);
+    }
+    return { scored: result.scored, totalMatched: result.totalMatched, facets: facets2 };
+  }
+  const outcomes = await dispatchToAllPartitions(partitions, params, language, schema, options, config.dispatcher);
   const allScoredArrays = outcomes.map((o) => o.result.scored);
   const merged = kWayMerge(allScoredArrays);
   let totalMatched = 0;
@@ -718,13 +776,19 @@ function buildSearchOptions(base, globalStats) {
     globalStats: globalStats ?? base?.globalStats
   };
 }
-function dispatchToAllPartitions(partitions, params, language, schema, options) {
-  const outcomes = [];
-  for (const partition of partitions) {
-    const result = dispatchSinglePartition(partition, params, language, schema, options);
-    outcomes.push({ result, partition });
+function dispatchToAllPartitions(partitions, params, language, schema, options, dispatcher) {
+  if (!dispatcher) {
+    const outcomes = [];
+    for (const partition of partitions) {
+      const result = dispatchSinglePartition(partition, params, language, schema, options);
+      outcomes.push({ result, partition });
+    }
+    return outcomes;
   }
-  return outcomes;
+  const promises = partitions.map(
+    (partition) => dispatcher.dispatch(partition, params, language, schema, options).then((result) => ({ result, partition }))
+  );
+  return Promise.all(promises);
 }
 function dispatchSinglePartition(partition, params, language, schema, options) {
   const isHybridMode = params.mode === "hybrid";
@@ -741,7 +805,8 @@ function dispatchSinglePartition(partition, params, language, schema, options) {
       value: vectorConfig.value,
       k: params.limit ?? 10,
       similarity: vectorConfig.similarity,
-      metric: vectorConfig.metric
+      metric: vectorConfig.metric,
+      efSearch: vectorConfig.efSearch
     });
   }
   return fulltextSearch(partition, params, language, schema, options);
@@ -871,6 +936,280 @@ function heapPop(heap) {
   }
   return top;
 }
+function encodeRawPayloadV2(wire) {
+  return encode(wire);
+}
+var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
+function validateHnswMetric(value) {
+  if (value === void 0 || value === null) return void 0;
+  if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
+    return value;
+  }
+  return void 0;
+}
+function deserializePayloadV2(data) {
+  const raw = decode(data);
+  const documents = {};
+  for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
+    documents[docId] = {
+      fields: doc.fields,
+      fieldLengths: doc.field_lengths ?? {}
+    };
+  }
+  const fieldNames = raw.inverted_index?.field_names ?? [];
+  const invertedIndex = {};
+  for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
+    const count = list.ids.length;
+    const postings = new Array(count);
+    for (let i = 0; i < count; i++) {
+      postings[i] = {
+        docId: list.ids[i],
+        termFrequency: list.tf[i],
+        field: fieldNames[list.fi[i]] ?? "",
+        positions: list.pos ? list.pos[i] : []
+      };
+    }
+    invertedIndex[token] = {
+      docFrequency: list.df,
+      postings
+    };
+  }
+  const numeric = {};
+  for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
+    numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
+  }
+  const boolean = {};
+  for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
+    boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
+  }
+  const geopoint = {};
+  for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
+    geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
+  }
+  const vectorData = {};
+  for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
+    vectorData[field] = {
+      dimension: data2.dimension,
+      vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
+      hnswGraph: data2.hnsw_graph ? {
+        entryPoint: data2.hnsw_graph.entry_point,
+        maxLayer: data2.hnsw_graph.max_layer,
+        m: data2.hnsw_graph.m,
+        efConstruction: data2.hnsw_graph.ef_construction,
+        metric: validateHnswMetric(data2.hnsw_graph.metric),
+        nodes: data2.hnsw_graph.nodes
+      } : null
+    };
+  }
+  return {
+    indexName: raw.index_name,
+    partitionId: raw.partition_id,
+    totalPartitions: raw.total_partitions,
+    language: raw.language ?? "english",
+    schema: raw.schema ?? {},
+    docCount: raw.doc_count ?? 0,
+    avgDocLength: raw.avg_doc_length ?? 0,
+    documents,
+    invertedIndex,
+    fieldIndexes: {
+      numeric,
+      boolean,
+      enum: raw.field_indexes?.enum ?? {},
+      geopoint
+    },
+    vectorData,
+    statistics: {
+      totalDocuments: raw.statistics?.total_documents ?? 0,
+      totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
+      averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
+      docFrequencies: raw.statistics?.doc_frequencies ?? {}
+    }
+  };
+}
+
+// src/runtime/detect.ts
+function detectRuntimeName() {
+  if ("Bun" in globalThis) {
+    return "bun";
+  }
+  if ("Deno" in globalThis) {
+    return "deno";
+  }
+  const proc = globalThis.process;
+  if (typeof proc === "object" && proc !== null && typeof proc.versions === "object" && typeof proc.versions?.node === "string") {
+    return "node";
+  }
+  return "browser";
+}
+function getCpuCount() {
+  const nav = globalThis.navigator;
+  if (nav && typeof nav.hardwareConcurrency === "number" && nav.hardwareConcurrency >= 1) {
+    return nav.hardwareConcurrency;
+  }
+  return 1;
+}
+function detectRuntime() {
+  const runtime = detectRuntimeName();
+  return {
+    runtime,
+    cpuCount: getCpuCount(),
+    supportsWorkerThreads: runtime === "node" || runtime === "bun",
+    supportsWebWorkers: runtime === "browser" || runtime === "deno" || runtime === "bun",
+    supportsFileSystem: runtime === "node" || runtime === "bun" || runtime === "deno",
+    supportsIndexedDB: runtime === "browser" || "indexedDB" in globalThis,
+    supportsBroadcastChannel: "BroadcastChannel" in globalThis
+  };
+}
+
+// src/vector/promoter.ts
+function detectWorkerStrategy() {
+  const runtime = detectRuntime();
+  const url = import.meta.url;
+  const hasBundledWorker = url.endsWith(".mjs") || url.endsWith(".js");
+  if (runtime.supportsWorkerThreads && hasBundledWorker) return "worker-threads";
+  if (runtime.supportsWebWorkers && hasBundledWorker) return "web-worker";
+  return "synchronous";
+}
+function collectVectorsForWorker(engine) {
+  const vectors = [];
+  for (const [, entry] of engine.entries()) {
+    vectors.push({ docId: entry.docId, values: Array.from(entry.vector) });
+  }
+  return vectors;
+}
+function buildWorkerRequest(engine, hnswConfig) {
+  return {
+    type: "build",
+    vectors: collectVectorsForWorker(engine),
+    dimension: engine.dimension,
+    config: hnswConfig ?? {}
+  };
+}
+function resolveWorkerUrl() {
+  const base = import.meta.url;
+  if (base.endsWith(".mjs") || base.endsWith(".js")) {
+    return new URL("./hnsw-build-worker.mjs", base).href;
+  }
+  const replaced = base.replace(/\/src\/vector\/[^/]+$/, "/dist/vector/hnsw-build-worker.mjs");
+  return replaced;
+}
+function spawnNodeWorker(request, onComplete, onError) {
+  const entryUrl = resolveWorkerUrl();
+  let worker = null;
+  import('worker_threads').then((wt) => {
+    worker = new wt.Worker(new URL(entryUrl));
+    worker.on("message", (msg) => {
+      const response = msg;
+      worker?.terminate();
+      if (response.type === "success") {
+        onComplete(response.graph);
+      } else {
+        onError(new Error(response.message));
+      }
+    });
+    worker.on("error", (err) => {
+      worker?.terminate();
+      onError(err instanceof Error ? err : new Error(String(err)));
+    });
+    worker.postMessage(request);
+  }).catch(onError);
+  return {
+    terminate() {
+      worker?.terminate();
+    }
+  };
+}
+function spawnWebWorker(request, onComplete, onError) {
+  const entryUrl = resolveWorkerUrl();
+  const WorkerCtor = globalThis.Worker;
+  const worker = new WorkerCtor(entryUrl, { type: "module" });
+  worker.onmessage = (event) => {
+    const response = event.data;
+    worker.terminate();
+    if (response.type === "success") {
+      onComplete(response.graph);
+    } else {
+      onError(new Error(response.message));
+    }
+  };
+  worker.onerror = (event) => {
+    worker.terminate();
+    onError(event.error ?? new Error(event.message ?? "Web Worker error"));
+  };
+  worker.postMessage(request);
+  return {
+    terminate() {
+      worker.terminate();
+    }
+  };
+}
+function createVectorPromoter(config) {
+  const threshold = config?.promotionThreshold ?? 1e4;
+  const hnswConfig = config?.hnswConfig;
+  const promoting = /* @__PURE__ */ new Set();
+  const strategy = config?.workerStrategy ?? detectWorkerStrategy();
+  const pendingWorkers = /* @__PURE__ */ new Set();
+  return {
+    get strategy() {
+      return strategy;
+    },
+    check(engines) {
+      for (const [field, engine] of engines) {
+        if (engine.isPromoted || promoting.has(field)) continue;
+        if (engine.size < threshold) continue;
+        promoting.add(field);
+        if (strategy === "worker-threads") {
+          const request = buildWorkerRequest(engine, hnswConfig);
+          const handle = spawnNodeWorker(
+            request,
+            (graph) => {
+              pendingWorkers.delete(handle);
+              if (!engine.isPromoted) {
+                engine.deserializeHNSW(graph);
+              }
+              promoting.delete(field);
+            },
+            () => {
+              pendingWorkers.delete(handle);
+              promoting.delete(field);
+            }
+          );
+          pendingWorkers.add(handle);
+        } else if (strategy === "web-worker") {
+          const request = buildWorkerRequest(engine, hnswConfig);
+          const handle = spawnWebWorker(
+            request,
+            (graph) => {
+              pendingWorkers.delete(handle);
+              if (!engine.isPromoted) {
+                engine.deserializeHNSW(graph);
+              }
+              promoting.delete(field);
+            },
+            () => {
+              pendingWorkers.delete(handle);
+              promoting.delete(field);
+            }
+          );
+          pendingWorkers.add(handle);
+        } else {
+          try {
+            engine.promoteToHNSW(hnswConfig);
+          } finally {
+            promoting.delete(field);
+          }
+        }
+      }
+    },
+    shutdown() {
+      for (const handle of pendingWorkers) {
+        handle.terminate();
+      }
+      pendingWorkers.clear();
+      promoting.clear();
+    }
+  };
+}
 
 // src/core/document-store.ts
 function createDocumentStore() {
@@ -879,6 +1218,9 @@ function createDocumentStore() {
     store(docId, document, fieldLengths) {
       docs.set(docId, { fields: structuredClone(document), fieldLengths });
     },
+    storeRef(docId, document, fieldLengths) {
+      docs.set(docId, { fields: document, fieldLengths });
+    },
     get(docId) {
       return docs.get(docId);
     },
@@ -914,7 +1256,10 @@ function createDocumentStore() {
 }
 
 // src/core/inverted-index.ts
-function createInvertedIndex() {
+var INITIAL_CAPACITY = 8;
+var MAX_TERM_FREQUENCY = 65535;
+var COMPACTION_THRESHOLD = 0.3;
+function createInvertedIndex(fieldNameTable) {
   const index = /* @__PURE__ */ new Map();
   const charBuckets = /* @__PURE__ */ new Map();
   function trackToken(token) {
@@ -953,33 +1298,112 @@ function createInvertedIndex() {
     }
     return filtered;
   }
+  function getOrCreateList(token) {
+    let list = index.get(token);
+    if (!list) {
+      list = {
+        length: 0,
+        docIds: [],
+        termFrequencies: new Uint16Array(INITIAL_CAPACITY),
+        fieldNameIndices: new Uint8Array(INITIAL_CAPACITY),
+        positions: null,
+        docIdSet: /* @__PURE__ */ new Set(),
+        deletedDocs: /* @__PURE__ */ new Set()
+      };
+      index.set(token, list);
+      trackToken(token);
+    }
+    return list;
+  }
+  function growTypedArrays(list) {
+    const newCap = list.termFrequencies.length * 2;
+    const newTF = new Uint16Array(newCap);
+    newTF.set(list.termFrequencies);
+    list.termFrequencies = newTF;
+    const newFNI = new Uint8Array(newCap);
+    newFNI.set(list.fieldNameIndices);
+    list.fieldNameIndices = newFNI;
+  }
+  function compactList(list) {
+    if (list.deletedDocs.size === 0) return;
+    let writeIdx = 0;
+    for (let i = 0; i < list.length; i++) {
+      if (!list.deletedDocs.has(list.docIds[i])) {
+        if (writeIdx !== i) {
+          list.docIds[writeIdx] = list.docIds[i];
+          list.termFrequencies[writeIdx] = list.termFrequencies[i];
+          list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
+          if (list.positions) {
+            list.positions[writeIdx] = list.positions[i];
+          }
+        }
+        writeIdx++;
+      }
+    }
+    list.docIds.length = writeIdx;
+    if (list.positions) list.positions.length = writeIdx;
+    list.length = writeIdx;
+    list.deletedDocs.clear();
+  }
+  function compactDocEntries(list, docId) {
+    let writeIdx = 0;
+    for (let i = 0; i < list.length; i++) {
+      if (list.docIds[i] !== docId) {
+        if (writeIdx !== i) {
+          list.docIds[writeIdx] = list.docIds[i];
+          list.termFrequencies[writeIdx] = list.termFrequencies[i];
+          list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
+          if (list.positions) {
+            list.positions[writeIdx] = list.positions[i];
+          }
+        }
+        writeIdx++;
+      }
+    }
+    list.docIds.length = writeIdx;
+    if (list.positions) list.positions.length = writeIdx;
+    list.length = writeIdx;
+  }
   return {
-    insert(token, entry) {
-      let list = index.get(token);
-      if (!list) {
-        list = { docFrequency: 0, postings: [] };
-        index.set(token, list);
-        trackToken(token);
+    insert(token, docId, termFrequency, fieldNameIndex, positions) {
+      const list = getOrCreateList(token);
+      if (list.deletedDocs.size > 0 && list.deletedDocs.has(docId)) {
+        compactDocEntries(list, docId);
+        list.deletedDocs.delete(docId);
       }
-      const dupeIdx = list.postings.findIndex((p) => p.docId === entry.docId && p.fieldName === entry.fieldName);
-      if (dupeIdx !== -1) {
-        list.postings[dupeIdx] = entry;
-        return;
+      if (list.length >= list.termFrequencies.length) {
+        growTypedArrays(list);
+      }
+      const idx = list.length;
+      list.docIds.push(docId);
+      list.termFrequencies[idx] = termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : termFrequency;
+      list.fieldNameIndices[idx] = fieldNameIndex;
+      if (positions !== null) {
+        if (!list.positions) {
+          list.positions = [];
+          for (let j = 0; j < idx; j++) list.positions.push([]);
+        }
+        list.positions.push(positions);
+      } else if (list.positions !== null) {
+        list.positions.push([]);
+      }
+      if (!list.docIdSet.has(docId)) {
+        list.docIdSet.add(docId);
       }
-      const isNewDoc = !list.postings.some((p) => p.docId === entry.docId);
-      list.postings.push(entry);
-      if (isNewDoc) list.docFrequency++;
+      list.length++;
     },
     remove(token, docId) {
       const list = index.get(token);
-      if (!list) return;
-      const lengthBefore = list.postings.length;
-      list.postings = list.postings.filter((p) => p.docId !== docId);
-      const removed = lengthBefore - list.postings.length;
-      if (removed > 0) list.docFrequency--;
-      if (list.postings.length === 0) {
+      if (!list || !list.docIdSet.has(docId)) return;
+      list.docIdSet.delete(docId);
+      list.deletedDocs.add(docId);
+      if (list.docIdSet.size === 0) {
         index.delete(token);
         untrackToken(token);
+        return;
+      }
+      if (list.deletedDocs.size / list.length > COMPACTION_THRESHOLD) {
+        compactList(list);
       }
     },
     lookup(token) {
@@ -1001,6 +1425,20 @@ function createInvertedIndex() {
       }
       return results;
     },
+    prefixSearch(prefix, limit) {
+      if (prefix.length === 0 || limit <= 0) return [];
+      const candidates = candidatesForPrefix(prefix, prefix.length);
+      const results = [];
+      for (const term of candidates) {
+        if (!term.startsWith(prefix)) continue;
+        const list = index.get(term);
+        if (!list) continue;
+        results.push({ term, documentFrequency: list.docIdSet.size });
+      }
+      results.sort((a, b) => b.documentFrequency - a.documentFrequency);
+      if (results.length > limit) results.length = limit;
+      return results;
+    },
     has(token) {
       return index.has(token);
     },
@@ -1017,7 +1455,19 @@ function createInvertedIndex() {
     serialize() {
       const result = /* @__PURE__ */ Object.create(null);
       for (const [token, list] of index) {
-        result[token] = list;
+        const postings = [];
+        for (let i = 0; i < list.length; i++) {
+          if (list.deletedDocs.size > 0 && list.deletedDocs.has(list.docIds[i])) continue;
+          postings.push({
+            docId: list.docIds[i],
+            termFrequency: list.termFrequencies[i],
+            fieldName: fieldNameTable.names[list.fieldNameIndices[i]],
+            positions: list.positions ? list.positions[i] : []
+          });
+        }
+        if (postings.length > 0) {
+          result[token] = { docFrequency: list.docIdSet.size, postings };
+        }
       }
       return result;
     },
@@ -1025,7 +1475,43 @@ function createInvertedIndex() {
       index.clear();
       charBuckets.clear();
       for (const token of Object.keys(data)) {
-        index.set(token, data[token]);
+        const src = data[token];
+        const count = src.postings.length;
+        const docIdSet = /* @__PURE__ */ new Set();
+        const docIds = new Array(count);
+        const termFrequencies = new Uint16Array(count);
+        const fieldNameIndices = new Uint8Array(count);
+        let hasPositions = false;
+        for (let i = 0; i < count; i++) {
+          const p = src.postings[i];
+          docIds[i] = p.docId;
+          termFrequencies[i] = p.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : p.termFrequency;
+          let fnIndex = fieldNameTable.indexMap.get(p.fieldName);
+          if (fnIndex === void 0) {
+            fnIndex = fieldNameTable.names.length;
+            fieldNameTable.names.push(p.fieldName);
+            fieldNameTable.indexMap.set(p.fieldName, fnIndex);
+          }
+          fieldNameIndices[i] = fnIndex;
+          docIdSet.add(p.docId);
+          if (p.positions.length > 0) hasPositions = true;
+        }
+        let positions = null;
+        if (hasPositions) {
+          positions = new Array(count);
+          for (let i = 0; i < count; i++) {
+            positions[i] = src.postings[i].positions;
+          }
+        }
+        index.set(token, {
+          length: count,
+          docIds,
+          termFrequencies,
+          fieldNameIndices,
+          positions,
+          docIdSet,
+          deletedDocs: /* @__PURE__ */ new Set()
+        });
         trackToken(token);
       }
     }
@@ -1047,9 +1533,9 @@ function collectUniqueTokens(tokens) {
 function createPartitionStats() {
   const stats = {
     totalDocuments: 0,
-    totalFieldLengths: {},
-    averageFieldLengths: {},
-    docFrequencies: {},
+    totalFieldLengths: /* @__PURE__ */ Object.create(null),
+    averageFieldLengths: /* @__PURE__ */ Object.create(null),
+    docFrequencies: /* @__PURE__ */ Object.create(null),
     addDocument(fieldLengths, tokens) {
       stats.totalDocuments++;
       const fields = Object.keys(fieldLengths);
@@ -1089,7 +1575,7 @@ function createPartitionStats() {
     },
     recalculateAverages() {
       const fields = Object.keys(stats.totalFieldLengths);
-      const newAverages = {};
+      const newAverages = /* @__PURE__ */ Object.create(null);
       for (let i = 0; i < fields.length; i++) {
         const field = fields[i];
         newAverages[field] = stats.totalDocuments > 0 ? stats.totalFieldLengths[field] / stats.totalDocuments : 0;
@@ -1097,18 +1583,42 @@ function createPartitionStats() {
       stats.averageFieldLengths = newAverages;
     },
     serialize() {
+      const tfl = /* @__PURE__ */ Object.create(null);
+      for (const key of Object.keys(stats.totalFieldLengths)) {
+        tfl[key] = stats.totalFieldLengths[key];
+      }
+      const afl = /* @__PURE__ */ Object.create(null);
+      for (const key of Object.keys(stats.averageFieldLengths)) {
+        afl[key] = stats.averageFieldLengths[key];
+      }
+      const df = /* @__PURE__ */ Object.create(null);
+      for (const key of Object.keys(stats.docFrequencies)) {
+        df[key] = stats.docFrequencies[key];
+      }
       return {
         totalDocuments: stats.totalDocuments,
-        totalFieldLengths: { ...stats.totalFieldLengths },
-        averageFieldLengths: { ...stats.averageFieldLengths },
-        docFrequencies: { ...stats.docFrequencies }
+        totalFieldLengths: tfl,
+        averageFieldLengths: afl,
+        docFrequencies: df
       };
     },
     deserialize(data) {
       stats.totalDocuments = data.totalDocuments;
-      stats.totalFieldLengths = { ...data.totalFieldLengths };
-      stats.averageFieldLengths = { ...data.averageFieldLengths };
-      stats.docFrequencies = { ...data.docFrequencies };
+      const tfl = /* @__PURE__ */ Object.create(null);
+      for (const key of Object.keys(data.totalFieldLengths)) {
+        tfl[key] = data.totalFieldLengths[key];
+      }
+      stats.totalFieldLengths = tfl;
+      const afl = /* @__PURE__ */ Object.create(null);
+      for (const key of Object.keys(data.averageFieldLengths)) {
+        afl[key] = data.averageFieldLengths[key];
+      }
+      stats.averageFieldLengths = afl;
+      const df = /* @__PURE__ */ Object.create(null);
+      for (const key of Object.keys(data.docFrequencies)) {
+        df[key] = data.docFrequencies[key];
+      }
+      stats.docFrequencies = df;
     }
   };
   return stats;
@@ -1259,34 +1769,6 @@ function createGeoIndex() {
   };
 }
 
-// src/vector/similarity.ts
-function magnitude(v) {
-  let sum = 0;
-  for (let i = 0; i < v.length; i++) {
-    sum += v[i] * v[i];
-  }
-  return Math.sqrt(sum);
-}
-function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
-  if (magA === 0 || magB === 0) return 0;
-  return dotProduct(a, b) / (magA * magB);
-}
-function dotProduct(a, b) {
-  let sum = 0;
-  for (let i = 0; i < a.length; i++) {
-    sum += a[i] * b[i];
-  }
-  return sum;
-}
-function euclideanDistance(a, b) {
-  let sum = 0;
-  for (let i = 0; i < a.length; i++) {
-    const diff = a[i] - b[i];
-    sum += diff * diff;
-  }
-  return Math.sqrt(sum);
-}
-
 // src/vector/brute-force.ts
 function createBruteForceVectorStore(dimension) {
   const vectors = /* @__PURE__ */ new Map();
@@ -1355,6 +1837,120 @@ function createBruteForceVectorStore(dimension) {
   };
 }
 
+// src/search/vector-search.ts
+function createVectorSearchEngine(dimension, defaultHNSWConfig) {
+  const bruteForce = createBruteForceVectorStore(dimension);
+  const hnswConfig = defaultHNSWConfig;
+  let hnsw = null;
+  return {
+    get dimension() {
+      return dimension;
+    },
+    get size() {
+      return bruteForce.size;
+    },
+    get isPromoted() {
+      return hnsw !== null;
+    },
+    insert(docId, vector) {
+      bruteForce.insert(docId, vector);
+      if (hnsw) {
+        hnsw.insert(docId, vector);
+      }
+    },
+    remove(docId) {
+      bruteForce.remove(docId);
+      if (hnsw) {
+        hnsw.remove(docId);
+      }
+    },
+    has(docId) {
+      return bruteForce.has(docId);
+    },
+    search(query, k, metric, minSimilarity, filterDocIds, efSearch) {
+      if (hnsw) {
+        return hnsw.search(query, k, metric, minSimilarity, filterDocIds, efSearch);
+      }
+      return bruteForce.search(query, k, metric, minSimilarity, filterDocIds);
+    },
+    clear() {
+      bruteForce.clear();
+      if (hnsw) {
+        hnsw.clear();
+        hnsw = null;
+      }
+    },
+    entries() {
+      return bruteForce.entries();
+    },
+    promoteToHNSW(overrideConfig) {
+      const cfg = overrideConfig ?? hnswConfig;
+      const newHnsw = createHNSWIndex(dimension, cfg);
+      for (const [, entry] of bruteForce.entries()) {
+        newHnsw.insert(entry.docId, entry.vector);
+      }
+      hnsw = newHnsw;
+    },
+    demoteToLinear() {
+      if (hnsw) {
+        hnsw.clear();
+        hnsw = null;
+      }
+    },
+    getHNSWIndex() {
+      return hnsw;
+    },
+    getBruteForceStore() {
+      return bruteForce;
+    },
+    estimateMemoryBytes() {
+      const count = bruteForce.size;
+      if (count === 0) return 0;
+      const MAP_OVERHEAD = 64;
+      const MAP_ENTRY = 72;
+      const VECTOR_ENTRY_OBJ = 32;
+      const AVG_DOCID_BYTES = 56;
+      const TYPED_ARRAY_HEADER = 64;
+      const MAGNITUDE_BYTES = 8;
+      const perBruteForceEntry = MAP_ENTRY + VECTOR_ENTRY_OBJ + AVG_DOCID_BYTES + TYPED_ARRAY_HEADER + MAGNITUDE_BYTES;
+      let bytes = MAP_OVERHEAD + count * (perBruteForceEntry + dimension * 4);
+      if (hnsw) {
+        const HNSW_NODE_OBJ = 64;
+        const SHARED_REFS = 16;
+        const CONN_ARRAY_HEADER = 32;
+        const SET_OVERHEAD = 64;
+        const SET_ENTRY_COST = 72;
+        const m = hnsw.m;
+        const avgLayers = m / (m - 1);
+        const avgConnsLayer0 = m;
+        const avgConnsUpper = Math.ceil(m / 2);
+        const connMemPerNode = CONN_ARRAY_HEADER + (SET_OVERHEAD + avgConnsLayer0 * SET_ENTRY_COST) + Math.max(0, avgLayers - 1) * (SET_OVERHEAD + avgConnsUpper * SET_ENTRY_COST);
+        const perHnswNode = MAP_ENTRY + HNSW_NODE_OBJ + SHARED_REFS + connMemPerNode;
+        bytes += MAP_OVERHEAD + count * perHnswNode;
+      }
+      return Math.round(bytes);
+    },
+    serializeHNSW() {
+      if (!hnsw) return null;
+      return hnsw.serialize();
+    },
+    deserializeHNSW(data) {
+      const cfg = {};
+      const restoredHnsw = createHNSWIndex(dimension, {
+        m: data.m ?? cfg.m,
+        efConstruction: data.efConstruction ?? cfg.efConstruction,
+        metric: data.metric ?? cfg.metric
+      });
+      const vectorMap = /* @__PURE__ */ new Map();
+      for (const [, entry] of bruteForce.entries()) {
+        vectorMap.set(entry.docId, { vector: entry.vector, mag: entry.magnitude });
+      }
+      restoredHnsw.deserialize(data, vectorMap);
+      hnsw = restoredHnsw;
+    }
+  };
+}
+
 // src/core/field-index.ts
 function lowerBound(entries, value) {
   let lo = 0;
@@ -1385,12 +1981,20 @@ function collectDocIds(entries, from, to) {
 }
 function createNumericIndex() {
   let entries = [];
+  let sorted = true;
+  function ensureSorted() {
+    if (!sorted) {
+      entries.sort((a, b) => a.value - b.value);
+      sorted = true;
+    }
+  }
   return {
     insert(docId, value) {
-      const pos = upperBound(entries, value);
-      entries.splice(pos, 0, { value, docId });
+      entries.push({ value, docId });
+      sorted = false;
     },
     remove(docId, value) {
+      ensureSorted();
       const start = lowerBound(entries, value);
       const end = upperBound(entries, value);
       for (let i = start; i < end; i++) {
@@ -1401,9 +2005,11 @@ function createNumericIndex() {
       }
     },
     queryEq(value) {
+      ensureSorted();
       return collectDocIds(entries, lowerBound(entries, value), upperBound(entries, value));
     },
     queryNe(value) {
+      ensureSorted();
       const excluded = this.queryEq(value);
       const result = /* @__PURE__ */ new Set();
       for (const entry of entries) {
@@ -1412,18 +2018,23 @@ function createNumericIndex() {
       return result;
     },
     queryGt(value) {
+      ensureSorted();
       return collectDocIds(entries, upperBound(entries, value), entries.length);
     },
     queryGte(value) {
+      ensureSorted();
       return collectDocIds(entries, lowerBound(entries, value), entries.length);
     },
     queryLt(value) {
+      ensureSorted();
       return collectDocIds(entries, 0, lowerBound(entries, value));
     },
     queryLte(value) {
+      ensureSorted();
       return collectDocIds(entries, 0, upperBound(entries, value));
     },
     queryBetween(min, max) {
+      ensureSorted();
       return collectDocIds(entries, lowerBound(entries, min), upperBound(entries, max));
     },
     getAllDocIds() {
@@ -1434,12 +2045,15 @@ function createNumericIndex() {
     },
     clear() {
       entries = [];
+      sorted = true;
     },
     serialize() {
+      ensureSorted();
       return entries.slice();
     },
     deserialize(data) {
       entries = data.slice();
+      sorted = true;
     }
   };
 }
@@ -1567,6 +2181,14 @@ function createEnumIndex() {
 }
 
 // src/core/partition/utils.ts
+function getOrCreateFieldNameIndex(table, fieldName) {
+  const existing = table.indexMap.get(fieldName);
+  if (existing !== void 0) return existing;
+  const idx = table.names.length;
+  table.names.push(fieldName);
+  table.indexMap.set(fieldName, idx);
+  return idx;
+}
 var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
 function getNestedValue(obj, path) {
   const parts = path.split(".");
@@ -1643,7 +2265,7 @@ function ensureFieldIndex(state, fieldPath, fieldType) {
   } else {
     const dim = parseVectorDimension(fieldType);
     if (dim !== null && !state.vectorStores.has(fieldPath)) {
-      state.vectorStores.set(fieldPath, createBruteForceVectorStore(dim));
+      state.vectorStores.set(fieldPath, createVectorSearchEngine(dim));
     }
   }
 }
@@ -1667,53 +2289,71 @@ function indexStringField(state, docId, fieldPath, text, language, options, fiel
   const result = tokenize(text, language, tokenizeOptions(options));
   fieldLengths[fieldPath] = result.tokens.length;
   const fieldTokenList = [];
-  const tokenFreqs = /* @__PURE__ */ new Map();
-  for (const t of result.tokens) {
-    const existing = tokenFreqs.get(t.token);
-    if (existing) {
-      existing.count++;
-      existing.positions.push(t.position);
-    } else {
-      tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
-    }
-    fieldTokenList.push(t.token);
-  }
-  for (const [token, freq] of tokenFreqs) {
-    state.invertedIdx.insert(token, {
-      docId,
-      termFrequency: freq.count,
-      fieldName: fieldPath,
-      positions: freq.positions
-    });
-  }
-  tokensByField[fieldPath] = fieldTokenList;
-}
-function indexStringArrayField(state, docId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
-  const tokenFreqs = /* @__PURE__ */ new Map();
-  const fieldTokenList = [];
-  let positionOffset = 0;
-  for (const item of arr) {
-    const result = tokenize(item, language, tokenizeOptions(options));
+  const fieldNameIndex = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
+  if (state.trackPositions) {
+    const tokenFreqs = /* @__PURE__ */ new Map();
     for (const t of result.tokens) {
       const existing = tokenFreqs.get(t.token);
       if (existing) {
         existing.count++;
-        existing.positions.push(positionOffset + t.position);
+        existing.positions.push(t.position);
       } else {
-        tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
+        tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
       }
       fieldTokenList.push(t.token);
     }
-    positionOffset += result.tokens.length;
+    for (const [token, freq] of tokenFreqs) {
+      state.invertedIdx.insert(token, docId, freq.count, fieldNameIndex, freq.positions);
+    }
+  } else {
+    const tokenCounts = /* @__PURE__ */ new Map();
+    for (const t of result.tokens) {
+      tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
+      fieldTokenList.push(t.token);
+    }
+    for (const [token, count] of tokenCounts) {
+      state.invertedIdx.insert(token, docId, count, fieldNameIndex, null);
+    }
   }
-  fieldLengths[fieldPath] = fieldTokenList.length;
-  for (const [token, freq] of tokenFreqs) {
-    state.invertedIdx.insert(token, {
-      docId,
-      termFrequency: freq.count,
-      fieldName: fieldPath,
-      positions: freq.positions
-    });
+  tokensByField[fieldPath] = fieldTokenList;
+}
+function indexStringArrayField(state, docId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
+  const fieldTokenList = [];
+  const fieldNameIndex = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
+  if (state.trackPositions) {
+    const tokenFreqs = /* @__PURE__ */ new Map();
+    let positionOffset = 0;
+    for (const item of arr) {
+      const result = tokenize(item, language, tokenizeOptions(options));
+      for (const t of result.tokens) {
+        const existing = tokenFreqs.get(t.token);
+        if (existing) {
+          existing.count++;
+          existing.positions.push(positionOffset + t.position);
+        } else {
+          tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
+        }
+        fieldTokenList.push(t.token);
+      }
+      positionOffset += result.tokens.length;
+    }
+    fieldLengths[fieldPath] = fieldTokenList.length;
+    for (const [token, freq] of tokenFreqs) {
+      state.invertedIdx.insert(token, docId, freq.count, fieldNameIndex, freq.positions);
+    }
+  } else {
+    const tokenCounts = /* @__PURE__ */ new Map();
+    for (const item of arr) {
+      const result = tokenize(item, language, tokenizeOptions(options));
+      for (const t of result.tokens) {
+        tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
+        fieldTokenList.push(t.token);
+      }
+    }
+    fieldLengths[fieldPath] = fieldTokenList.length;
+    for (const [token, count] of tokenCounts) {
+      state.invertedIdx.insert(token, docId, count, fieldNameIndex, null);
+    }
   }
   tokensByField[fieldPath] = fieldTokenList;
 }
@@ -2332,66 +2972,188 @@ function computeBM25WithGlobalStats(termFrequency, globalDocFrequency, globalTot
 }
 
 // src/core/partition/search.ts
+var DEFAULT_MAX_RESULTS = 1e3;
 function searchFulltext(state, params) {
-  const { queryTokens, fields, boost, tolerance = 0, prefixLength = 2, exact = false, bm25Params, globalStats } = params;
+  const {
+    queryTokens,
+    fields,
+    boost,
+    tolerance = 0,
+    prefixLength = 2,
+    exact = false,
+    bm25Params,
+    globalStats,
+    maxResults,
+    termMatch
+  } = params;
   if (queryTokens.length === 0) {
     return { scored: [], totalMatched: 0 };
   }
-  const docScores = /* @__PURE__ */ new Map();
   const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
   const avgFieldLengths = globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths;
   const globalDocFreqs = globalStats?.docFrequencies ?? state.stats.docFrequencies;
-  for (const qt of queryTokens) {
-    const matchingPostings = exact ? (() => {
-      const postingList = state.invertedIdx.lookup(qt.token);
-      return postingList ? [{ token: qt.token, postingList }] : [];
-    })() : state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength);
-    for (const match of matchingPostings) {
-      const docFreq = globalStats ? globalDocFreqs[match.token] ?? match.postingList.docFrequency : match.postingList.docFrequency;
-      const idf = computeIDF(docFreq, totalDocs);
-      for (const posting of match.postingList.postings) {
-        if (fields && !fields.includes(posting.fieldName)) continue;
-        const fieldBoost = boost?.[posting.fieldName] ?? 1;
-        const avgLen = avgFieldLengths[posting.fieldName] ?? 1;
-        const storedDoc = state.docStore.get(posting.docId);
-        const actualFieldLength = storedDoc?.fieldLengths[posting.fieldName] ?? avgLen;
-        const scoreFn = globalStats ? computeBM25WithGlobalStats : computeBM25;
-        let termScore = scoreFn(posting.termFrequency, docFreq, totalDocs, actualFieldLength, avgLen, bm25Params);
-        termScore *= fieldBoost;
-        const existing = docScores.get(posting.docId);
-        if (existing) {
-          existing.score += termScore;
-          existing.termFrequencies[`${posting.fieldName}:${match.token}`] = posting.termFrequency;
-          existing.fieldLengths[posting.fieldName] = actualFieldLength;
-          existing.idf[match.token] = idf;
-          existing.matchedTermCount++;
-        } else {
-          docScores.set(posting.docId, {
-            score: termScore,
-            termFrequencies: { [`${posting.fieldName}:${match.token}`]: posting.termFrequency },
-            fieldLengths: { [posting.fieldName]: actualFieldLength },
-            idf: { [match.token]: idf },
-            matchedTermCount: 1
-          });
+  const scoreFn = globalStats ? computeBM25WithGlobalStats : computeBM25;
+  const docScores = /* @__PURE__ */ new Map();
+  const fieldLengthCache = /* @__PURE__ */ new Map();
+  const useIntersection = termMatch === "all" && queryTokens.length > 1;
+  const fieldNames = state.fieldNameTable.names;
+  if (useIntersection) {
+    const resolved = [];
+    for (const qt of queryTokens) {
+      const rawMatches = exact ? (() => {
+        const postingList = state.invertedIdx.lookup(qt.token);
+        return postingList ? [{ token: qt.token, postingList }] : [];
+      })() : state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength);
+      let totalPostings = 0;
+      const matches = [];
+      for (const m of rawMatches) {
+        const docFreq = globalStats ? globalDocFreqs[m.token] ?? m.postingList.docIdSet.size : m.postingList.docIdSet.size;
+        const idf = computeIDF(docFreq, totalDocs);
+        totalPostings += m.postingList.length;
+        matches.push({ token: m.token, docFreq, idf, postingList: m.postingList });
+      }
+      resolved.push({ token: qt.token, matches, totalPostings });
+    }
+    resolved.sort((a, b) => a.totalPostings - b.totalPostings);
+    for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
+      for (const match of resolved[tokenIndex].matches) {
+        const list = match.postingList;
+        const hasDeleted = list.deletedDocs.size > 0;
+        for (let pi = 0; pi < list.length; pi++) {
+          const docId = list.docIds[pi];
+          if (hasDeleted && list.deletedDocs.has(docId)) continue;
+          if (tokenIndex > 0 && !docScores.has(docId)) continue;
+          const fieldName = fieldNames[list.fieldNameIndices[pi]];
+          if (fields && !fields.includes(fieldName)) continue;
+          const termFrequency = list.termFrequencies[pi];
+          const fieldBoost = boost?.[fieldName] ?? 1;
+          const avgLen = avgFieldLengths[fieldName] ?? 1;
+          let cachedLengths = fieldLengthCache.get(docId);
+          if (cachedLengths === void 0) {
+            const storedDoc = state.docStore.get(docId);
+            cachedLengths = storedDoc?.fieldLengths ?? null;
+            fieldLengthCache.set(docId, cachedLengths);
+          }
+          const actualFieldLength = cachedLengths?.[fieldName] ?? avgLen;
+          let termScore = scoreFn(termFrequency, match.docFreq, totalDocs, actualFieldLength, avgLen, bm25Params);
+          termScore *= fieldBoost;
+          const existing = docScores.get(docId);
+          if (existing) {
+            existing.score += termScore;
+            existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
+            existing.fieldLengths[fieldName] = actualFieldLength;
+            existing.idf[match.token] = match.idf;
+          } else {
+            docScores.set(docId, {
+              score: termScore,
+              termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
+              fieldLengths: { [fieldName]: actualFieldLength },
+              idf: { [match.token]: match.idf }
+            });
+          }
+        }
+      }
+    }
+  } else {
+    for (const qt of queryTokens) {
+      const matchingPostings = exact ? (() => {
+        const postingList = state.invertedIdx.lookup(qt.token);
+        return postingList ? [{ token: qt.token, postingList }] : [];
+      })() : state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength);
+      for (const match of matchingPostings) {
+        const docFreq = globalStats ? globalDocFreqs[match.token] ?? match.postingList.docIdSet.size : match.postingList.docIdSet.size;
+        const idf = computeIDF(docFreq, totalDocs);
+        const list = match.postingList;
+        const hasDeleted = list.deletedDocs.size > 0;
+        for (let pi = 0; pi < list.length; pi++) {
+          const docId = list.docIds[pi];
+          if (hasDeleted && list.deletedDocs.has(docId)) continue;
+          const fieldName = fieldNames[list.fieldNameIndices[pi]];
+          if (fields && !fields.includes(fieldName)) continue;
+          const termFrequency = list.termFrequencies[pi];
+          const fieldBoost = boost?.[fieldName] ?? 1;
+          const avgLen = avgFieldLengths[fieldName] ?? 1;
+          let cachedLengths = fieldLengthCache.get(docId);
+          if (cachedLengths === void 0) {
+            const storedDoc = state.docStore.get(docId);
+            cachedLengths = storedDoc?.fieldLengths ?? null;
+            fieldLengthCache.set(docId, cachedLengths);
+          }
+          const actualFieldLength = cachedLengths?.[fieldName] ?? avgLen;
+          let termScore = scoreFn(termFrequency, docFreq, totalDocs, actualFieldLength, avgLen, bm25Params);
+          termScore *= fieldBoost;
+          const existing = docScores.get(docId);
+          if (existing) {
+            existing.score += termScore;
+            existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
+            existing.fieldLengths[fieldName] = actualFieldLength;
+            existing.idf[match.token] = idf;
+          } else {
+            docScores.set(docId, {
+              score: termScore,
+              termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
+              fieldLengths: { [fieldName]: actualFieldLength },
+              idf: { [match.token]: idf }
+            });
+          }
         }
       }
     }
   }
-  const scored = [];
+  const totalMatched = docScores.size;
+  const k = maxResults !== void 0 && maxResults > 0 ? maxResults : DEFAULT_MAX_RESULTS;
+  const scored = topKFromMap(docScores, Math.min(k, totalMatched));
+  return { scored, totalMatched };
+}
+function topKFromMap(docScores, k) {
+  if (k <= 0) return [];
+  const heap = [];
   for (const [docId, data] of docScores) {
-    scored.push({
-      docId,
+    if (heap.length < k) {
+      heap.push({ docId, score: data.score });
+      if (heap.length === k) buildMinHeap(heap);
+    } else if (data.score > heap[0].score) {
+      heap[0] = { docId, score: data.score };
+      siftDown(heap, 0);
+    }
+  }
+  heap.sort((a, b) => b.score - a.score);
+  const result = new Array(heap.length);
+  for (let i = 0; i < heap.length; i++) {
+    const data = docScores.get(heap[i].docId);
+    if (!data) continue;
+    result[i] = {
+      docId: heap[i].docId,
       score: data.score,
       termFrequencies: data.termFrequencies,
       fieldLengths: data.fieldLengths,
       idf: data.idf
-    });
+    };
+  }
+  return result;
+}
+function buildMinHeap(heap) {
+  for (let i = (heap.length >> 1) - 1; i >= 0; i--) {
+    siftDown(heap, i);
+  }
+}
+function siftDown(heap, idx) {
+  const len = heap.length;
+  while (true) {
+    let smallest = idx;
+    const left = 2 * idx + 1;
+    const right = 2 * idx + 2;
+    if (left < len && heap[left].score < heap[smallest].score) smallest = left;
+    if (right < len && heap[right].score < heap[smallest].score) smallest = right;
+    if (smallest === idx) break;
+    const tmp = heap[idx];
+    heap[idx] = heap[smallest];
+    heap[smallest] = tmp;
+    idx = smallest;
   }
-  scored.sort((a, b) => b.score - a.score);
-  return { scored, totalMatched: scored.length };
 }
 function searchVector(state, params) {
-  const { field, value, k, similarity = 0, metric = "cosine", filterDocIds } = params;
+  const { field, value, k, similarity = 0, metric = "cosine", filterDocIds, efSearch } = params;
   const vecStore = state.vectorStores.get(field);
   if (!vecStore) {
     return { scored: [], totalMatched: 0 };
@@ -2404,7 +3166,7 @@ function searchVector(state, params) {
     );
   }
   const queryVec = new Float32Array(value);
-  const results = vecStore.search(queryVec, k, metric, similarity, filterDocIds);
+  const results = vecStore.search(queryVec, k, metric, similarity, filterDocIds, efSearch);
   return { scored: results, totalMatched: results.length };
 }
 function buildFilterContext(state, schema) {
@@ -2587,7 +3349,7 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
     serializedVectors[path] = {
       dimension: store.dimension,
       vectors,
-      hnswGraph: null
+      hnswGraph: store.serializeHNSW()
     };
   }
   const serializedStats = state.stats.serialize();
@@ -2611,6 +3373,109 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
     statistics: serializedStats
   };
 }
+function serializePartitionToWirePayloadV2(state, partitionId, indexName, totalPartitions, language, schema) {
+  const flatSchema = getFlatSchema(state, schema);
+  const flatSchemaStrings = {};
+  for (const [k, v] of Object.entries(flatSchema)) {
+    flatSchemaStrings[k] = v;
+  }
+  const wireDocs = /* @__PURE__ */ Object.create(null);
+  for (const [docId, stored] of state.docStore.all()) {
+    wireDocs[docId] = {
+      fields: structuredClone(stored.fields),
+      field_lengths: { ...stored.fieldLengths }
+    };
+  }
+  const fieldNames = [...state.fieldNameTable.names];
+  const wireEntries = /* @__PURE__ */ Object.create(null);
+  for (const token of state.invertedIdx.tokens()) {
+    const list = state.invertedIdx.lookup(token);
+    if (!list) continue;
+    const hasDeleted = list.deletedDocs.size > 0;
+    const ids = [];
+    const tf = [];
+    const fi = [];
+    const pos = list.positions ? [] : null;
+    for (let i = 0; i < list.length; i++) {
+      if (hasDeleted && list.deletedDocs.has(list.docIds[i])) continue;
+      ids.push(list.docIds[i]);
+      tf.push(list.termFrequencies[i]);
+      fi.push(list.fieldNameIndices[i]);
+      if (pos && list.positions) {
+        pos.push(list.positions[i]);
+      }
+    }
+    if (ids.length > 0) {
+      wireEntries[token] = { df: list.docIdSet.size, ids, tf, fi, pos };
+    }
+  }
+  const wireNumeric = {};
+  for (const [path, idx] of state.numericIndexes) {
+    wireNumeric[path] = idx.serialize().map((e) => ({ value: e.value, doc_id: e.docId }));
+  }
+  const wireBoolean = {};
+  for (const [path, idx] of state.booleanIndexes) {
+    const s = idx.serialize();
+    wireBoolean[path] = { true_docs: s.trueDocs, false_docs: s.falseDocs };
+  }
+  const wireEnum = {};
+  for (const [path, idx] of state.enumIndexes) {
+    wireEnum[path] = idx.serialize();
+  }
+  const wireGeo = {};
+  for (const [path, idx] of state.geoIndexes) {
+    wireGeo[path] = idx.serialize().map((e) => ({ lat: e.lat, lon: e.lon, doc_id: e.docId }));
+  }
+  const wireVectors = {};
+  for (const [path, store] of state.vectorStores) {
+    const vectors = [];
+    for (const [, entry] of store.entries()) {
+      vectors.push({ doc_id: entry.docId, vector: Array.from(entry.vector) });
+    }
+    const hnswData = store.serializeHNSW();
+    wireVectors[path] = {
+      dimension: store.dimension,
+      vectors,
+      hnsw_graph: hnswData ? {
+        entry_point: hnswData.entryPoint,
+        max_layer: hnswData.maxLayer,
+        m: hnswData.m,
+        ef_construction: hnswData.efConstruction,
+        metric: hnswData.metric,
+        nodes: hnswData.nodes
+      } : null
+    };
+  }
+  const serializedStats = state.stats.serialize();
+  return {
+    v: 2,
+    index_name: indexName,
+    partition_id: partitionId,
+    total_partitions: totalPartitions,
+    language,
+    schema: flatSchemaStrings,
+    doc_count: state.docStore.count(),
+    avg_doc_length: Object.values(serializedStats.averageFieldLengths).reduce((sum, v) => sum + v, 0),
+    documents: wireDocs,
+    inverted_index: {
+      field_names: fieldNames,
+      entries: wireEntries
+    },
+    field_indexes: {
+      numeric: wireNumeric,
+      boolean: wireBoolean,
+      enum: wireEnum,
+      geopoint: wireGeo
+    },
+    vector_data: wireVectors,
+    statistics: {
+      total_documents: serializedStats.totalDocuments,
+      total_field_lengths: serializedStats.totalFieldLengths,
+      average_field_lengths: serializedStats.averageFieldLengths,
+      doc_frequencies: serializedStats.docFrequencies
+    }
+  };
+}
 function deserializePartition(state, data, clearFn, schema) {
   clearFn();
   const invertedData = /* @__PURE__ */ Object.create(null);
@@ -2621,7 +3486,7 @@ function deserializePartition(state, data, clearFn, schema) {
         docId: p.docId,
         termFrequency: p.termFrequency,
         fieldName: p.field,
-        positions: [...p.positions]
+        positions: state.trackPositions ? [...p.positions] : []
       }))
     };
   }
@@ -2652,10 +3517,13 @@ function deserializePartition(state, data, clearFn, schema) {
     state.geoIndexes.set(path, geoIdx);
   }
   for (const [path, vecData] of Object.entries(data.vectorData)) {
-    const store = createBruteForceVectorStore(vecData.dimension);
+    const store = createVectorSearchEngine(vecData.dimension);
     for (const entry of vecData.vectors) {
       store.insert(entry.docId, new Float32Array(entry.vector));
     }
+    if (vecData.hnswGraph) {
+      store.deserializeHNSW(vecData.hnswGraph);
+    }
     state.vectorStores.set(path, store);
   }
   state.stats.deserialize(data.statistics);
@@ -2676,9 +3544,15 @@ function deserializePartition(state, data, clearFn, schema) {
 }
 
 // src/core/partition/index.ts
-function createPartitionIndex(partitionId) {
+function createPartitionIndex(partitionId, trackPositions = true, vectorPromotionConfig) {
+  const vectorPromoter = vectorPromotionConfig ? createVectorPromoter({
+    promotionThreshold: vectorPromotionConfig.threshold,
+    hnswConfig: vectorPromotionConfig.hnswConfig,
+    workerStrategy: vectorPromotionConfig.workerStrategy
+  }) : null;
+  const fieldNameTable = { names: [], indexMap: /* @__PURE__ */ new Map() };
   const state = {
-    invertedIdx: createInvertedIndex(),
+    invertedIdx: createInvertedIndex(fieldNameTable),
     docStore: createDocumentStore(),
     stats: createPartitionStats(),
     numericIndexes: /* @__PURE__ */ new Map(),
@@ -2686,10 +3560,13 @@ function createPartitionIndex(partitionId) {
     enumIndexes: /* @__PURE__ */ new Map(),
     geoIndexes: /* @__PURE__ */ new Map(),
     vectorStores: /* @__PURE__ */ new Map(),
+    fieldNameTable,
     flatSchemaCache: null,
-    lastSchemaRef: null
+    lastSchemaRef: null,
+    trackPositions
   };
   function clearAll() {
+    vectorPromoter?.shutdown();
     state.invertedIdx.clear();
     state.docStore.clear();
     for (const idx of state.numericIndexes.values()) idx.clear();
@@ -2723,6 +3600,9 @@ function createPartitionIndex(partitionId) {
       }
       if (options?.validate !== false) {
         validateDocument(document, schema);
+        if (options?.strict) {
+          validateDocumentStrict(document, schema);
+        }
       }
       const flatSchema = getFlatSchema(state, schema);
       const { fieldLengths, tokensByField } = indexDocument(
@@ -2733,8 +3613,13 @@ function createPartitionIndex(partitionId) {
         language,
         options
       );
-      state.docStore.store(docId, document, fieldLengths);
+      if (options?.skipClone) {
+        state.docStore.storeRef(docId, document, fieldLengths);
+      } else {
+        state.docStore.store(docId, document, fieldLengths);
+      }
       state.stats.addDocument(fieldLengths, tokensByField);
+      vectorPromoter?.check(state.vectorStores);
     },
     remove(docId, schema, language, options) {
       const stored = state.docStore.get(docId);
@@ -2759,6 +3644,9 @@ function createPartitionIndex(partitionId) {
       }
       if (options?.validate !== false) {
         validateDocument(document, schema);
+        if (options?.strict) {
+          validateDocumentStrict(document, schema);
+        }
       }
       const flatSchema = getFlatSchema(state, schema);
       const needsTextReindex = textFieldsChanged(stored.fields, document, flatSchema);
@@ -2793,6 +3681,11 @@ function createPartitionIndex(partitionId) {
       if (!stored) return void 0;
       return structuredClone(stored.fields);
     },
+    getRef(docId) {
+      const stored = state.docStore.get(docId);
+      if (!stored) return void 0;
+      return stored.fields;
+    },
     has(docId) {
       return state.docStore.has(docId);
     },
@@ -2805,6 +3698,41 @@ function createPartitionIndex(partitionId) {
       }
     },
     clear: clearAll,
+    estimateMemoryBytes() {
+      const docCount = state.docStore.count();
+      if (docCount === 0) return 0;
+      const AVG_DOC_OVERHEAD = 350;
+      let bytes = docCount * AVG_DOC_OVERHEAD;
+      const docFreqs = state.stats.docFrequencies;
+      const POSTING_ENTRY_SIZE = 24;
+      const PER_TERM_OVERHEAD = 180;
+      let totalPostings = 0;
+      let termCount = 0;
+      for (const term in docFreqs) {
+        totalPostings += docFreqs[term];
+        termCount++;
+      }
+      bytes += totalPostings * POSTING_ENTRY_SIZE;
+      bytes += termCount * PER_TERM_OVERHEAD;
+      const FIELD_ENTRY_OVERHEAD = 42;
+      bytes += docCount * state.numericIndexes.size * FIELD_ENTRY_OVERHEAD;
+      bytes += docCount * state.booleanIndexes.size * FIELD_ENTRY_OVERHEAD;
+      bytes += docCount * state.enumIndexes.size * FIELD_ENTRY_OVERHEAD;
+      bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD;
+      for (const store of state.vectorStores.values()) {
+        bytes += store.estimateMemoryBytes();
+      }
+      return bytes;
+    },
+    vectorFieldCount() {
+      return state.vectorStores.size;
+    },
+    hasPromotedHnsw() {
+      for (const store of state.vectorStores.values()) {
+        if (store.isPromoted) return true;
+      }
+      return false;
+    },
     searchFulltext(params) {
       return searchFulltext(state, params);
     },
@@ -2817,9 +3745,16 @@ function createPartitionIndex(partitionId) {
     computeFacets(docIds, config, schema) {
       return computeFacets(state, docIds, config, schema);
     },
+    suggestTerms(prefix, limit) {
+      return state.invertedIdx.prefixSearch(prefix, limit);
+    },
     serialize(indexName, totalPartitions, language, schema) {
       return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
     },
+    serializeToBytes(indexName, totalPartitions, language, schema) {
+      const wire = serializePartitionToWirePayloadV2(state, partitionId, indexName, totalPartitions, language, schema);
+      return encodeRawPayloadV2(wire);
+    },
     deserialize(data, schema) {
       deserializePartition(state, data, clearAll, schema);
     }
@@ -2830,10 +3765,12 @@ function createPartitionIndex(partitionId) {
 // src/partitioning/manager.ts
 function createPartitionManager(indexName, config, language, router, initialPartitionCount) {
   const count = initialPartitionCount ?? 1;
+  const trackPositions = config.trackPositions ?? true;
+  const vectorPromotionConfig = config.vectorPromotion;
   let partitions = [];
   const docPartitionMap = /* @__PURE__ */ new Map();
   for (let i = 0; i < count; i++) {
-    partitions.push(createPartitionIndex(i));
+    partitions.push(createPartitionIndex(i, trackPositions, vectorPromotionConfig));
   }
   function validatePartitionId(partitionId) {
     if (partitionId < 0 || partitionId >= partitions.length) {
@@ -2882,6 +3819,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
       validatePartitionId(partitionId);
       return partitions[partitionId];
     },
+    partitionAt(index) {
+      return partitions[index];
+    },
     getAllPartitions() {
       return [...partitions];
     },
@@ -2890,7 +3830,15 @@ function createPartitionManager(indexName, config, language, router, initialPart
       rebuildDocPartitionMap();
     },
     addPartition() {
-      const partition = createPartitionIndex(partitions.length);
+      const currentMaxPartitions = config.partitions?.maxPartitions;
+      if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
+        throw new NarsilError(
+          ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
+          `Cannot add partition: maximum of ${currentMaxPartitions} partitions reached`,
+          { maxPartitions: currentMaxPartitions, currentCount: partitions.length }
+        );
+      }
+      const partition = createPartitionIndex(partitions.length, trackPositions, vectorPromotionConfig);
       partitions.push(partition);
       return partition;
     },
@@ -2900,8 +3848,26 @@ function createPartitionManager(indexName, config, language, router, initialPart
       rebuildDocPartitionMap();
     },
     insert(docId, document, options) {
+      const currentMaxDocs = config.partitions?.maxDocsPerPartition;
+      if (currentMaxDocs !== void 0) {
+        const totalCapacity = currentMaxDocs * partitions.length;
+        if (docPartitionMap.size >= totalCapacity) {
+          throw new NarsilError(
+            ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
+            `Index "${indexName}" has reached its capacity of ${totalCapacity} documents (${currentMaxDocs} per partition \xD7 ${partitions.length} partitions)`,
+            {
+              indexName,
+              currentCount: docPartitionMap.size,
+              totalCapacity,
+              maxDocsPerPartition: currentMaxDocs,
+              partitionCount: partitions.length
+            }
+          );
+        }
+      }
       const pid = router.route(docId, partitions.length);
-      partitions[pid].insert(docId, document, config.schema, language, options);
+      const insertOpts = config.strict ? { ...options, strict: true } : options;
+      partitions[pid].insert(docId, document, config.schema, language, insertOpts);
       docPartitionMap.set(docId, pid);
     },
     remove(docId) {
@@ -2917,13 +3883,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
       if (pid === void 0) {
         throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
       }
-      partitions[pid].update(docId, document, config.schema, language, options);
+      const updateOpts = config.strict ? { ...options, strict: true } : options;
+      partitions[pid].update(docId, document, config.schema, language, updateOpts);
     },
     get(docId) {
       const pid = docPartitionMap.get(docId);
       if (pid === void 0) return void 0;
       return partitions[pid].get(docId);
     },
+    getRef(docId) {
+      const pid = docPartitionMap.get(docId);
+      if (pid === void 0) return void 0;
+      return partitions[pid].getRef(docId);
+    },
     has(docId) {
       return docPartitionMap.has(docId);
     },
@@ -2934,6 +3906,10 @@ function createPartitionManager(indexName, config, language, router, initialPart
       validatePartitionId(partitionId);
       return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
     },
+    serializePartitionToBytes(partitionId) {
+      validatePartitionId(partitionId);
+      return partitions[partitionId].serializeToBytes(indexName, partitions.length, language.name, config.schema);
+    },
     deserializePartition(partitionId, data) {
       validatePartitionId(partitionId);
       partitions[partitionId].deserialize(data, config.schema);
@@ -2954,6 +3930,22 @@ function createPartitionManager(indexName, config, language, router, initialPart
         }
       }
       return { totalDocuments, docFrequencies, totalFieldLengths };
+    },
+    estimateMemoryBytes() {
+      let total = 0;
+      for (let i = 0; i < partitions.length; i++) {
+        total += partitions[i].estimateMemoryBytes();
+      }
+      return total;
+    },
+    getPartitionStats() {
+      return partitions.map((p, i) => ({
+        partitionId: i,
+        documentCount: p.count(),
+        estimatedMemoryBytes: p.estimateMemoryBytes(),
+        vectorFieldCount: p.vectorFieldCount(),
+        isHnswPromoted: p.hasPromotedHnsw()
+      }));
     }
   };
   return manager;
@@ -3050,7 +4042,7 @@ function createDirectExecutor() {
       }
       case "insert": {
         const entry = requireIndex(action.indexName);
-        entry.manager.insert(action.docId, action.document);
+        entry.manager.insert(action.docId, action.document, action.skipClone ? { skipClone: true } : void 0);
         return void 0;
       }
       case "remove": {
@@ -3065,14 +4057,15 @@ function createDirectExecutor() {
       }
       case "query": {
         const entry = requireIndex(action.indexName);
-        const result = fanOutQuery(entry.manager, action.params, entry.language, entry.config.schema, {
-          scoringMode: entry.config.defaultScoring ?? "local"
+        const result = await fanOutQuery(entry.manager, action.params, entry.language, entry.config.schema, {
+          scoringMode: entry.config.defaultScoring ?? "local",
+          partitionIds: action.partitionIds
         });
         return result;
       }
       case "preflight": {
         const entry = requireIndex(action.indexName);
-        const result = fanOutQuery(entry.manager, action.params, entry.language, entry.config.schema, {
+        const result = await fanOutQuery(entry.manager, action.params, entry.language, entry.config.schema, {
           scoringMode: entry.config.defaultScoring ?? "local"
         });
         return { count: result.totalMatched };
@@ -3144,4 +4137,4 @@ function createDirectExecutor() {
   };
 }
 
-export { createDirectExecutor, fanOutQuery, getLanguage, registerLanguage, tokenize, validateSchema };
+export { createDirectExecutor, createPartitionIndex, createPartitionRouter, deserializePayloadV2, detectRuntime, fanOutQuery, fnv1a, getLanguage, kWayMerge, registerLanguage, tokenize, validateSchema };