@delali/narsil 0.1.5 → 0.1.7

package/dist/{chunk-TFWBALU4.mjs → chunk-KHL4RPC3.mjs}

@@ -1,6 +1,6 @@
-import { createHNSWIndex, magnitude, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-GVN2W4VA.mjs';
-import { english } from './chunk-WYC635GI.mjs';
-import { NarsilError, ErrorCodes } from './chunk-7GKPWGCN.mjs';
+import { createVectorStore, createHNSWIndex, magnitude, createBoundedMaxHeap, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-VVORJOXR.mjs';
+import { english } from './chunk-I24SGLFS.mjs';
+import { NarsilError, ErrorCodes, createNarsilError } from './chunk-R2C2EQGO.mjs';
 import { decode, encode } from '@msgpack/msgpack';
 
 // src/languages/registry.ts
@@ -43,6 +43,67 @@ function mergeFacets(partitionFacets) {
   return result;
 }
 
+// src/core/bitset.ts
+function createBitSet(capacity) {
+  return new Uint32Array(capacity + 31 >>> 5 || 1);
+}
+function bitsetSet(bs, index) {
+  const word = index >>> 5;
+  if (word < bs.length) {
+    bs[word] |= 1 << (index & 31);
+  }
+}
+function bitsetHas(bs, index) {
+  const word = index >>> 5;
+  if (word >= bs.length) return false;
+  return (bs[word] & 1 << (index & 31)) !== 0;
+}
+function bitsetAnd(a, b) {
+  const len = Math.min(a.length, b.length);
+  const result = new Uint32Array(len);
+  for (let i = 0; i < len; i++) {
+    result[i] = a[i] & b[i];
+  }
+  return result;
+}
+function bitsetOr(a, b) {
+  const longer = a.length >= b.length ? a : b;
+  const shorter = a.length >= b.length ? b : a;
+  const result = new Uint32Array(longer.length);
+  for (let i = 0; i < shorter.length; i++) {
+    result[i] = longer[i] | shorter[i];
+  }
+  for (let i = shorter.length; i < longer.length; i++) {
+    result[i] = longer[i];
+  }
+  return result;
+}
+function bitsetNot(bs, capacity) {
+  const result = new Uint32Array(bs.length);
+  const fullWords = capacity >>> 5;
+  for (let i = 0; i < fullWords; i++) {
+    result[i] = ~bs[i];
+  }
+  const remainder = capacity & 31;
+  if (remainder > 0 && fullWords < bs.length) {
+    result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
+  }
+  return result;
+}
+function bitsetIsEmpty(bs) {
+  for (let i = 0; i < bs.length; i++) {
+    if (bs[i] !== 0) return false;
+  }
+  return true;
+}
+function bitsetFromSet(set, capacity) {
+  const bs = createBitSet(capacity);
+  for (const id of set) {
+    bs[id >>> 5] |= 1 << (id & 31);
+  }
+  return bs;
+}
+
 // src/core/fuzzy.ts
 function boundedLevenshtein(a, b, tolerance) {
   if (tolerance < 0) return { distance: -1, withinTolerance: false };
@@ -95,8 +156,67 @@ function boundedLevenshtein(a, b, tolerance) {
 // src/core/tokenizer.ts
 var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{N}_'-]+/u;
 var DEFAULT_MIN_TOKEN_LENGTH = 1;
+var CACHE_SIZE_FLOOR = 5e4;
+var CACHE_SIZE_CEILING = 2e6;
+var BYTES_PER_ENTRY = 200;
+function computeDefaultCacheSize() {
+  try {
+    if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
+      const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
+      if (constrainedMemory > 0) {
+        const budget = Math.floor(constrainedMemory * 0.05 / BYTES_PER_ENTRY);
+        return Math.max(CACHE_SIZE_FLOOR, Math.min(budget, CACHE_SIZE_CEILING));
+      }
+      return Math.max(CACHE_SIZE_FLOOR, Math.min(1e6, CACHE_SIZE_CEILING));
+    }
+    if (typeof navigator !== "undefined") {
+      const mem = navigator.deviceMemory;
+      if (typeof mem === "number" && mem > 0) {
+        let entries;
+        if (mem <= 1) entries = 1e5;
+        else if (mem <= 4) entries = 25e4;
+        else entries = 5e5;
+        return Math.max(CACHE_SIZE_FLOOR, Math.min(entries, CACHE_SIZE_CEILING));
+      }
+      return Math.max(CACHE_SIZE_FLOOR, Math.min(2e5, CACHE_SIZE_CEILING));
+    }
+  } catch {
+  }
+  return Math.max(CACHE_SIZE_FLOOR, Math.min(2e5, CACHE_SIZE_CEILING));
+}
 var normalizationCache = /* @__PURE__ */ new Map();
-var MAX_CACHE_SIZE = 65536;
+var maxCacheSize = computeDefaultCacheSize();
+function configureNormalizationCache(maxSize) {
+  if (!Number.isFinite(maxSize)) {
+    throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must be a finite number", {
+      received: maxSize
+    });
+  }
+  if (maxSize < 0) {
+    throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must not be negative", {
+      received: maxSize
+    });
+  }
+  if (maxSize === 0) {
+    throw createNarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      "tokenizerCacheSize must be greater than zero; the normalization cache cannot be disabled",
+      {
+        received: maxSize
+      }
+    );
+  }
+  maxCacheSize = Math.max(CACHE_SIZE_FLOOR, Math.min(Math.floor(maxSize), CACHE_SIZE_CEILING));
+  if (normalizationCache.size > maxCacheSize) {
+    const excess = normalizationCache.size - maxCacheSize;
+    let count = 0;
+    for (const key of normalizationCache.keys()) {
+      if (count >= excess) break;
+      normalizationCache.delete(key);
+      count++;
+    }
+  }
+}
 var cachedLangName = "";
 var cachedFlags = "";
 var cachedPrefix = "";
@@ -128,9 +248,14 @@ function transformToken(raw, language, stem, removeDiacritics) {
   if (stem && language.stemmer) {
     normalized = language.stemmer(normalized);
   }
-  if (normalizationCache.size >= MAX_CACHE_SIZE) {
-    const firstKey = normalizationCache.keys().next().value;
-    normalizationCache.delete(firstKey);
+  if (normalizationCache.size >= maxCacheSize) {
+    const evictCount = Math.max(1, maxCacheSize >>> 2);
+    let count = 0;
+    for (const key of normalizationCache.keys()) {
+      if (count >= evictCount) break;
+      normalizationCache.delete(key);
+      count++;
+    }
   }
   normalizationCache.set(cacheKey, normalized);
   return normalized;
@@ -186,6 +311,53 @@ function tokenize(text, language, options) {
   }
   return { tokens, originalTokens };
 }
+function* tokenizeIterator(text, language, options) {
+  const {
+    stem = true,
+    removeStopWords = true,
+    removeDiacritics = false,
+    customTokenizer,
+    stopWordOverride
+  } = options ?? {};
+  if (customTokenizer) {
+    const customResult = customTokenizer.tokenize(text);
+    for (const entry of customResult) {
+      yield entry;
+    }
+    return;
+  }
+  const normalized = isAscii(text) ? text.toLowerCase() : text.normalize("NFC").toLowerCase();
+  const rawParts = splitText(normalized, language);
+  const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
+  const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
+  const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
+  let position = 0;
+  for (const part of rawParts) {
+    if (part.length < minLength) {
+      position++;
+      continue;
+    }
+    if (stopWords.has(part)) {
+      position++;
+      continue;
+    }
+    const processed = transformToken(part, language, stem, effectiveDiacritics);
+    if (processed.length > 0) {
+      yield { token: processed, position };
+    }
+    position++;
+  }
+}
+function clearNormalizationCache() {
+  normalizationCache.clear();
+}
+function resetNormalizationCache() {
+  normalizationCache.clear();
+  maxCacheSize = computeDefaultCacheSize();
+}
+function getNormalizationCacheSize() {
+  return normalizationCache.size;
+}
 
 // src/schema/validator.ts
 var SCALAR_FIELD_TYPES = /* @__PURE__ */ new Set([
@@ -201,6 +373,7 @@ var SCALAR_FIELD_TYPES = /* @__PURE__ */ new Set([
 ]);
 var VECTOR_PATTERN = /^vector\[(\d+)]$/;
 var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
+var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
 var MAX_NESTING_DEPTH = 4;
 function isPlainObject(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -215,6 +388,12 @@ function validateSchemaFields(schema, depth, prefix) {
   }
   for (const [field, type] of Object.entries(schema)) {
     const path = prefix ? `${prefix}.${field}` : field;
+    if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
+      throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
+        field,
+        path
+      });
+    }
     if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
       throw new NarsilError(
         ErrorCodes.SCHEMA_INVALID_TYPE,
@@ -481,6 +660,27 @@ function validateDocumentStrict(document, schema) {
     );
   }
 }
+function resolveNestedValue(document, path) {
+  if (!path.includes(".")) return document[path];
+  const segments = path.split(".");
+  let current = document;
+  for (const segment of segments) {
+    if (current === null || current === void 0 || typeof current !== "object") return void 0;
+    current = current[segment];
+  }
+  return current;
+}
+function validateRequiredFields(document, required) {
+  if (required.length === 0) return;
+  for (const field of required) {
+    const value = resolveNestedValue(document, field);
+    if (value === void 0 || value === null) {
+      throw new NarsilError(ErrorCodes.DOC_MISSING_REQUIRED_FIELD, `Document is missing required field "${field}"`, {
+        field
+      });
+    }
+  }
+}
 function flattenRecursive(schema, prefix, result) {
   for (const [field, type] of Object.entries(schema)) {
     const path = prefix ? `${prefix}.${field}` : field;
@@ -496,6 +696,17 @@ function flattenSchema(schema) {
   flattenRecursive(schema, "", result);
   return result;
 }
+function extractVectorFieldsFromSchema(schema) {
+  const flat = flattenSchema(schema);
+  const result = /* @__PURE__ */ new Map();
+  for (const [fieldPath, fieldType] of Object.entries(flat)) {
+    const match = VECTOR_PATTERN.exec(fieldType);
+    if (match) {
+      result.set(fieldPath, Number.parseInt(match[1], 10));
+    }
+  }
+  return result;
+}
 
 // src/search/fulltext.ts
 function fulltextSearch(partition, params, language, schema, options) {
@@ -519,9 +730,16 @@ function fulltextSearch(partition, params, language, schema, options) {
     return { scored: [], totalMatched: 0 };
   }
   const queryTokens = deduplicateTokens(queryTokenResult.tokens);
-  const hasPostFilters = params.filters !== void 0 || params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any";
+  let filterBitset;
+  if (params.filters) {
+    filterBitset = partition.applyFiltersBitset(params.filters, schema);
+    if (bitsetIsEmpty(filterBitset)) {
+      return { scored: [], totalMatched: 0 };
+    }
+  }
+  const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
   const requestedLimit = params.limit !== void 0 || params.offset !== void 0 ? (params.limit ?? 10) + (params.offset ?? 0) + 1 : void 0;
-  const maxResults = requestedLimit !== void 0 && !hasPostFilters ? requestedLimit : void 0;
+  const maxResults = requestedLimit !== void 0 && !needsAllResults ? requestedLimit : void 0;
   const rawResult = partition.searchFulltext({
     queryTokens,
     fields: params.fields,
@@ -532,7 +750,8 @@ function fulltextSearch(partition, params, language, schema, options) {
     bm25Params: options?.bm25Params,
     globalStats: options?.globalStats,
     maxResults,
-    termMatch: params.termMatch
+    termMatch: params.termMatch,
+    filterBitset
   });
   let scored = rawResult.scored;
   const termMatch = params.termMatch ?? "any";
@@ -543,11 +762,7 @@ function fulltextSearch(partition, params, language, schema, options) {
     const threshold = params.minScore;
     scored = scored.filter((doc) => doc.score >= threshold);
   }
-  if (params.filters) {
-    const matchingDocIds = partition.applyFilters(params.filters, schema);
-    scored = scored.filter((doc) => matchingDocIds.has(doc.docId));
-  }
-  const totalMatched = hasPostFilters ? scored.length : rawResult.totalMatched;
+  const totalMatched = needsAllResults ? scored.length : rawResult.totalMatched;
   return { scored, totalMatched };
 }
 function validateSearchFields(fields, flatSchema) {
@@ -608,109 +823,6 @@ function queryTermSatisfied(queryToken, indexTokens, tolerance, exact) {
   return false;
 }
 
-// src/search/hybrid.ts
-function hybridSearch(partition, params, language, schema, options) {
-  const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
-  const hasVector = params.vector !== void 0;
-  if (!hasTerm && !hasVector) {
-    return { scored: [], totalMatched: 0 };
-  }
-  if (hasTerm && !hasVector) {
-    return fulltextSearch(partition, params, language, schema, options);
-  }
-  if (!hasTerm && hasVector) {
-    return runVectorOnly(partition, params);
-  }
-  const textResult = fulltextSearch(partition, params, language, schema, options);
-  const vectorConfig = params.vector;
-  const vectorResult = partition.searchVector({
-    field: vectorConfig.field,
-    value: vectorConfig.value,
-    k: params.limit ?? 10,
-    similarity: vectorConfig.similarity,
-    metric: vectorConfig.metric,
-    efSearch: vectorConfig.efSearch
-  });
-  const normalizedText = normalizeScores(textResult.scored);
-  const normalizedVector = normalizeScores(vectorResult.scored);
-  const textDocMap = /* @__PURE__ */ new Map();
-  for (const doc of textResult.scored) {
-    textDocMap.set(doc.docId, doc);
-  }
-  const mergedMap = /* @__PURE__ */ new Map();
-  for (const doc of normalizedText) {
-    mergedMap.set(doc.docId, {
-      textScore: doc.score,
-      vectorScore: 0,
-      textDoc: textDocMap.get(doc.docId) ?? null
-    });
-  }
-  for (const doc of normalizedVector) {
-    const existing = mergedMap.get(doc.docId);
-    if (existing) {
-      existing.vectorScore = doc.score;
-    } else {
-      mergedMap.set(doc.docId, { textScore: 0, vectorScore: doc.score, textDoc: null });
-    }
-  }
-  const alpha = clampAlpha(params.hybrid?.alpha);
-  const scored = [];
-  for (const [docId, entry] of mergedMap) {
-    const combinedScore = alpha * entry.vectorScore + (1 - alpha) * entry.textScore;
-    scored.push({
-      docId,
-      score: combinedScore,
-      termFrequencies: entry.textDoc?.termFrequencies ?? {},
-      fieldLengths: entry.textDoc?.fieldLengths ?? {},
-      idf: entry.textDoc?.idf ?? {}
-    });
-  }
-  scored.sort((a, b) => b.score - a.score || a.docId.localeCompare(b.docId));
-  let filtered = scored;
-  if (params.minScore !== void 0 && params.minScore > 0) {
-    const threshold = params.minScore;
-    filtered = filtered.filter((doc) => doc.score >= threshold);
-  }
-  if (params.filters) {
-    const matchingDocIds = partition.applyFilters(params.filters, schema);
-    filtered = filtered.filter((doc) => matchingDocIds.has(doc.docId));
-  }
-  return { scored: filtered, totalMatched: filtered.length };
-}
-function runVectorOnly(partition, params) {
-  const vectorConfig = params.vector;
-  const result = partition.searchVector({
-    field: vectorConfig.field,
-    value: vectorConfig.value,
-    k: params.limit ?? 10,
-    similarity: vectorConfig.similarity,
-    metric: vectorConfig.metric,
-    efSearch: vectorConfig.efSearch
-  });
-  return result;
-}
-function normalizeScores(docs) {
-  if (docs.length === 0) return [];
-  let min = docs[0].score;
-  let max = docs[0].score;
-  for (let i = 1; i < docs.length; i++) {
-    if (docs[i].score < min) min = docs[i].score;
-    if (docs[i].score > max) max = docs[i].score;
-  }
-  const range = max - min;
-  if (range === 0) {
-    return docs.map((d) => ({ docId: d.docId, score: 1 }));
-  }
-  return docs.map((d) => ({ docId: d.docId, score: (d.score - min) / range }));
-}
-function clampAlpha(alpha) {
-  if (alpha === void 0) return 0.5;
-  if (!Number.isFinite(alpha)) return 0.5;
-  if (alpha < 0) return 0;
-  if (alpha > 1) return 1;
-  return alpha;
-}
-
 // src/partitioning/distributed-scoring.ts
 function collectGlobalStats(manager) {
   const aggregate = manager.getAggregateStats();
@@ -791,24 +903,6 @@ function dispatchToAllPartitions(partitions, params, language, schema, options,
   return Promise.all(promises);
 }
 function dispatchSinglePartition(partition, params, language, schema, options) {
-  const isHybridMode = params.mode === "hybrid";
-  const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
-  const hasVector = params.vector !== void 0;
-  if (isHybridMode || hasTerm && hasVector) {
-    return hybridSearch(partition, params, language, schema, options);
-  }
-  if (params.mode === "vector" || hasVector && !hasTerm) {
-    if (!params.vector) return { scored: [], totalMatched: 0 };
-    const vectorConfig = params.vector;
-    return partition.searchVector({
-      field: vectorConfig.field,
-      value: vectorConfig.value,
-      k: params.limit ?? 10,
-      similarity: vectorConfig.similarity,
-      metric: vectorConfig.metric,
-      efSearch: vectorConfig.efSearch
-    });
-  }
   return fulltextSearch(partition, params, language, schema, options);
 }
 function collectAndMergeFacets(outcomes, params, schema) {
@@ -936,190 +1030,6 @@ function heapPop(heap) {
   }
   return top;
 }
-
-// src/runtime/detect.ts
-function detectRuntimeName() {
-  if ("Bun" in globalThis) {
-    return "bun";
-  }
-  if ("Deno" in globalThis) {
-    return "deno";
-  }
-  const proc = globalThis.process;
-  if (typeof proc === "object" && proc !== null && typeof proc.versions === "object" && typeof proc.versions?.node === "string") {
-    return "node";
-  }
-  return "browser";
-}
-function getCpuCount() {
-  const nav = globalThis.navigator;
-  if (nav && typeof nav.hardwareConcurrency === "number" && nav.hardwareConcurrency >= 1) {
-    return nav.hardwareConcurrency;
-  }
-  return 1;
-}
-function detectRuntime() {
-  const runtime = detectRuntimeName();
-  return {
-    runtime,
-    cpuCount: getCpuCount(),
-    supportsWorkerThreads: runtime === "node" || runtime === "bun",
-    supportsWebWorkers: runtime === "browser" || runtime === "deno" || runtime === "bun",
-    supportsFileSystem: runtime === "node" || runtime === "bun" || runtime === "deno",
-    supportsIndexedDB: runtime === "browser" || "indexedDB" in globalThis,
-    supportsBroadcastChannel: "BroadcastChannel" in globalThis
-  };
-}
-
-// src/vector/promoter.ts
-function detectWorkerStrategy() {
-  const runtime = detectRuntime();
-  const url = import.meta.url;
-  const hasBundledWorker = url.endsWith(".mjs") || url.endsWith(".js");
-  if (runtime.supportsWorkerThreads && hasBundledWorker) return "worker-threads";
-  if (runtime.supportsWebWorkers && hasBundledWorker) return "web-worker";
-  return "synchronous";
-}
-function collectVectorsForWorker(engine) {
-  const vectors = [];
-  for (const [, entry] of engine.entries()) {
-    vectors.push({ docId: entry.docId, values: Array.from(entry.vector) });
-  }
-  return vectors;
-}
-function buildWorkerRequest(engine, hnswConfig) {
-  return {
-    type: "build",
-    vectors: collectVectorsForWorker(engine),
-    dimension: engine.dimension,
-    config: hnswConfig ?? {}
-  };
-}
-function resolveWorkerUrl() {
-  const base = import.meta.url;
-  const distIndex = base.lastIndexOf("/dist/");
-  if (distIndex !== -1) {
-    return new URL("vector/hnsw-build-worker.mjs", base.slice(0, distIndex + 6)).href;
-  }
-  return base.replace(/\/src\/vector\/[^/]+$/, "/dist/vector/hnsw-build-worker.mjs");
-}
-function spawnNodeWorker(request, onComplete, onError) {
-  const entryUrl = resolveWorkerUrl();
-  let worker = null;
-  import('worker_threads').then((wt) => {
-    worker = new wt.Worker(new URL(entryUrl));
-    worker.on("message", (msg) => {
-      const response = msg;
-      worker?.terminate();
-      if (response.type === "success") {
-        onComplete(response.graph);
-      } else {
-        onError(new Error(response.message));
-      }
-    });
-    worker.on("error", (err) => {
-      worker?.terminate();
-      onError(err instanceof Error ? err : new Error(String(err)));
-    });
-    worker.postMessage(request);
-  }).catch(onError);
-  return {
-    terminate() {
-      worker?.terminate();
-    }
-  };
-}
-function spawnWebWorker(request, onComplete, onError) {
-  const entryUrl = resolveWorkerUrl();
-  const WorkerCtor = globalThis.Worker;
-  const worker = new WorkerCtor(entryUrl, { type: "module" });
-  worker.onmessage = (event) => {
-    const response = event.data;
-    worker.terminate();
-    if (response.type === "success") {
-      onComplete(response.graph);
-    } else {
-      onError(new Error(response.message));
-    }
-  };
-  worker.onerror = (event) => {
-    worker.terminate();
-    onError(event.error ?? new Error(event.message ?? "Web Worker error"));
-  };
-  worker.postMessage(request);
-  return {
-    terminate() {
-      worker.terminate();
-    }
-  };
-}
-function createVectorPromoter(config) {
-  const threshold = config?.promotionThreshold ?? 1e4;
-  const hnswConfig = config?.hnswConfig;
-  const promoting = /* @__PURE__ */ new Set();
-  const strategy = config?.workerStrategy ?? detectWorkerStrategy();
-  const pendingWorkers = /* @__PURE__ */ new Set();
-  return {
-    get strategy() {
-      return strategy;
-    },
-    check(engines) {
-      for (const [field, engine] of engines) {
-        if (engine.isPromoted || promoting.has(field)) continue;
-        if (engine.size < threshold) continue;
-        promoting.add(field);
-        if (strategy === "worker-threads") {
-          const request = buildWorkerRequest(engine, hnswConfig);
-          const handle = spawnNodeWorker(
-            request,
-            (graph) => {
-              pendingWorkers.delete(handle);
-              if (!engine.isPromoted) {
-                engine.deserializeHNSW(graph);
-              }
-              promoting.delete(field);
-            },
-            () => {
-              pendingWorkers.delete(handle);
-              promoting.delete(field);
-            }
-          );
-          pendingWorkers.add(handle);
-        } else if (strategy === "web-worker") {
-          const request = buildWorkerRequest(engine, hnswConfig);
-          const handle = spawnWebWorker(
-            request,
-            (graph) => {
-              pendingWorkers.delete(handle);
-              if (!engine.isPromoted) {
-                engine.deserializeHNSW(graph);
-              }
-              promoting.delete(field);
-            },
-            () => {
-              pendingWorkers.delete(handle);
-              promoting.delete(field);
-            }
-          );
-          pendingWorkers.add(handle);
-        } else {
-          try {
-            engine.promoteToHNSW(hnswConfig);
-          } finally {
-            promoting.delete(field);
-          }
-        }
-      }
-    },
-    shutdown() {
-      for (const handle of pendingWorkers) {
-        handle.terminate();
-      }
-      pendingWorkers.clear();
-      promoting.clear();
-    }
-  };
-}
 function encodeRawPayloadV2(wire) {
   return encode(wire);
 }
@@ -1182,6 +1092,13 @@ function deserializePayloadV2(data) {
         efConstruction: data2.hnsw_graph.ef_construction,
         metric: validateHnswMetric(data2.hnsw_graph.metric),
         nodes: data2.hnsw_graph.nodes
+      } : null,
+      sq8: data2.sq8 ? {
+        alpha: data2.sq8.alpha,
+        offset: data2.sq8.offset,
+        quantizedVectors: data2.sq8.quantized_vectors,
+        vectorSums: data2.sq8.vector_sums,
+        vectorSumSqs: data2.sq8.vector_sum_sqs
       } : null
     };
   }
@@ -1214,18 +1131,52 @@ function deserializePayloadV2(data) {
 // src/core/document-store.ts
 function createDocumentStore() {
   const docs = /* @__PURE__ */ new Map();
+  const forwardMap = /* @__PURE__ */ new Map();
+  const reverseMap = [];
+  let nextInternalId = 0;
+  function assignInternalId(docId) {
+    const existing = forwardMap.get(docId);
+    if (existing !== void 0) return existing;
+    const internalId = nextInternalId++;
+    forwardMap.set(docId, internalId);
+    reverseMap[internalId] = docId;
+    return internalId;
+  }
+  function clearMappings() {
+    forwardMap.clear();
+    reverseMap.length = 0;
+    nextInternalId = 0;
+  }
+  const idResolver = {
+    toExternal(internalId) {
+      return reverseMap[internalId];
+    },
+    toInternal(externalId) {
+      return forwardMap.get(externalId);
+    }
+  };
   return {
     store(docId, document, fieldLengths) {
-      docs.set(docId, { fields: structuredClone(document), fieldLengths });
+      assignInternalId(docId);
+      docs.set(docId, { fields: document, fieldLengths });
     },
     storeRef(docId, document, fieldLengths) {
+      assignInternalId(docId);
       docs.set(docId, { fields: document, fieldLengths });
     },
     get(docId) {
       return docs.get(docId);
     },
     remove(docId) {
-      return docs.delete(docId);
+      const removed = docs.delete(docId);
+      if (removed) {
+        const internalId = forwardMap.get(docId);
+        if (internalId !== void 0) {
+          forwardMap.delete(docId);
+          reverseMap[internalId] = void 0;
+        }
+      }
+      return removed;
     },
     has(docId) {
       return docs.has(docId);
@@ -1238,6 +1189,7 @@ function createDocumentStore() {
     },
     clear() {
       docs.clear();
+      clearMappings();
     },
     serialize() {
       const result = {};
@@ -1248,9 +1200,31 @@ function createDocumentStore() {
     },
     deserialize(data) {
       docs.clear();
+      clearMappings();
       for (const docId of Object.keys(data)) {
+        assignInternalId(docId);
         docs.set(docId, data[docId]);
       }
+    },
+    ensureInternalId(docId) {
+      return assignInternalId(docId);
+    },
+    getInternalId(docId) {
+      return forwardMap.get(docId);
+    },
+    getExternalId(internalId) {
+      return reverseMap[internalId];
+    },
+    *allInternalIds() {
+      for (const internalId of forwardMap.values()) {
+        yield internalId;
+      }
+    },
+    internalIdCapacity() {
+      return nextInternalId;
+    },
+    resolver() {
+      return idResolver;
     }
   };
 }
@@ -1262,6 +1236,8 @@ var COMPACTION_THRESHOLD = 0.3;
 function createInvertedIndex(fieldNameTable) {
   const index = /* @__PURE__ */ new Map();
   const charBuckets = /* @__PURE__ */ new Map();
+  let batchMode = false;
+  const batchDirtyTokens = /* @__PURE__ */ new Set();
   function trackToken(token) {
     if (token.length === 0) return;
     const ch = token[0];
@@ -1345,10 +1321,10 @@ function createInvertedIndex(fieldNameTable) {
     list.length = writeIdx;
     list.deletedDocs.clear();
   }
-  function compactDocEntries(list, docId) {
+  function compactDocEntries(list, internalId) {
     let writeIdx = 0;
     for (let i = 0; i < list.length; i++) {
-      if (list.docIds[i] !== docId) {
+      if (list.docIds[i] !== internalId) {
         if (writeIdx !== i) {
           list.docIds[writeIdx] = list.docIds[i];
           list.termFrequencies[writeIdx] = list.termFrequencies[i];
@@ -1365,17 +1341,17 @@ function createInvertedIndex(fieldNameTable) {
     list.length = writeIdx;
   }
   return {
-    insert(token, docId, termFrequency, fieldNameIndex, positions) {
+    insert(token, internalId, termFrequency, fieldNameIndex, positions) {
       const list = getOrCreateList(token);
-      if (list.deletedDocs.size > 0 && list.deletedDocs.has(docId)) {
-        compactDocEntries(list, docId);
-        list.deletedDocs.delete(docId);
+      if (list.deletedDocs.size > 0 && list.deletedDocs.has(internalId)) {
+        compactDocEntries(list, internalId);
+        list.deletedDocs.delete(internalId);
       }
       if (list.length >= list.termFrequencies.length) {
         growTypedArrays(list);
       }
       const idx = list.length;
-      list.docIds.push(docId);
+      list.docIds.push(internalId);
       list.termFrequencies[idx] = termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : termFrequency;
       list.fieldNameIndices[idx] = fieldNameIndex;
       if (positions !== null) {
@@ -1387,25 +1363,45 @@ function createInvertedIndex(fieldNameTable) {
       } else if (list.positions !== null) {
         list.positions.push([]);
       }
-      if (!list.docIdSet.has(docId)) {
-        list.docIdSet.add(docId);
+      if (!list.docIdSet.has(internalId)) {
+        list.docIdSet.add(internalId);
       }
       list.length++;
     },
-    remove(token, docId) {
+    remove(token, internalId) {
       const list = index.get(token);
-      if (!list || !list.docIdSet.has(docId)) return;
-      list.docIdSet.delete(docId);
-      list.deletedDocs.add(docId);
+      if (!list || !list.docIdSet.has(internalId)) return;
+      list.docIdSet.delete(internalId);
+      list.deletedDocs.add(internalId);
       if (list.docIdSet.size === 0) {
         index.delete(token);
         untrackToken(token);
+        if (batchMode) batchDirtyTokens.delete(token);
+        return;
+      }
+      if (batchMode) {
+        batchDirtyTokens.add(token);
         return;
       }
       if (list.deletedDocs.size / list.length > COMPACTION_THRESHOLD) {
         compactList(list);
       }
     },
+    beginBatch() {
+      batchMode = true;
+      batchDirtyTokens.clear();
+    },
+    endBatch() {
+      batchMode = false;
+      for (const token of batchDirtyTokens) {
+        const list = index.get(token);
+        if (!list) continue;
+        if (list.deletedDocs.size / list.length > COMPACTION_THRESHOLD) {
+          compactList(list);
+        }
+      }
+      batchDirtyTokens.clear();
+    },
     lookup(token) {
       return index.get(token);
     },
@@ -1452,14 +1448,16 @@ function createInvertedIndex(fieldNameTable) {
       index.clear();
       charBuckets.clear();
     },
-    serialize() {
+    serialize(resolver) {
       const result = /* @__PURE__ */ Object.create(null);
       for (const [token, list] of index) {
         const postings = [];
         for (let i = 0; i < list.length; i++) {
           if (list.deletedDocs.size > 0 && list.deletedDocs.has(list.docIds[i])) continue;
+          const externalId = resolver.toExternal(list.docIds[i]);
+          if (externalId === void 0) continue;
           postings.push({
-            docId: list.docIds[i],
+            docId: externalId,
             termFrequency: list.termFrequencies[i],
             fieldName: fieldNameTable.names[list.fieldNameIndices[i]],
             positions: list.positions ? list.positions[i] : []
@@ -1471,7 +1469,7 @@ function createInvertedIndex(fieldNameTable) {
       }
       return result;
     },
-    deserialize(data) {
+    deserialize(data, resolver) {
       index.clear();
       charBuckets.clear();
       for (const token of Object.keys(data)) {
@@ -1482,32 +1480,44 @@ function createInvertedIndex(fieldNameTable) {
         const termFrequencies = new Uint16Array(count);
         const fieldNameIndices = new Uint8Array(count);
         let hasPositions = false;
+        let validCount = 0;
         for (let i = 0; i < count; i++) {
           const p = src.postings[i];
-          docIds[i] = p.docId;
-          termFrequencies[i] = p.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : p.termFrequency;
+          const internalId = resolver.toInternal(p.docId);
+          if (internalId === void 0) continue;
+          docIds[validCount] = internalId;
+          termFrequencies[validCount] = p.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : p.termFrequency;
           let fnIndex = fieldNameTable.indexMap.get(p.fieldName);
           if (fnIndex === void 0) {
             fnIndex = fieldNameTable.names.length;
             fieldNameTable.names.push(p.fieldName);
             fieldNameTable.indexMap.set(p.fieldName, fnIndex);
           }
-          fieldNameIndices[i] = fnIndex;
-          docIdSet.add(p.docId);
+          fieldNameIndices[validCount] = fnIndex;
+          docIdSet.add(internalId);
           if (p.positions.length > 0) hasPositions = true;
+          validCount++;
         }
+        if (validCount === 0) continue;
+        docIds.length = validCount;
+        const finalTF = validCount < count ? termFrequencies.slice(0, validCount) : termFrequencies;
+        const finalFNI = validCount < count ? fieldNameIndices.slice(0, validCount) : fieldNameIndices;
         let positions = null;
         if (hasPositions) {
-          positions = new Array(count);
+          positions = new Array(validCount);
+          let vi = 0;
           for (let i = 0; i < count; i++) {
-            positions[i] = src.postings[i].positions;
+            const internalId = resolver.toInternal(src.postings[i].docId);
+            if (internalId === void 0) continue;
+            positions[vi] = src.postings[i].positions;
+            vi++;
           }
         }
         index.set(token, {
-          length: count,
+          length: validCount,
           docIds,
-          termFrequencies,
-          fieldNameIndices,
+          termFrequencies: finalTF,
+          fieldNameIndices: finalFNI,
           positions,
           docIdSet,
           deletedDocs: /* @__PURE__ */ new Set()
@@ -1519,16 +1529,55 @@ function createInvertedIndex(fieldNameTable) {
 }
 
 // src/core/statistics.ts
-function collectUniqueTokens(tokens) {
-  const unique = /* @__PURE__ */ new Set();
+function applyUniqueTokens(tokens, docFrequencies, increment) {
   const fields = Object.keys(tokens);
+  if (fields.length === 1) {
+    const fieldTokens = tokens[fields[0]];
+    if (increment === 1) {
+      const seen = /* @__PURE__ */ new Set();
+      for (let j = 0; j < fieldTokens.length; j++) {
+        const tok = fieldTokens[j];
+        if (seen.has(tok)) continue;
+        seen.add(tok);
+        docFrequencies[tok] = (docFrequencies[tok] ?? 0) + 1;
+      }
+    } else {
+      const seen = /* @__PURE__ */ new Set();
+      for (let j = 0; j < fieldTokens.length; j++) {
+        const tok = fieldTokens[j];
+        if (seen.has(tok)) continue;
+        seen.add(tok);
+        if (tok in docFrequencies) {
+          docFrequencies[tok]--;
+          if (docFrequencies[tok] <= 0) {
+            delete docFrequencies[tok];
+          }
+        }
+      }
+    }
+    return;
+  }
+  const unique = /* @__PURE__ */ new Set();
   for (let i = 0; i < fields.length; i++) {
     const fieldTokens = tokens[fields[i]];
     for (let j = 0; j < fieldTokens.length; j++) {
       unique.add(fieldTokens[j]);
     }
   }
-  return unique;
+  if (increment === 1) {
+    for (const tok of unique) {
+      docFrequencies[tok] = (docFrequencies[tok] ?? 0) + 1;
+    }
+  } else {
+    for (const tok of unique) {
+      if (tok in docFrequencies) {
+        docFrequencies[tok]--;
+        if (docFrequencies[tok] <= 0) {
+          delete docFrequencies[tok];
+        }
+      }
+    }
+  }
 }
 function createPartitionStats() {
   const stats = {
@@ -1543,10 +1592,7 @@ function createPartitionStats() {
         const field = fields[i];
         stats.totalFieldLengths[field] = (stats.totalFieldLengths[field] ?? 0) + fieldLengths[field];
       }
-      const uniqueTokens = collectUniqueTokens(tokens);
-      for (const token of uniqueTokens) {
-        stats.docFrequencies[token] = (stats.docFrequencies[token] ?? 0) + 1;
-      }
+      applyUniqueTokens(tokens, stats.docFrequencies, 1);
       stats.recalculateAverages();
     },
     removeDocument(fieldLengths, tokens) {
@@ -1562,15 +1608,7 @@ function createPartitionStats() {
           }
         }
       }
-      const uniqueTokens = collectUniqueTokens(tokens);
-      for (const token of uniqueTokens) {
-        if (token in stats.docFrequencies) {
-          stats.docFrequencies[token]--;
-          if (stats.docFrequencies[token] <= 0) {
-            delete stats.docFrequencies[token];
-          }
-        }
-      }
+      applyUniqueTokens(tokens, stats.docFrequencies, -1);
       stats.recalculateAverages();
     },
     recalculateAverages() {
@@ -1719,12 +1757,12 @@ function createGeoIndex() {
     get entries() {
       return entries;
     },
-    insert(docId, lat, lon) {
-      entries.push({ lat, lon, docId });
+    insert(internalId, lat, lon) {
+      entries.push({ lat, lon, docId: internalId });
     },
-    remove(docId) {
+    remove(internalId) {
       for (let i = entries.length - 1; i >= 0; i--) {
-        if (entries[i].docId === docId) {
+        if (entries[i].docId === internalId) {
           entries.splice(i, 1);
           return;
         }
@@ -1769,188 +1807,6 @@ function createGeoIndex() {
   };
 }
 
-// src/vector/brute-force.ts
-function createBruteForceVectorStore(dimension) {
-  const vectors = /* @__PURE__ */ new Map();
-  return {
-    get dimension() {
-      return dimension;
-    },
-    get size() {
-      return vectors.size;
-    },
-    insert(docId, vector) {
-      if (vector.length !== dimension) {
-        throw new Error(`Vector dimension mismatch: expected ${dimension}, got ${vector.length}`);
-      }
-      const mag = magnitude(vector);
-      vectors.set(docId, { docId, vector, magnitude: mag });
-    },
-    remove(docId) {
-      vectors.delete(docId);
-    },
-    has(docId) {
-      return vectors.has(docId);
-    },
-    search(query, k, metric, minSimilarity, filterDocIds) {
-      if (query.length !== dimension) {
-        throw new Error(`Query dimension mismatch: expected ${dimension}, got ${query.length}`);
-      }
-      const queryMag = magnitude(query);
-      const results = [];
-      for (const [docId, entry] of vectors) {
-        if (filterDocIds && !filterDocIds.has(docId)) continue;
-        let score;
-        switch (metric) {
-          case "cosine":
-            score = cosineSimilarityWithMagnitudes(query, entry.vector, queryMag, entry.magnitude);
-            break;
-          case "dotProduct":
-            score = dotProduct(query, entry.vector);
-            break;
-          case "euclidean": {
-            const dist = euclideanDistance(query, entry.vector);
-            score = 1 / (1 + dist);
-            break;
-          }
-        }
-        if (score >= minSimilarity) {
-          results.push({ docId, score });
-        }
-      }
-      results.sort((a, b) => b.score - a.score);
-      const topK = results.slice(0, k);
-      return topK.map((r) => ({
-        docId: r.docId,
-        score: r.score,
-        termFrequencies: {},
-        fieldLengths: {},
-        idf: {}
-      }));
-    },
-    clear() {
-      vectors.clear();
-    },
-    entries() {
-      return vectors.entries();
-    }
-  };
-}
-
-// src/search/vector-search.ts
-function createVectorSearchEngine(dimension, defaultHNSWConfig) {
-  const bruteForce = createBruteForceVectorStore(dimension);
-  const hnswConfig = defaultHNSWConfig;
-  let hnsw = null;
-  return {
-    get dimension() {
-      return dimension;
-    },
-    get size() {
-      return bruteForce.size;
-    },
-    get isPromoted() {
-      return hnsw !== null;
-    },
-    insert(docId, vector) {
-      bruteForce.insert(docId, vector);
-      if (hnsw) {
-        hnsw.insert(docId, vector);
-      }
-    },
-    remove(docId) {
-      bruteForce.remove(docId);
-      if (hnsw) {
-        hnsw.remove(docId);
-      }
-    },
-    has(docId) {
-      return bruteForce.has(docId);
-    },
-    search(query, k, metric, minSimilarity, filterDocIds, efSearch) {
-      if (hnsw) {
-        return hnsw.search(query, k, metric, minSimilarity, filterDocIds, efSearch);
-      }
-      return bruteForce.search(query, k, metric, minSimilarity, filterDocIds);
-    },
-    clear() {
-      bruteForce.clear();
-      if (hnsw) {
-        hnsw.clear();
-        hnsw = null;
-      }
-    },
-    entries() {
-      return bruteForce.entries();
-    },
-    promoteToHNSW(overrideConfig) {
-      const cfg = overrideConfig ?? hnswConfig;
-      const newHnsw = createHNSWIndex(dimension, cfg);
-      for (const [, entry] of bruteForce.entries()) {
-        newHnsw.insert(entry.docId, entry.vector);
-      }
-      hnsw = newHnsw;
-    },
-    demoteToLinear() {
-      if (hnsw) {
-        hnsw.clear();
-        hnsw = null;
-      }
-    },
-    getHNSWIndex() {
-      return hnsw;
-    },
-    getBruteForceStore() {
-      return bruteForce;
-    },
-    estimateMemoryBytes() {
-      const count = bruteForce.size;
-      if (count === 0) return 0;
-      const MAP_OVERHEAD = 64;
-      const MAP_ENTRY = 72;
-      const VECTOR_ENTRY_OBJ = 32;
-      const AVG_DOCID_BYTES = 56;
-      const TYPED_ARRAY_HEADER = 64;
-      const MAGNITUDE_BYTES = 8;
-      const perBruteForceEntry = MAP_ENTRY + VECTOR_ENTRY_OBJ + AVG_DOCID_BYTES + TYPED_ARRAY_HEADER + MAGNITUDE_BYTES;
-      let bytes = MAP_OVERHEAD + count * (perBruteForceEntry + dimension * 4);
-      if (hnsw) {
-        const HNSW_NODE_OBJ = 64;
-        const SHARED_REFS = 16;
-        const CONN_ARRAY_HEADER = 32;
-        const SET_OVERHEAD = 64;
-        const SET_ENTRY_COST = 72;
-        const m = hnsw.m;
-        const avgLayers = m / (m - 1);
-        const avgConnsLayer0 = m;
-        const avgConnsUpper = Math.ceil(m / 2);
-        const connMemPerNode = CONN_ARRAY_HEADER + (SET_OVERHEAD + avgConnsLayer0 * SET_ENTRY_COST) + Math.max(0, avgLayers - 1) * (SET_OVERHEAD + avgConnsUpper * SET_ENTRY_COST);
-        const perHnswNode = MAP_ENTRY + HNSW_NODE_OBJ + SHARED_REFS + connMemPerNode;
-        bytes += MAP_OVERHEAD + count * perHnswNode;
-      }
-      return Math.round(bytes);
-    },
-    serializeHNSW() {
-      if (!hnsw) return null;
-      return hnsw.serialize();
-    },
-    deserializeHNSW(data) {
-      const cfg = {};
-      const restoredHnsw = createHNSWIndex(dimension, {
-        m: data.m ?? cfg.m,
-        efConstruction: data.efConstruction ?? cfg.efConstruction,
-        metric: data.metric ?? cfg.metric
-      });
-      const vectorMap = /* @__PURE__ */ new Map();
-      for (const [, entry] of bruteForce.entries()) {
-        vectorMap.set(entry.docId, { vector: entry.vector, mag: entry.magnitude });
-      }
-      restoredHnsw.deserialize(data, vectorMap);
-      hnsw = restoredHnsw;
-    }
-  };
-}
-
 // src/core/field-index.ts
 function lowerBound(entries, value) {
   let lo = 0;
@@ -1979,6 +1835,13 @@ function collectDocIds(entries, from, to) {
   }
   return result;
 }
+function collectDocIdsBitset(entries, from, to, capacity) {
+  const bs = createBitSet(capacity);
+  for (let i = from; i < to; i++) {
+    bitsetSet(bs, entries[i].docId);
+  }
+  return bs;
+}
 function createNumericIndex() {
   let entries = [];
   let sorted = true;
@@ -1989,16 +1852,16 @@ function createNumericIndex() {
     }
   }
   return {
-    insert(docId, value) {
-      entries.push({ value, docId });
+    insert(internalId, value) {
+      entries.push({ value, docId: internalId });
       sorted = false;
     },
-    remove(docId, value) {
+    remove(internalId, value) {
       ensureSorted();
       const start = lowerBound(entries, value);
       const end = upperBound(entries, value);
       for (let i = start; i < end; i++) {
-        if (entries[i].docId === docId) {
+        if (entries[i].docId === internalId) {
           entries.splice(i, 1);
           return;
         }
@@ -2040,6 +1903,33 @@ function createNumericIndex() {
     getAllDocIds() {
       return collectDocIds(entries, 0, entries.length);
     },
+    queryEqBitset(value, capacity) {
+      ensureSorted();
+      return collectDocIdsBitset(entries, lowerBound(entries, value), upperBound(entries, value), capacity);
+    },
+    queryGtBitset(value, capacity) {
+      ensureSorted();
+      return collectDocIdsBitset(entries, upperBound(entries, value), entries.length, capacity);
+    },
+    queryGteBitset(value, capacity) {
+      ensureSorted();
+      return collectDocIdsBitset(entries, lowerBound(entries, value), entries.length, capacity);
+    },
+    queryLtBitset(value, capacity) {
+      ensureSorted();
+      return collectDocIdsBitset(entries, 0, lowerBound(entries, value), capacity);
+    },
+    queryLteBitset(value, capacity) {
+      ensureSorted();
+      return collectDocIdsBitset(entries, 0, upperBound(entries, value), capacity);
+    },
+    queryBetweenBitset(min, max, capacity) {
+      ensureSorted();
+      return collectDocIdsBitset(entries, lowerBound(entries, min), upperBound(entries, max), capacity);
+    },
+    getAllDocIdsBitset(capacity) {
+      return collectDocIdsBitset(entries, 0, entries.length, capacity);
+    },
     count() {
       return entries.length;
     },
@@ -2061,13 +1951,13 @@ function createBooleanIndex() {
   const trueDocs = /* @__PURE__ */ new Set();
   const falseDocs = /* @__PURE__ */ new Set();
   return {
-    insert(docId, value) {
-      if (value) trueDocs.add(docId);
-      else falseDocs.add(docId);
+    insert(internalId, value) {
+      if (value) trueDocs.add(internalId);
+      else falseDocs.add(internalId);
     },
-    remove(docId, value) {
-      if (value) trueDocs.delete(docId);
-      else falseDocs.delete(docId);
+    remove(internalId, value) {
+      if (value) trueDocs.delete(internalId);
+      else falseDocs.delete(internalId);
     },
     queryEq(value) {
       return new Set(value ? trueDocs : falseDocs);
@@ -2080,6 +1970,20 @@ function createBooleanIndex() {
       for (const id of falseDocs) all.add(id);
       return all;
     },
+    queryEqBitset(value, capacity) {
+      const source = value ? trueDocs : falseDocs;
+      const bs = createBitSet(capacity);
+      for (const id of source) {
+        bitsetSet(bs, id);
+      }
+      return bs;
+    },
+    getAllDocIdsBitset(capacity) {
+      const bs = createBitSet(capacity);
+      for (const id of trueDocs) bitsetSet(bs, id);
+      for (const id of falseDocs) bitsetSet(bs, id);
+      return bs;
+    },
     count() {
       return trueDocs.size + falseDocs.size;
     },
@@ -2111,18 +2015,18 @@ function createEnumIndex() {
     return result;
   }
   return {
-    insert(docId, value) {
+    insert(internalId, value) {
       let docSet = valueMap.get(value);
       if (!docSet) {
         docSet = /* @__PURE__ */ new Set();
         valueMap.set(value, docSet);
       }
-      docSet.add(docId);
+      docSet.add(internalId);
     },
-    remove(docId, value) {
+    remove(internalId, value) {
       const docSet = valueMap.get(value);
       if (!docSet) return;
-      docSet.delete(docId);
+      docSet.delete(internalId);
       if (docSet.size === 0) valueMap.delete(value);
     },
     queryEq(value) {
@@ -2156,6 +2060,31 @@ function createEnumIndex() {
     getAllDocIds() {
       return allDocs();
     },
+    queryEqBitset(value, capacity) {
+      const bs = createBitSet(capacity);
+      const docSet = valueMap.get(value);
+      if (docSet) {
+        for (const id of docSet) bitsetSet(bs, id);
+      }
+      return bs;
+    },
+    queryInBitset(values, capacity) {
+      const bs = createBitSet(capacity);
+      for (const val of values) {
+        const docSet = valueMap.get(val);
+        if (docSet) {
+          for (const id of docSet) bitsetSet(bs, id);
+        }
+      }
+      return bs;
+    },
+    getAllDocIdsBitset(capacity) {
+      const bs = createBitSet(capacity);
+      for (const docSet of valueMap.values()) {
+        for (const id of docSet) bitsetSet(bs, id);
+      }
+      return bs;
+    },
     count() {
       let total = 0;
       for (const docSet of valueMap.values()) total += docSet.size;
@@ -2189,7 +2118,6 @@ function getOrCreateFieldNameIndex(table, fieldName) {
   table.indexMap.set(fieldName, idx);
   return idx;
 }
-var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
 function getNestedValue(obj, path) {
   const parts = path.split(".");
   let current = obj;
@@ -2221,10 +2149,6 @@ function textFieldsChanged(oldDoc, newDoc, flatSchema) {
   }
   return false;
 }
-function parseVectorDimension(fieldType) {
-  const match = VECTOR_PATTERN2.exec(fieldType);
-  return match ? Number.parseInt(match[1], 10) : null;
-}
 function tokenizeOptions(options) {
   return {
     stem: true,
@@ -2244,10 +2168,15 @@ function getFieldValueForDoc(docStore, docId, fieldPath) {
   if (!stored) return void 0;
   return getNestedValue(stored.fields, fieldPath);
 }
-function getAllDocIds(docStore) {
+function getFieldValueByInternalId(docStore, internalId, fieldPath) {
+  const externalId = docStore.getExternalId(internalId);
+  if (externalId === void 0) return void 0;
+  return getFieldValueForDoc(docStore, externalId, fieldPath);
+}
+function getAllInternalDocIds(docStore) {
   const ids = /* @__PURE__ */ new Set();
-  for (const [id] of docStore.all()) {
-    ids.add(id);
+  for (const internalId of docStore.allInternalIds()) {
+    ids.add(internalId);
   }
   return ids;
 }
@@ -2262,11 +2191,6 @@ function ensureFieldIndex(state, fieldPath, fieldType) {
     state.enumIndexes.set(fieldPath, createEnumIndex());
   } else if (fieldType === "geopoint" && !state.geoIndexes.has(fieldPath)) {
     state.geoIndexes.set(fieldPath, createGeoIndex());
-  } else {
-    const dim = parseVectorDimension(fieldType);
-    if (dim !== null && !state.vectorStores.has(fieldPath)) {
-      state.vectorStores.set(fieldPath, createVectorSearchEngine(dim));
-    }
   }
 }
 function getOrCreateNumericIndex(state, fieldPath) {
@@ -2285,14 +2209,20 @@ function getOrCreateBooleanIndex(state, fieldPath) {
   }
   return idx;
 }
-function indexStringField(state, docId, fieldPath, text, language, options, fieldLengths, tokensByField) {
-  const result = tokenize(text, language, tokenizeOptions(options));
-  fieldLengths[fieldPath] = result.tokens.length;
+function resolveInternalId(state, docId) {
+  const internalId = state.docStore.getInternalId(docId);
+  if (internalId === void 0) {
+    throw new Error(`Internal ID not found for document "${docId}"`);
+  }
+  return internalId;
+}
+function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField) {
   const fieldTokenList = [];
   const fieldNameIndex = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
+  const opts = tokenizeOptions(options);
   if (state.trackPositions) {
     const tokenFreqs = /* @__PURE__ */ new Map();
-    for (const t of result.tokens) {
+    for (const t of tokenizeIterator(text, language, opts)) {
       const existing = tokenFreqs.get(t.token);
       if (existing) {
         existing.count++;
@@ -2303,29 +2233,31 @@ function indexStringField(state, docId, fieldPath, text, language, options, fiel
       fieldTokenList.push(t.token);
     }
     for (const [token, freq] of tokenFreqs) {
-      state.invertedIdx.insert(token, docId, freq.count, fieldNameIndex, freq.positions);
+      state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex, freq.positions);
     }
   } else {
     const tokenCounts = /* @__PURE__ */ new Map();
-    for (const t of result.tokens) {
+    for (const t of tokenizeIterator(text, language, opts)) {
       tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
       fieldTokenList.push(t.token);
     }
     for (const [token, count] of tokenCounts) {
-      state.invertedIdx.insert(token, docId, count, fieldNameIndex, null);
+      state.invertedIdx.insert(token, internalId, count, fieldNameIndex, null);
     }
   }
+  fieldLengths[fieldPath] = fieldTokenList.length;
   tokensByField[fieldPath] = fieldTokenList;
 }
-function indexStringArrayField(state, docId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
+function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
   const fieldTokenList = [];
   const fieldNameIndex = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
+  const opts = tokenizeOptions(options);
   if (state.trackPositions) {
     const tokenFreqs = /* @__PURE__ */ new Map();
     let positionOffset = 0;
     for (const item of arr) {
-      const result = tokenize(item, language, tokenizeOptions(options));
-      for (const t of result.tokens) {
+      let itemTokenCount = 0;
+      for (const t of tokenizeIterator(item, language, opts)) {
         const existing = tokenFreqs.get(t.token);
         if (existing) {
           existing.count++;
@@ -2334,25 +2266,25 @@ function indexStringArrayField(state, docId, fieldPath, arr, language, options,
           tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
         }
         fieldTokenList.push(t.token);
+        itemTokenCount++;
       }
-      positionOffset += result.tokens.length;
+      positionOffset += itemTokenCount;
     }
     fieldLengths[fieldPath] = fieldTokenList.length;
     for (const [token, freq] of tokenFreqs) {
-      state.invertedIdx.insert(token, docId, freq.count, fieldNameIndex, freq.positions);
+      state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex, freq.positions);
     }
   } else {
     const tokenCounts = /* @__PURE__ */ new Map();
     for (const item of arr) {
-      const result = tokenize(item, language, tokenizeOptions(options));
-      for (const t of result.tokens) {
+      for (const t of tokenizeIterator(item, language, opts)) {
         tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
         fieldTokenList.push(t.token);
       }
     }
     fieldLengths[fieldPath] = fieldTokenList.length;
     for (const [token, count] of tokenCounts) {
-      state.invertedIdx.insert(token, docId, count, fieldNameIndex, null);
+      state.invertedIdx.insert(token, internalId, count, fieldNameIndex, null);
     }
   }
   tokensByField[fieldPath] = fieldTokenList;
@@ -2360,39 +2292,43 @@ function indexStringArrayField(state, docId, fieldPath, arr, language, options,
 function indexDocument(state, docId, document, flatSchema, language, options) {
   const fieldLengths = {};
   const tokensByField = {};
+  const internalId = resolveInternalId(state, docId);
   for (const [fieldPath, fieldType] of Object.entries(flatSchema)) {
     const value = getNestedValue(document, fieldPath);
     if (value === void 0 || value === null) continue;
     ensureFieldIndex(state, fieldPath, fieldType);
     if (fieldType === "string") {
-      indexStringField(state, docId, fieldPath, value, language, options, fieldLengths, tokensByField);
+      indexStringField(state, internalId, fieldPath, value, language, options, fieldLengths, tokensByField);
     } else if (fieldType === "number") {
-      getOrCreateNumericIndex(state, fieldPath).insert(docId, value);
+      getOrCreateNumericIndex(state, fieldPath).insert(internalId, value);
     } else if (fieldType === "boolean") {
-      getOrCreateBooleanIndex(state, fieldPath).insert(docId, value);
+      getOrCreateBooleanIndex(state, fieldPath).insert(internalId, value);
     } else if (fieldType === "enum") {
-      state.enumIndexes.get(fieldPath)?.insert(docId, value);
+      state.enumIndexes.get(fieldPath)?.insert(internalId, value);
     } else if (fieldType === "geopoint") {
       const geo = value;
-      state.geoIndexes.get(fieldPath)?.insert(docId, geo.lat, geo.lon);
+      state.geoIndexes.get(fieldPath)?.insert(internalId, geo.lat, geo.lon);
     } else if (fieldType === "string[]") {
-      indexStringArrayField(state, docId, fieldPath, value, language, options, fieldLengths, tokensByField);
+      indexStringArrayField(
+        state,
+        internalId,
+        fieldPath,
+        value,
+        language,
+        options,
+        fieldLengths,
+        tokensByField
+      );
     } else if (fieldType === "number[]") {
       const numIdx = getOrCreateNumericIndex(state, fieldPath);
-      for (const num of value) numIdx.insert(docId, num);
+      for (const num of value) numIdx.insert(internalId, num);
     } else if (fieldType === "boolean[]") {
       const boolIdx = getOrCreateBooleanIndex(state, fieldPath);
-      for (const b of value) boolIdx.insert(docId, b);
+      for (const b of value) boolIdx.insert(internalId, b);
     } else if (fieldType === "enum[]") {
       const enumIdx = state.enumIndexes.get(fieldPath);
       if (enumIdx) {
-        for (const e of value) enumIdx.insert(docId, e);
-      }
-    } else {
-      const dim = parseVectorDimension(fieldType);
-      if (dim !== null) {
-        const vec = value instanceof Float32Array ? value : new Float32Array(value);
-        state.vectorStores.get(fieldPath)?.insert(docId, vec);
+        for (const e of value) enumIdx.insert(internalId, e);
       }
     }
   }
@@ -2402,6 +2338,7 @@ function removeFromIndexes(state, docId, storedDoc, flatSchema, language, option
   const fieldLengths = {};
   const tokensByField = {};
   const fields = storedDoc.fields;
+  const internalId = resolveInternalId(state, docId);
   for (const [fieldPath, fieldType] of Object.entries(flatSchema)) {
     const value = getNestedValue(fields, fieldPath);
     if (value === void 0 || value === null) continue;
@@ -2414,16 +2351,16 @@ function removeFromIndexes(state, docId, storedDoc, flatSchema, language, option
         uniqueTokens.add(t.token);
         fieldTokenList.push(t.token);
       }
-      for (const token of uniqueTokens) state.invertedIdx.remove(token, docId);
+      for (const token of uniqueTokens) state.invertedIdx.remove(token, internalId);
       tokensByField[fieldPath] = fieldTokenList;
     } else if (fieldType === "number") {
-      state.numericIndexes.get(fieldPath)?.remove(docId, value);
+      state.numericIndexes.get(fieldPath)?.remove(internalId, value);
     } else if (fieldType === "boolean") {
-      state.booleanIndexes.get(fieldPath)?.remove(docId, value);
+      state.booleanIndexes.get(fieldPath)?.remove(internalId, value);
     } else if (fieldType === "enum") {
-      state.enumIndexes.get(fieldPath)?.remove(docId, value);
+      state.enumIndexes.get(fieldPath)?.remove(internalId, value);
     } else if (fieldType === "geopoint") {
-      state.geoIndexes.get(fieldPath)?.remove(docId);
+      state.geoIndexes.get(fieldPath)?.remove(internalId);
     } else if (fieldType === "string[]") {
       const arr = value;
       const uniqueTokens = /* @__PURE__ */ new Set();
@@ -2436,32 +2373,29 @@ function removeFromIndexes(state, docId, storedDoc, flatSchema, language, option
         }
       }
       fieldLengths[fieldPath] = fieldTokenList.length;
-      for (const token of uniqueTokens) state.invertedIdx.remove(token, docId);
+      for (const token of uniqueTokens) state.invertedIdx.remove(token, internalId);
       tokensByField[fieldPath] = fieldTokenList;
     } else if (fieldType === "number[]") {
       const numIdx = state.numericIndexes.get(fieldPath);
       if (numIdx) {
-        for (const num of value) numIdx.remove(docId, num);
+        for (const num of value) numIdx.remove(internalId, num);
       }
     } else if (fieldType === "boolean[]") {
       const boolIdx = state.booleanIndexes.get(fieldPath);
       if (boolIdx) {
-        for (const b of value) boolIdx.remove(docId, b);
+        for (const b of value) boolIdx.remove(internalId, b);
       }
     } else if (fieldType === "enum[]") {
       const enumIdx = state.enumIndexes.get(fieldPath);
       if (enumIdx) {
-        for (const e of value) enumIdx.remove(docId, e);
-      }
-    } else {
-      if (parseVectorDimension(fieldType) !== null) {
-        state.vectorStores.get(fieldPath)?.remove(docId);
+        for (const e of value) enumIdx.remove(internalId, e);
       }
     }
   }
   return { fieldLengths, tokensByField };
 }
 function updateFieldIndexOnly(state, docId, oldFields, newDoc, flatSchema) {
+  const internalId = resolveInternalId(state, docId);
   for (const [fieldPath, fieldType] of Object.entries(flatSchema)) {
     if (fieldType === "string" || fieldType === "string[]") continue;
     const oldVal = getNestedValue(oldFields, fieldPath);
@@ -2475,67 +2409,58 @@ function updateFieldIndexOnly(state, docId, oldFields, newDoc, flatSchema) {
     if (fieldType === "number") {
       const numIdx = state.numericIndexes.get(fieldPath);
       if (numIdx) {
-        if (oldVal !== void 0 && oldVal !== null) numIdx.remove(docId, oldVal);
-        if (newVal !== void 0 && newVal !== null) numIdx.insert(docId, newVal);
+        if (oldVal !== void 0 && oldVal !== null) numIdx.remove(internalId, oldVal);
+        if (newVal !== void 0 && newVal !== null) numIdx.insert(internalId, newVal);
       }
     } else if (fieldType === "boolean") {
       const boolIdx = state.booleanIndexes.get(fieldPath);
       if (boolIdx) {
-        if (oldVal !== void 0 && oldVal !== null) boolIdx.remove(docId, oldVal);
-        if (newVal !== void 0 && newVal !== null) boolIdx.insert(docId, newVal);
+        if (oldVal !== void 0 && oldVal !== null) boolIdx.remove(internalId, oldVal);
+        if (newVal !== void 0 && newVal !== null) boolIdx.insert(internalId, newVal);
       }
     } else if (fieldType === "enum") {
       const enumIdx = state.enumIndexes.get(fieldPath);
       if (enumIdx) {
-        if (oldVal !== void 0 && oldVal !== null) enumIdx.remove(docId, oldVal);
-        if (newVal !== void 0 && newVal !== null) enumIdx.insert(docId, newVal);
+        if (oldVal !== void 0 && oldVal !== null) enumIdx.remove(internalId, oldVal);
+        if (newVal !== void 0 && newVal !== null) enumIdx.insert(internalId, newVal);
       }
     } else if (fieldType === "geopoint") {
       const geoIdx = state.geoIndexes.get(fieldPath);
       if (geoIdx) {
-        if (oldVal !== void 0 && oldVal !== null) geoIdx.remove(docId);
+        if (oldVal !== void 0 && oldVal !== null) geoIdx.remove(internalId);
         if (newVal !== void 0 && newVal !== null) {
           const geo = newVal;
-          geoIdx.insert(docId, geo.lat, geo.lon);
+          geoIdx.insert(internalId, geo.lat, geo.lon);
         }
       }
     } else if (fieldType === "number[]") {
       const numIdx = state.numericIndexes.get(fieldPath);
       if (numIdx) {
         if (oldVal !== void 0 && oldVal !== null) {
-          for (const n of oldVal) numIdx.remove(docId, n);
+          for (const n of oldVal) numIdx.remove(internalId, n);
         }
         if (newVal !== void 0 && newVal !== null) {
-          for (const n of newVal) numIdx.insert(docId, n);
+          for (const n of newVal) numIdx.insert(internalId, n);
         }
       }
     } else if (fieldType === "boolean[]") {
       const boolIdx = state.booleanIndexes.get(fieldPath);
       if (boolIdx) {
         if (oldVal !== void 0 && oldVal !== null) {
-          for (const b of oldVal) boolIdx.remove(docId, b);
+          for (const b of oldVal) boolIdx.remove(internalId, b);
         }
         if (newVal !== void 0 && newVal !== null) {
-          for (const b of newVal) boolIdx.insert(docId, b);
+          for (const b of newVal) boolIdx.insert(internalId, b);
         }
       }
     } else if (fieldType === "enum[]") {
       const enumIdx = state.enumIndexes.get(fieldPath);
       if (enumIdx) {
         if (oldVal !== void 0 && oldVal !== null) {
-          for (const e of oldVal) enumIdx.remove(docId, e);
-        }
-        if (newVal !== void 0 && newVal !== null) {
-          for (const e of newVal) enumIdx.insert(docId, e);
+          for (const e of oldVal) enumIdx.remove(internalId, e);
         }
-      }
-    } else if (parseVectorDimension(fieldType) !== null) {
-      const vecStore = state.vectorStores.get(fieldPath);
-      if (vecStore) {
-        vecStore.remove(docId);
         if (newVal !== void 0 && newVal !== null) {
-          const vec = newVal instanceof Float32Array ? newVal : new Float32Array(newVal);
-          vecStore.insert(docId, vec);
+          for (const e of newVal) enumIdx.insert(internalId, e);
         }
       }
     }
@@ -2543,39 +2468,26 @@ function updateFieldIndexOnly(state, docId, oldFields, newDoc, flatSchema) {
 }
 
 // src/filters/combinators.ts
-function applyAnd(sets) {
-  if (sets.length === 0) return /* @__PURE__ */ new Set();
-  if (sets.length === 1) return sets[0];
-  const sorted = [...sets].sort((a, b) => a.size - b.size);
-  let result = sorted[0];
-  for (let i = 1; i < sorted.length; i++) {
-    const next = sorted[i];
-    const intersection = /* @__PURE__ */ new Set();
-    for (const item of result) {
-      if (next.has(item)) intersection.add(item);
-    }
-    result = intersection;
-    if (result.size === 0) break;
+function applyAndBitset(bitsets) {
+  if (bitsets.length === 0) return new Uint32Array(0);
+  if (bitsets.length === 1) return bitsets[0];
+  let result = bitsets[0];
+  for (let i = 1; i < bitsets.length; i++) {
+    result = bitsetAnd(result, bitsets[i]);
   }
   return result;
 }
-function applyOr(sets) {
-  if (sets.length === 0) return /* @__PURE__ */ new Set();
-  if (sets.length === 1) return sets[0];
-  const result = /* @__PURE__ */ new Set();
-  for (const set of sets) {
-    for (const item of set) {
-      result.add(item);
-    }
+function applyOrBitset(bitsets) {
+  if (bitsets.length === 0) return new Uint32Array(0);
+  if (bitsets.length === 1) return bitsets[0];
+  let result = bitsets[0];
+  for (let i = 1; i < bitsets.length; i++) {
+    result = bitsetOr(result, bitsets[i]);
   }
   return result;
 }
-function applyNot(universe, excluded) {
-  const result = /* @__PURE__ */ new Set();
-  for (const item of universe) {
-    if (!excluded.has(item)) result.add(item);
-  }
-  return result;
+function applyNotBitset(excluded, capacity) {
+  return bitsetNot(excluded, capacity);
 }
 
 // src/filters/operators.ts
@@ -2586,251 +2498,176 @@ function convertToMeters(distance, unit) {
   if (unit === "mi") return distance * METERS_PER_MI;
   return distance;
 }
-function setDifference(a, b) {
-  const result = /* @__PURE__ */ new Set();
-  for (const item of a) {
-    if (!b.has(item)) result.add(item);
+function resolveBitset(bs) {
+  return typeof bs === "function" ? bs() : bs;
+}
+function scanToBitset(docIds, capacity, getValue, predicate) {
+  const resolved = resolveBitset(docIds);
+  const result = createBitSet(capacity);
+  for (let wi = 0; wi < resolved.length; wi++) {
+    let word = resolved[wi];
+    if (word === 0) continue;
+    const base = wi << 5;
+    while (word !== 0) {
+      const tz = Math.clz32(word & -word) ^ 31;
+      const id = base + tz;
+      if (predicate(getValue(id))) {
+        bitsetSet(result, id);
+      }
+      word &= word - 1;
+    }
   }
   return result;
 }
-function applyEq(value, docIds, getValue, fieldIndex) {
+function applyEqBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
   if (fieldIndex?.type === "numeric" && typeof value === "number") {
-    return fieldIndex.index.eq(value);
+    return fieldIndex.index.eqBitset(value, capacity);
   }
   if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
-    return value ? fieldIndex.index.getTrue() : fieldIndex.index.getFalse();
+    return value ? fieldIndex.index.getTrueBitset(capacity) : fieldIndex.index.getFalseBitset(capacity);
   }
   if (fieldIndex?.type === "enum" && typeof value === "string") {
-    return fieldIndex.index.getDocIds(value);
+    return fieldIndex.index.getDocIdsBitset(value, capacity);
   }
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    if (getValue(docId) === value) result.add(docId);
-  }
-  return result;
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => v === value);
 }
-function applyNe(value, docIds, getValue, fieldIndex) {
+function applyNeBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
   if (fieldIndex?.type === "numeric" && typeof value === "number") {
-    return setDifference(fieldIndex.index.allDocIds(), fieldIndex.index.eq(value));
+    const all = fieldIndex.index.allDocIdsBitset(capacity);
+    const eq = fieldIndex.index.eqBitset(value, capacity);
+    return bitsetAnd(all, bitsetNot(eq, capacity));
   }
   if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
-    return value ? fieldIndex.index.getFalse() : fieldIndex.index.getTrue();
+    return value ? fieldIndex.index.getFalseBitset(capacity) : fieldIndex.index.getTrueBitset(capacity);
   }
   if (fieldIndex?.type === "enum" && typeof value === "string") {
-    return setDifference(fieldIndex.index.allDocIds(), fieldIndex.index.getDocIds(value));
-  }
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (v !== void 0 && v !== null && v !== value) result.add(docId);
+    const all = fieldIndex.index.allDocIdsBitset(capacity);
+    const eq = fieldIndex.index.getDocIdsBitset(value, capacity);
+    return bitsetAnd(all, bitsetNot(eq, capacity));
   }
-  return result;
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => v !== void 0 && v !== null && v !== value);
 }
-function applyGt(value, docIds, getValue, fieldIndex) {
+function applyGtBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
   if (fieldIndex?.type === "numeric") {
-    return fieldIndex.index.gt(value);
-  }
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (typeof v === "number" && v > value) result.add(docId);
+    return fieldIndex.index.gtBitset(value, capacity);
   }
-  return result;
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v > value);
 }
-function applyLt(value, docIds, getValue, fieldIndex) {
+function applyLtBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
   if (fieldIndex?.type === "numeric") {
-    return fieldIndex.index.lt(value);
-  }
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (typeof v === "number" && v < value) result.add(docId);
+    return fieldIndex.index.ltBitset(value, capacity);
   }
-  return result;
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v < value);
 }
-function applyGte(value, docIds, getValue, fieldIndex) {
+function applyGteBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
   if (fieldIndex?.type === "numeric") {
-    return fieldIndex.index.gte(value);
+    return fieldIndex.index.gteBitset(value, capacity);
   }
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (typeof v === "number" && v >= value) result.add(docId);
-  }
-  return result;
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v >= value);
 }
-function applyLte(value, docIds, getValue, fieldIndex) {
+function applyLteBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
   if (fieldIndex?.type === "numeric") {
-    return fieldIndex.index.lte(value);
-  }
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (typeof v === "number" && v <= value) result.add(docId);
+    return fieldIndex.index.lteBitset(value, capacity);
   }
-  return result;
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v <= value);
 }
-function applyBetween(range, docIds, getValue, fieldIndex) {
+function applyBetweenBitset(range, allDocsBitset, capacity, getValue, fieldIndex) {
   if (fieldIndex?.type === "numeric") {
-    return fieldIndex.index.between(range[0], range[1]);
-  }
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (typeof v === "number" && v >= range[0] && v <= range[1]) result.add(docId);
+    return fieldIndex.index.betweenBitset(range[0], range[1], capacity);
   }
-  return result;
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v >= range[0] && v <= range[1]);
 }
-function applyIn(values, docIds, getValue, fieldIndex) {
+function applyInBitset(values, allDocsBitset, capacity, getValue, fieldIndex) {
   if (fieldIndex?.type === "enum") {
-    const result2 = /* @__PURE__ */ new Set();
-    for (const val of values) {
-      for (const docId of fieldIndex.index.getDocIds(val)) {
-        result2.add(docId);
-      }
-    }
-    return result2;
+    return fieldIndex.index.getDocIdsInBitset(values, capacity);
   }
   const valSet = new Set(values);
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (typeof v === "string" && valSet.has(v)) result.add(docId);
-  }
-  return result;
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "string" && valSet.has(v));
 }
-function applyNin(values, docIds, getValue, fieldIndex) {
+function applyNinBitset(values, allDocsBitset, capacity, getValue, fieldIndex) {
   if (fieldIndex?.type === "enum") {
-    const matched = /* @__PURE__ */ new Set();
-    for (const val of values) {
-      for (const docId of fieldIndex.index.getDocIds(val)) {
-        matched.add(docId);
-      }
-    }
-    return setDifference(fieldIndex.index.allDocIds(), matched);
+    const all = fieldIndex.index.allDocIdsBitset(capacity);
+    const matched = fieldIndex.index.getDocIdsInBitset(values, capacity);
+    return bitsetAnd(all, bitsetNot(matched, capacity));
   }
   const valSet = new Set(values);
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (v !== void 0 && v !== null && typeof v === "string" && !valSet.has(v)) result.add(docId);
-  }
-  return result;
+  return scanToBitset(
+    allDocsBitset,
+    capacity,
+    getValue,
+    (v) => v !== void 0 && v !== null && typeof v === "string" && !valSet.has(v)
+  );
 }
-function applyStartsWith(prefix, docIds, getValue) {
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (typeof v === "string" && v.startsWith(prefix)) result.add(docId);
-  }
-  return result;
+function applyStartsWithBitset(prefix, allDocsBitset, capacity, getValue) {
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "string" && v.startsWith(prefix));
 }
-function applyEndsWith(suffix, docIds, getValue) {
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (typeof v === "string" && v.endsWith(suffix)) result.add(docId);
-  }
-  return result;
+function applyEndsWithBitset(suffix, allDocsBitset, capacity, getValue) {
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "string" && v.endsWith(suffix));
 }
-function applyContainsAll(values, docIds, getValue) {
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (!Array.isArray(v)) continue;
+function applyContainsAllBitset(values, allDocsBitset, capacity, getValue) {
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
+    if (!Array.isArray(v)) return false;
     const arr = v;
-    let allFound = true;
     for (const val of values) {
-      if (!arr.includes(val)) {
-        allFound = false;
-        break;
-      }
+      if (!arr.includes(val)) return false;
     }
-    if (allFound) result.add(docId);
-  }
-  return result;
+    return true;
+  });
 }
-function applyMatchesAny(values, docIds, getValue) {
+function applyMatchesAnyBitset(values, allDocsBitset, capacity, getValue) {
   const valSet = new Set(values);
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (!Array.isArray(v)) continue;
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
+    if (!Array.isArray(v)) return false;
     for (const item of v) {
-      if (valSet.has(item)) {
-        result.add(docId);
-        break;
-      }
+      if (valSet.has(item)) return true;
     }
-  }
-  return result;
+    return false;
+  });
 }
-function applySize(sizeFilter, docIds, getValue) {
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (!Array.isArray(v)) continue;
-    if (matchesNumericComparison(v.length, sizeFilter)) result.add(docId);
-  }
-  return result;
+function applySizeBitset(sizeFilter, allDocsBitset, capacity, getValue) {
+  return scanToBitset(
+    allDocsBitset,
+    capacity,
+    getValue,
+    (v) => Array.isArray(v) && matchesNumericComparison(v.length, sizeFilter)
+  );
 }
-function applyExists(docIds, getValue) {
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (v !== void 0 && v !== null) result.add(docId);
-  }
-  return result;
+function applyExistsBitset(allDocsBitset, capacity, getValue) {
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => v !== void 0 && v !== null);
 }
-function applyNotExists(docIds, getValue) {
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (v === void 0 || v === null) result.add(docId);
-  }
-  return result;
+function applyNotExistsBitset(allDocsBitset, capacity, getValue) {
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => v === void 0 || v === null);
 }
-function applyIsEmpty(docIds, getValue) {
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (v === void 0 || v === null) {
-      result.add(docId);
-      continue;
-    }
-    if (typeof v === "string" && v === "") {
-      result.add(docId);
-      continue;
-    }
-    if (Array.isArray(v) && v.length === 0) {
-      result.add(docId);
-    }
-  }
-  return result;
+function applyIsEmptyBitset(allDocsBitset, capacity, getValue) {
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
+    if (v === void 0 || v === null) return true;
+    if (typeof v === "string" && v === "") return true;
+    if (Array.isArray(v) && v.length === 0) return true;
+    return false;
+  });
 }
-function applyIsNotEmpty(docIds, getValue) {
-  const result = /* @__PURE__ */ new Set();
-  for (const docId of docIds) {
-    const v = getValue(docId);
-    if (v === void 0 || v === null) continue;
-    if (typeof v === "string" && v === "") continue;
-    if (Array.isArray(v) && v.length === 0) continue;
-    result.add(docId);
-  }
-  return result;
+function applyIsNotEmptyBitset(allDocsBitset, capacity, getValue) {
+  return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
+    if (v === void 0 || v === null) return false;
+    if (typeof v === "string" && v === "") return false;
+    if (Array.isArray(v) && v.length === 0) return false;
+    return true;
+  });
 }
-function applyGeoRadius(filter, geoIndex) {
+function applyGeoRadiusBitset(filter, geoIndex, capacity) {
   const distanceMeters = convertToMeters(filter.distance, filter.unit);
-  return geoIndex.radiusQuery(
+  const setResult = geoIndex.radiusQuery(
     filter.lat,
     filter.lon,
     distanceMeters,
     filter.inside ?? true,
     filter.highPrecision ?? false
   );
+  return bitsetFromSet(setResult, capacity);
 }
-function applyGeoPolygon(filter, geoIndex) {
-  return geoIndex.polygonQuery(filter.points, filter.inside ?? true);
+function applyGeoPolygonBitset(filter, geoIndex, capacity) {
+  const setResult = geoIndex.polygonQuery(filter.points, filter.inside ?? true);
+  return bitsetFromSet(setResult, capacity);
 }
 function matchesNumericComparison(value, filter) {
   if (filter.eq !== void 0 && value !== filter.eq) return false;
@@ -2845,98 +2682,119 @@ function matchesNumericComparison(value, filter) {
 
 // src/filters/evaluator.ts
 function evaluateFilters(expression, context) {
-  const sets = [];
+  const bitsets = [];
   if (expression.fields) {
     for (const [fieldPath, filter] of Object.entries(expression.fields)) {
-      sets.push(evaluateFieldFilter(fieldPath, filter, context));
+      bitsets.push(evaluateFieldFilter(fieldPath, filter, context));
     }
   }
   if (expression.and?.length) {
-    const andSets = expression.and.map((expr) => evaluateFilters(expr, context));
-    sets.push(applyAnd(andSets));
+    const andBitsets = expression.and.map((expr) => evaluateFilters(expr, context));
+    bitsets.push(applyAndBitset(andBitsets));
   }
   if (expression.or?.length) {
-    const orSets = expression.or.map((expr) => evaluateFilters(expr, context));
-    sets.push(applyOr(orSets));
+    const orBitsets = expression.or.map((expr) => evaluateFilters(expr, context));
+    bitsets.push(applyOrBitset(orBitsets));
   }
   if (expression.not) {
     const excluded = evaluateFilters(expression.not, context);
-    sets.push(applyNot(context.allDocIds, excluded));
+    const universe = context.allDocIdsBitset;
+    bitsets.push(applyAndBitset([universe, applyNotBitset(excluded, context.capacity)]));
   }
-  if (sets.length === 0) return new Set(context.allDocIds);
-  if (sets.length === 1) return sets[0];
-  return applyAnd(sets);
+  if (bitsets.length === 0) return context.allDocIdsBitset;
+  if (bitsets.length === 1) return bitsets[0];
+  return applyAndBitset(bitsets);
 }
 function evaluateFieldFilter(fieldPath, filter, context) {
   const fieldIndex = context.fieldIndexes[fieldPath];
-  const getValue = (docId) => context.getFieldValue(docId, fieldPath);
+  const getValue = (internalId) => context.getFieldValue(internalId, fieldPath);
   const f = filter;
-  const sets = [];
+  const bitsets = [];
+  const { capacity } = context;
+  let allDocsBitset = null;
+  const getAllDocsBitset = () => {
+    if (allDocsBitset === null) {
+      allDocsBitset = context.allDocIdsBitset;
+    }
+    return allDocsBitset;
+  };
   if ("radius" in f && f.radius) {
     const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
-    sets.push(applyGeoRadius(filter.radius, geoIndex));
+    bitsets.push(applyGeoRadiusBitset(filter.radius, geoIndex, capacity));
   }
   if ("polygon" in f && f.polygon) {
     const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
-    sets.push(applyGeoPolygon(filter.polygon, geoIndex));
+    bitsets.push(applyGeoPolygonBitset(filter.polygon, geoIndex, capacity));
   }
   if ("eq" in f && f.eq !== void 0) {
-    sets.push(applyEq(f.eq, context.allDocIds, getValue, fieldIndex));
+    bitsets.push(applyEqBitset(f.eq, getAllDocsBitset, capacity, getValue, fieldIndex));
   }
   if ("ne" in f && f.ne !== void 0) {
-    sets.push(applyNe(f.ne, context.allDocIds, getValue, fieldIndex));
+    bitsets.push(applyNeBitset(f.ne, getAllDocsBitset, capacity, getValue, fieldIndex));
   }
   if ("gt" in f && f.gt !== void 0) {
-    sets.push(applyGt(f.gt, context.allDocIds, getValue, fieldIndex));
+    bitsets.push(applyGtBitset(f.gt, getAllDocsBitset, capacity, getValue, fieldIndex));
   }
   if ("lt" in f && f.lt !== void 0) {
-    sets.push(applyLt(f.lt, context.allDocIds, getValue, fieldIndex));
+    bitsets.push(applyLtBitset(f.lt, getAllDocsBitset, capacity, getValue, fieldIndex));
   }
   if ("gte" in f && f.gte !== void 0) {
-    sets.push(applyGte(f.gte, context.allDocIds, getValue, fieldIndex));
+    bitsets.push(applyGteBitset(f.gte, getAllDocsBitset, capacity, getValue, fieldIndex));
   }
   if ("lte" in f && f.lte !== void 0) {
-    sets.push(applyLte(f.lte, context.allDocIds, getValue, fieldIndex));
+    bitsets.push(applyLteBitset(f.lte, getAllDocsBitset, capacity, getValue, fieldIndex));
   }
   if ("between" in f && f.between !== void 0) {
-    sets.push(applyBetween(f.between, context.allDocIds, getValue, fieldIndex));
+    bitsets.push(applyBetweenBitset(f.between, getAllDocsBitset, capacity, getValue, fieldIndex));
   }
   if ("in" in f && f.in !== void 0) {
-    sets.push(applyIn(f.in, context.allDocIds, getValue, fieldIndex));
+    bitsets.push(applyInBitset(f.in, getAllDocsBitset, capacity, getValue, fieldIndex));
   }
   if ("nin" in f && f.nin !== void 0) {
-    sets.push(applyNin(f.nin, context.allDocIds, getValue, fieldIndex));
+    bitsets.push(applyNinBitset(f.nin, getAllDocsBitset, capacity, getValue, fieldIndex));
   }
   if ("startsWith" in f && f.startsWith !== void 0) {
-    sets.push(applyStartsWith(f.startsWith, context.allDocIds, getValue));
+    bitsets.push(applyStartsWithBitset(f.startsWith, getAllDocsBitset, capacity, getValue));
   }
   if ("endsWith" in f && f.endsWith !== void 0) {
-    sets.push(applyEndsWith(f.endsWith, context.allDocIds, getValue));
+    bitsets.push(applyEndsWithBitset(f.endsWith, getAllDocsBitset, capacity, getValue));
   }
   if ("containsAll" in f && f.containsAll !== void 0) {
-    sets.push(applyContainsAll(f.containsAll, context.allDocIds, getValue));
+    bitsets.push(
+      applyContainsAllBitset(f.containsAll, getAllDocsBitset, capacity, getValue)
+    );
   }
   if ("matchesAny" in f && f.matchesAny !== void 0) {
-    sets.push(applyMatchesAny(f.matchesAny, context.allDocIds, getValue));
+    bitsets.push(
+      applyMatchesAnyBitset(f.matchesAny, getAllDocsBitset, capacity, getValue)
+    );
   }
   if ("size" in f && f.size !== void 0) {
-    sets.push(applySize(f.size, context.allDocIds, getValue));
+    bitsets.push(applySizeBitset(f.size, getAllDocsBitset, capacity, getValue));
   }
   if ("exists" in f && f.exists !== void 0) {
-    sets.push(f.exists ? applyExists(context.allDocIds, getValue) : applyNotExists(context.allDocIds, getValue));
+    bitsets.push(
+      f.exists ? applyExistsBitset(getAllDocsBitset, capacity, getValue) : applyNotExistsBitset(getAllDocsBitset, capacity, getValue)
+    );
   }
   if ("notExists" in f && f.notExists !== void 0) {
-    sets.push(f.notExists ? applyNotExists(context.allDocIds, getValue) : applyExists(context.allDocIds, getValue));
+    bitsets.push(
+      f.notExists ? applyNotExistsBitset(getAllDocsBitset, capacity, getValue) : applyExistsBitset(getAllDocsBitset, capacity, getValue)
+    );
   }
   if ("isEmpty" in f && f.isEmpty !== void 0) {
-    sets.push(f.isEmpty ? applyIsEmpty(context.allDocIds, getValue) : applyIsNotEmpty(context.allDocIds, getValue));
+    bitsets.push(
+      f.isEmpty ? applyIsEmptyBitset(getAllDocsBitset, capacity, getValue) : applyIsNotEmptyBitset(getAllDocsBitset, capacity, getValue)
+    );
   }
   if ("isNotEmpty" in f && f.isNotEmpty !== void 0) {
-    sets.push(f.isNotEmpty ? applyIsNotEmpty(context.allDocIds, getValue) : applyIsEmpty(context.allDocIds, getValue));
+    bitsets.push(
+      f.isNotEmpty ? applyIsNotEmptyBitset(getAllDocsBitset, capacity, getValue) : applyIsEmptyBitset(getAllDocsBitset, capacity, getValue)
+    );
   }
-  if (sets.length === 0) return new Set(context.allDocIds);
-  if (sets.length === 1) return sets[0];
-  return applyAnd(sets);
+  if (bitsets.length === 0) return context.allDocIdsBitset;
+  if (bitsets.length === 1) return bitsets[0];
+  return applyAndBitset(bitsets);
 }
 function resolveGeoIndex(fieldPath, fieldIndex) {
   if (fieldIndex?.type !== "geopoint") {
@@ -2984,7 +2842,8 @@ function searchFulltext(state, params) {
     bm25Params,
     globalStats,
     maxResults,
-    termMatch
+    termMatch,
+    filterBitset
   } = params;
   if (queryTokens.length === 0) {
     return { scored: [], totalMatched: 0 };
@@ -2997,6 +2856,7 @@ function searchFulltext(state, params) {
   const fieldLengthCache = /* @__PURE__ */ new Map();
   const useIntersection = termMatch === "all" && queryTokens.length > 1;
   const fieldNames = state.fieldNameTable.names;
+  const resolver = state.docStore.resolver();
   if (useIntersection) {
     const resolved = [];
     for (const qt of queryTokens) {
@@ -3020,31 +2880,33 @@ function searchFulltext(state, params) {
         const list = match.postingList;
         const hasDeleted = list.deletedDocs.size > 0;
         for (let pi = 0; pi < list.length; pi++) {
-          const docId = list.docIds[pi];
-          if (hasDeleted && list.deletedDocs.has(docId)) continue;
-          if (tokenIndex > 0 && !docScores.has(docId)) continue;
+          const internalId = list.docIds[pi];
+          if (hasDeleted && list.deletedDocs.has(internalId)) continue;
+          if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
+          if (tokenIndex > 0 && !docScores.has(internalId)) continue;
           const fieldName = fieldNames[list.fieldNameIndices[pi]];
           if (fields && !fields.includes(fieldName)) continue;
           const termFrequency = list.termFrequencies[pi];
           const fieldBoost = boost?.[fieldName] ?? 1;
           const avgLen = avgFieldLengths[fieldName] ?? 1;
-          let cachedLengths = fieldLengthCache.get(docId);
+          let cachedLengths = fieldLengthCache.get(internalId);
           if (cachedLengths === void 0) {
-            const storedDoc = state.docStore.get(docId);
+            const externalId = resolver.toExternal(internalId);
+            const storedDoc = externalId !== void 0 ? state.docStore.get(externalId) : void 0;
             cachedLengths = storedDoc?.fieldLengths ?? null;
-            fieldLengthCache.set(docId, cachedLengths);
+            fieldLengthCache.set(internalId, cachedLengths);
           }
           const actualFieldLength = cachedLengths?.[fieldName] ?? avgLen;
           let termScore = scoreFn(termFrequency, match.docFreq, totalDocs, actualFieldLength, avgLen, bm25Params);
           termScore *= fieldBoost;
-          const existing = docScores.get(docId);
+          const existing = docScores.get(internalId);
           if (existing) {
             existing.score += termScore;
             existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
             existing.fieldLengths[fieldName] = actualFieldLength;
             existing.idf[match.token] = match.idf;
           } else {
-            docScores.set(docId, {
+            docScores.set(internalId, {
               score: termScore,
               termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
               fieldLengths: { [fieldName]: actualFieldLength },
@@ -3066,30 +2928,32 @@ function searchFulltext(state, params) {
         const list = match.postingList;
         const hasDeleted = list.deletedDocs.size > 0;
         for (let pi = 0; pi < list.length; pi++) {
-          const docId = list.docIds[pi];
-          if (hasDeleted && list.deletedDocs.has(docId)) continue;
+          const internalId = list.docIds[pi];
+          if (hasDeleted && list.deletedDocs.has(internalId)) continue;
+          if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
           const fieldName = fieldNames[list.fieldNameIndices[pi]];
           if (fields && !fields.includes(fieldName)) continue;
           const termFrequency = list.termFrequencies[pi];
           const fieldBoost = boost?.[fieldName] ?? 1;
           const avgLen = avgFieldLengths[fieldName] ?? 1;
-          let cachedLengths = fieldLengthCache.get(docId);
+          let cachedLengths = fieldLengthCache.get(internalId);
           if (cachedLengths === void 0) {
-            const storedDoc = state.docStore.get(docId);
+            const externalId = resolver.toExternal(internalId);
+            const storedDoc = externalId !== void 0 ? state.docStore.get(externalId) : void 0;
             cachedLengths = storedDoc?.fieldLengths ?? null;
-            fieldLengthCache.set(docId, cachedLengths);
+            fieldLengthCache.set(internalId, cachedLengths);
           }
           const actualFieldLength = cachedLengths?.[fieldName] ?? avgLen;
           let termScore = scoreFn(termFrequency, docFreq, totalDocs, actualFieldLength, avgLen, bm25Params);
           termScore *= fieldBoost;
-          const existing = docScores.get(docId);
+          const existing = docScores.get(internalId);
           if (existing) {
             existing.score += termScore;
             existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
             existing.fieldLengths[fieldName] = actualFieldLength;
             existing.idf[match.token] = idf;
           } else {
-            docScores.set(docId, {
+            docScores.set(internalId, {
               score: termScore,
               termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
               fieldLengths: { [fieldName]: actualFieldLength },
@@ -3102,33 +2966,35 @@ function searchFulltext(state, params) {
   }
   const totalMatched = docScores.size;
   const k = maxResults !== void 0 && maxResults > 0 ? maxResults : DEFAULT_MAX_RESULTS;
-  const scored = topKFromMap(docScores, Math.min(k, totalMatched));
+  const scored = topKFromMap(docScores, Math.min(k, totalMatched), resolver);
   return { scored, totalMatched };
 }
-function topKFromMap(docScores, k) {
+function topKFromMap(docScores, k, resolver) {
   if (k <= 0) return [];
   const heap = [];
-  for (const [docId, data] of docScores) {
+  for (const [internalId, data] of docScores) {
     if (heap.length < k) {
-      heap.push({ docId, score: data.score });
+      heap.push({ internalId, score: data.score });
       if (heap.length === k) buildMinHeap(heap);
     } else if (data.score > heap[0].score) {
-      heap[0] = { docId, score: data.score };
+      heap[0] = { internalId, score: data.score };
       siftDown(heap, 0);
     }
   }
   heap.sort((a, b) => b.score - a.score);
-  const result = new Array(heap.length);
+  const result = [];
   for (let i = 0; i < heap.length; i++) {
-    const data = docScores.get(heap[i].docId);
+    const data = docScores.get(heap[i].internalId);
     if (!data) continue;
-    result[i] = {
-      docId: heap[i].docId,
+    const externalId = resolver.toExternal(heap[i].internalId);
+    if (externalId === void 0) continue;
+    result.push({
+      docId: externalId,
       score: data.score,
       termFrequencies: data.termFrequencies,
       fieldLengths: data.fieldLengths,
       idf: data.idf
-    };
+    });
   }
   return result;
 }
@@ -3152,26 +3018,10 @@ function siftDown(heap, idx) {
     idx = smallest;
   }
 }
-function searchVector(state, params) {
-  const { field, value, k, similarity = 0, metric = "cosine", filterDocIds, efSearch } = params;
-  const vecStore = state.vectorStores.get(field);
-  if (!vecStore) {
-    return { scored: [], totalMatched: 0 };
-  }
-  if (value.length !== vecStore.dimension) {
-    throw new NarsilError(
-      ErrorCodes.SEARCH_INVALID_VECTOR_SIZE,
-      `Query vector dimension ${value.length} does not match field "${field}" dimension ${vecStore.dimension}`,
-      { field, expected: vecStore.dimension, received: value.length }
-    );
-  }
-  const queryVec = new Float32Array(value);
-  const results = vecStore.search(queryVec, k, metric, similarity, filterDocIds, efSearch);
-  return { scored: results, totalMatched: results.length };
-}
 function buildFilterContext(state, schema) {
   const flat = getFlatSchema(state, schema);
   const fieldIndexes = {};
+  const capacity = state.docStore.internalIdCapacity();
   for (const [fieldPath, fieldType] of Object.entries(flat)) {
     if (fieldType === "number" || fieldType === "number[]") {
       const numIdx = state.numericIndexes.get(fieldPath);
@@ -3185,7 +3035,14 @@ function buildFilterContext(state, schema) {
             lt: (v) => numIdx.queryLt(v),
             lte: (v) => numIdx.queryLte(v),
             between: (min, max) => numIdx.queryBetween(min, max),
-            allDocIds: () => numIdx.getAllDocIds()
+            allDocIds: () => numIdx.getAllDocIds(),
+            eqBitset: (v, cap) => numIdx.queryEqBitset(v, cap),
+            gtBitset: (v, cap) => numIdx.queryGtBitset(v, cap),
+            gteBitset: (v, cap) => numIdx.queryGteBitset(v, cap),
+            ltBitset: (v, cap) => numIdx.queryLtBitset(v, cap),
+            lteBitset: (v, cap) => numIdx.queryLteBitset(v, cap),
+            betweenBitset: (min, max, cap) => numIdx.queryBetweenBitset(min, max, cap),
+            allDocIdsBitset: (cap) => numIdx.getAllDocIdsBitset(cap)
           }
         };
       }
@@ -3197,7 +3054,10 @@ function buildFilterContext(state, schema) {
           index: {
             getTrue: () => boolIdx.queryEq(true),
             getFalse: () => boolIdx.queryEq(false),
-            allDocIds: () => boolIdx.getAllDocIds()
+            allDocIds: () => boolIdx.getAllDocIds(),
+            getTrueBitset: (cap) => boolIdx.queryEqBitset(true, cap),
+            getFalseBitset: (cap) => boolIdx.queryEqBitset(false, cap),
+            allDocIdsBitset: (cap) => boolIdx.getAllDocIdsBitset(cap)
           }
         };
       }
@@ -3208,7 +3068,10 @@ function buildFilterContext(state, schema) {
           type: "enum",
           index: {
             getDocIds: (v) => enumIdx.queryEq(v),
-            allDocIds: () => enumIdx.getAllDocIds()
+            allDocIds: () => enumIdx.getAllDocIds(),
+            getDocIdsBitset: (v, cap) => enumIdx.queryEqBitset(v, cap),
+            getDocIdsInBitset: (values, cap) => enumIdx.queryInBitset(values, cap),
+            allDocIdsBitset: (cap) => enumIdx.getAllDocIdsBitset(cap)
           }
         };
       }
@@ -3223,18 +3086,47 @@ function buildFilterContext(state, schema) {
     }
   }
   let cachedAllDocIds = null;
+  let cachedAllDocIdsBitset = null;
   return {
     fieldIndexes,
-    getFieldValue: (docId, fieldPath) => getFieldValueForDoc(state.docStore, docId, fieldPath),
+    getFieldValue: (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath),
     get allDocIds() {
       if (!cachedAllDocIds) {
-        cachedAllDocIds = getAllDocIds(state.docStore);
+        cachedAllDocIds = getAllInternalDocIds(state.docStore);
       }
       return cachedAllDocIds;
+    },
+    capacity,
+    get allDocIdsBitset() {
+      if (!cachedAllDocIdsBitset) {
+        cachedAllDocIdsBitset = bitsetFromSet(this.allDocIds, capacity);
+      }
+      return cachedAllDocIdsBitset;
     }
   };
 }
 function applyPartitionFilters(state, filters, schema) {
+  const context = buildFilterContext(state, schema);
+  const resultBitset = evaluateFilters(filters, context);
+  const resolver = state.docStore.resolver();
+  const externalResult = /* @__PURE__ */ new Set();
+  for (let wi = 0; wi < resultBitset.length; wi++) {
+    let word = resultBitset[wi];
+    if (word === 0) continue;
+    const base = wi << 5;
+    while (word !== 0) {
+      const tz = Math.clz32(word & -word) ^ 31;
+      const internalId = base + tz;
+      const externalId = resolver.toExternal(internalId);
+      if (externalId !== void 0) {
+        externalResult.add(externalId);
+      }
+      word &= word - 1;
+    }
+  }
+  return externalResult;
+}
+function applyPartitionFiltersBitset(state, filters, schema) {
   const context = buildFilterContext(state, schema);
   return evaluateFilters(filters, context);
 }
@@ -3304,53 +3196,66 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
   const serializedDocs = /* @__PURE__ */ Object.create(null);
   for (const [docId, stored] of state.docStore.all()) {
     serializedDocs[docId] = {
-      fields: structuredClone(stored.fields),
+      fields: stored.fields,
       fieldLengths: { ...stored.fieldLengths }
     };
   }
-  const serializedInverted = state.invertedIdx.serialize();
+  const resolver = state.docStore.resolver();
+  const fieldNames = state.fieldNameTable.names;
   const serializedInvertedIndex = /* @__PURE__ */ Object.create(null);
-  for (const [token, list] of Object.entries(serializedInverted)) {
-    serializedInvertedIndex[token] = {
-      docFrequency: list.docFrequency,
-      postings: list.postings.map((p) => ({
-        docId: p.docId,
-        termFrequency: p.termFrequency,
-        field: p.fieldName,
-        positions: [...p.positions]
-      }))
-    };
+  for (const token of state.invertedIdx.tokens()) {
+    const list = state.invertedIdx.lookup(token);
+    if (!list) continue;
+    const hasDeleted = list.deletedDocs.size > 0;
+    const postings = [];
+    for (let i = 0; i < list.length; i++) {
+      if (hasDeleted && list.deletedDocs.has(list.docIds[i])) continue;
+      const externalId = resolver.toExternal(list.docIds[i]);
+      if (externalId === void 0) continue;
+      postings.push({
+        docId: externalId,
+        termFrequency: list.termFrequencies[i],
+        field: fieldNames[list.fieldNameIndices[i]],
+        positions: list.positions ? list.positions[i] : []
+      });
+    }
+    if (postings.length > 0) {
+      serializedInvertedIndex[token] = { docFrequency: list.docIdSet.size, postings };
+    }
   }
   const serializedNumeric = {};
   for (const [path, idx] of state.numericIndexes) {
-    serializedNumeric[path] = idx.serialize();
+    const raw = idx.serialize();
+    serializedNumeric[path] = raw.map((e) => {
+      const externalId = resolver.toExternal(e.docId);
+      return { value: e.value, docId: externalId ?? "" };
+    });
   }
   const serializedBoolean = {};
   for (const [path, idx] of state.booleanIndexes) {
-    serializedBoolean[path] = idx.serialize();
+    const raw = idx.serialize();
+    serializedBoolean[path] = {
+      trueDocs: raw.trueDocs.map((id) => resolver.toExternal(id) ?? "").filter((id) => id !== ""),
+      falseDocs: raw.falseDocs.map((id) => resolver.toExternal(id) ?? "").filter((id) => id !== "")
+    };
   }
   const serializedEnum = {};
   for (const [path, idx] of state.enumIndexes) {
-    serializedEnum[path] = idx.serialize();
+    const raw = idx.serialize();
+    const converted = /* @__PURE__ */ Object.create(null);
+    for (const [value, internalIds] of Object.entries(raw)) {
+      converted[value] = internalIds.map((id) => resolver.toExternal(id) ?? "").filter((id) => id !== "");
+    }
+    serializedEnum[path] = converted;
   }
   const serializedGeo = {};
   for (const [path, idx] of state.geoIndexes) {
-    serializedGeo[path] = idx.serialize();
-  }
-  const serializedVectors = {};
-  for (const [path, store] of state.vectorStores) {
-    const vectors = [];
-    for (const [, entry] of store.entries()) {
-      vectors.push({
-        docId: entry.docId,
-        vector: Array.from(entry.vector)
-      });
-    }
-    serializedVectors[path] = {
-      dimension: store.dimension,
-      vectors,
-      hnswGraph: store.serializeHNSW()
-    };
+    const raw = idx.serialize();
+    serializedGeo[path] = raw.map((e) => ({
+      lat: e.lat,
+      lon: e.lon,
+      docId: resolver.toExternal(e.docId) ?? ""
+    }));
   }
   const serializedStats = state.stats.serialize();
   return {
@@ -3369,7 +3274,6 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
       enum: serializedEnum,
       geopoint: serializedGeo
     },
-    vectorData: serializedVectors,
     statistics: serializedStats
   };
 }
@@ -3382,10 +3286,11 @@ function serializePartitionToWirePayloadV2(state, partitionId, indexName, totalP
   const wireDocs = /* @__PURE__ */ Object.create(null);
   for (const [docId, stored] of state.docStore.all()) {
     wireDocs[docId] = {
-      fields: structuredClone(stored.fields),
+      fields: stored.fields,
       field_lengths: { ...stored.fieldLengths }
     };
   }
+  const resolver = state.docStore.resolver();
   const fieldNames = [...state.fieldNameTable.names];
   const wireEntries = /* @__PURE__ */ Object.create(null);
   for (const token of state.invertedIdx.tokens()) {
@@ -3394,57 +3299,57 @@ function serializePartitionToWirePayloadV2(state, partitionId, indexName, totalP
     const hasDeleted = list.deletedDocs.size > 0;
     const ids = [];
     const tf = [];
-    const fi = [];
+    const fi = new Uint8Array(list.length);
     const pos = list.positions ? [] : null;
+    let writeIdx = 0;
     for (let i = 0; i < list.length; i++) {
       if (hasDeleted && list.deletedDocs.has(list.docIds[i])) continue;
-      ids.push(list.docIds[i]);
+      const externalId = resolver.toExternal(list.docIds[i]);
+      if (externalId === void 0) continue;
+      ids.push(externalId);
       tf.push(list.termFrequencies[i]);
-      fi.push(list.fieldNameIndices[i]);
+      fi[writeIdx] = list.fieldNameIndices[i];
+      writeIdx++;
       if (pos && list.positions) {
         pos.push(list.positions[i]);
       }
     }
     if (ids.length > 0) {
-      wireEntries[token] = { df: list.docIdSet.size, ids, tf, fi, pos };
+      const finalFi = writeIdx < list.length ? fi.subarray(0, writeIdx) : fi;
+      wireEntries[token] = { df: list.docIdSet.size, ids, tf, fi: finalFi, pos };
     }
   }
   const wireNumeric = {};
   for (const [path, idx] of state.numericIndexes) {
-    wireNumeric[path] = idx.serialize().map((e) => ({ value: e.value, doc_id: e.docId }));
+    wireNumeric[path] = idx.serialize().map((e) => ({
+      value: e.value,
+      doc_id: resolver.toExternal(e.docId) ?? ""
+    }));
   }
   const wireBoolean = {};
   for (const [path, idx] of state.booleanIndexes) {
     const s = idx.serialize();
-    wireBoolean[path] = { true_docs: s.trueDocs, false_docs: s.falseDocs };
+    wireBoolean[path] = {
+      true_docs: s.trueDocs.map((id) => resolver.toExternal(id) ?? "").filter((id) => id !== ""),
+      false_docs: s.falseDocs.map((id) => resolver.toExternal(id) ?? "").filter((id) => id !== "")
+    };
   }
   const wireEnum = {};
   for (const [path, idx] of state.enumIndexes) {
-    wireEnum[path] = idx.serialize();
+    const raw = idx.serialize();
+    const converted = /* @__PURE__ */ Object.create(null);
+    for (const [value, internalIds] of Object.entries(raw)) {
+      converted[value] = internalIds.map((id) => resolver.toExternal(id) ?? "").filter((id) => id !== "");
+    }
+    wireEnum[path] = converted;
   }
   const wireGeo = {};
   for (const [path, idx] of state.geoIndexes) {
-    wireGeo[path] = idx.serialize().map((e) => ({ lat: e.lat, lon: e.lon, doc_id: e.docId }));
-  }
-  const wireVectors = {};
-  for (const [path, store] of state.vectorStores) {
-    const vectors = [];
-    for (const [, entry] of store.entries()) {
-      vectors.push({ doc_id: entry.docId, vector: Array.from(entry.vector) });
-    }
-    const hnswData = store.serializeHNSW();
-    wireVectors[path] = {
-      dimension: store.dimension,
-      vectors,
-      hnsw_graph: hnswData ? {
-        entry_point: hnswData.entryPoint,
-        max_layer: hnswData.maxLayer,
-        m: hnswData.m,
-        ef_construction: hnswData.efConstruction,
-        metric: hnswData.metric,
-        nodes: hnswData.nodes
-      } : null
-    };
+    wireGeo[path] = idx.serialize().map((e) => ({
+      lat: e.lat,
+      lon: e.lon,
+      doc_id: resolver.toExternal(e.docId) ?? ""
+    }));
   }
   const serializedStats = state.stats.serialize();
   return {
@@ -3467,7 +3372,6 @@ function serializePartitionToWirePayloadV2(state, partitionId, indexName, totalP
       enum: wireEnum,
       geopoint: wireGeo
     },
-    vector_data: wireVectors,
     statistics: {
       total_documents: serializedStats.totalDocuments,
       total_field_lengths: serializedStats.totalFieldLengths,
@@ -3478,6 +3382,12 @@ function serializePartitionToWirePayloadV2(state, partitionId, indexName, totalP
 }
 function deserializePartition(state, data, clearFn, schema) {
   clearFn();
+  const docsData = /* @__PURE__ */ Object.create(null);
+  for (const [docId, doc] of Object.entries(data.documents)) {
+    docsData[docId] = { fields: doc.fields, fieldLengths: doc.fieldLengths };
+  }
+  state.docStore.deserialize(docsData);
+  const resolver = state.docStore.resolver();
   const invertedData = /* @__PURE__ */ Object.create(null);
   for (const [token, list] of Object.entries(data.invertedIndex)) {
     invertedData[token] = {
@@ -3490,42 +3400,42 @@ function deserializePartition(state, data, clearFn, schema) {
       }))
     };
   }
-  state.invertedIdx.deserialize(invertedData);
-  const docsData = /* @__PURE__ */ Object.create(null);
-  for (const [docId, doc] of Object.entries(data.documents)) {
-    docsData[docId] = { fields: doc.fields, fieldLengths: doc.fieldLengths };
-  }
-  state.docStore.deserialize(docsData);
+  state.invertedIdx.deserialize(invertedData, resolver);
   for (const [path, entries] of Object.entries(data.fieldIndexes.numeric)) {
     const idx = createNumericIndex();
-    idx.deserialize(entries);
+    const converted = entries.map((e) => {
+      const internalId = resolver.toInternal(e.docId);
+      return { value: e.value, docId: internalId ?? -1 };
+    }).filter((e) => e.docId !== -1);
+    idx.deserialize(converted);
     state.numericIndexes.set(path, idx);
   }
   for (const [path, serialized] of Object.entries(data.fieldIndexes.boolean)) {
     const idx = createBooleanIndex();
-    idx.deserialize(serialized);
+    const trueDocs = serialized.trueDocs.map((id) => resolver.toInternal(id)).filter((id) => id !== void 0);
+    const falseDocs = serialized.falseDocs.map((id) => resolver.toInternal(id)).filter((id) => id !== void 0);
+    idx.deserialize({ trueDocs, falseDocs });
     state.booleanIndexes.set(path, idx);
   }
   for (const [path, serialized] of Object.entries(data.fieldIndexes.enum)) {
     const idx = createEnumIndex();
-    idx.deserialize(serialized);
+    const converted = /* @__PURE__ */ Object.create(null);
+    for (const [value, docIds] of Object.entries(serialized)) {
+      converted[value] = docIds.map((id) => resolver.toInternal(id)).filter((id) => id !== void 0);
+    }
+    idx.deserialize(converted);
     state.enumIndexes.set(path, idx);
   }
   for (const [path, entries] of Object.entries(data.fieldIndexes.geopoint)) {
     const geoIdx = createGeoIndex();
-    geoIdx.deserialize(entries);
+    const converted = entries.map((e) => ({
+      lat: e.lat,
+      lon: e.lon,
+      docId: resolver.toInternal(e.docId) ?? -1
+    })).filter((e) => e.docId !== -1);
+    geoIdx.deserialize(converted);
     state.geoIndexes.set(path, geoIdx);
   }
-  for (const [path, vecData] of Object.entries(data.vectorData)) {
-    const store = createVectorSearchEngine(vecData.dimension);
-    for (const entry of vecData.vectors) {
-      store.insert(entry.docId, new Float32Array(entry.vector));
-    }
-    if (vecData.hnswGraph) {
-      store.deserializeHNSW(vecData.hnswGraph);
-    }
-    state.vectorStores.set(path, store);
-  }
   state.stats.deserialize(data.statistics);
   const flatSchema = getFlatSchema(state, schema);
   const serializedFields = data.schema;
@@ -3544,12 +3454,7 @@ function deserializePartition(state, data, clearFn, schema) {
 }
 
 // src/core/partition/index.ts
-function createPartitionIndex(partitionId, trackPositions = true, vectorPromotionConfig) {
-  const vectorPromoter = vectorPromotionConfig ? createVectorPromoter({
-    promotionThreshold: vectorPromotionConfig.threshold,
-    hnswConfig: vectorPromotionConfig.hnswConfig,
-    workerStrategy: vectorPromotionConfig.workerStrategy
-  }) : null;
+function createPartitionIndex(partitionId, trackPositions = true) {
   const fieldNameTable = { names: [], indexMap: /* @__PURE__ */ new Map() };
   const state = {
     invertedIdx: createInvertedIndex(fieldNameTable),
@@ -3559,26 +3464,22 @@ function createPartitionIndex(partitionId, trackPositions = true, vectorPromotio
     booleanIndexes: /* @__PURE__ */ new Map(),
     enumIndexes: /* @__PURE__ */ new Map(),
     geoIndexes: /* @__PURE__ */ new Map(),
-    vectorStores: /* @__PURE__ */ new Map(),
     fieldNameTable,
     flatSchemaCache: null,
     lastSchemaRef: null,
     trackPositions
   };
   function clearAll() {
-    vectorPromoter?.shutdown();
     state.invertedIdx.clear();
     state.docStore.clear();
     for (const idx of state.numericIndexes.values()) idx.clear();
     for (const idx of state.booleanIndexes.values()) idx.clear();
     for (const idx of state.enumIndexes.values()) idx.clear();
     for (const idx of state.geoIndexes.values()) idx.clear();
-    for (const store of state.vectorStores.values()) store.clear();
     state.numericIndexes.clear();
     state.booleanIndexes.clear();
     state.enumIndexes.clear();
     state.geoIndexes.clear();
-    state.vectorStores.clear();
     state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
     state.flatSchemaCache = null;
     state.lastSchemaRef = null;
@@ -3604,6 +3505,7 @@ function createPartitionIndex(partitionId, trackPositions = true, vectorPromotio
           validateDocumentStrict(document, schema);
         }
       }
+      state.docStore.ensureInternalId(docId);
       const flatSchema = getFlatSchema(state, schema);
       const { fieldLengths, tokensByField } = indexDocument(
         state,
@@ -3619,7 +3521,6 @@ function createPartitionIndex(partitionId, trackPositions = true, vectorPromotio
         state.docStore.store(docId, document, fieldLengths);
       }
       state.stats.addDocument(fieldLengths, tokensByField);
-      vectorPromoter?.check(state.vectorStores);
     },
     remove(docId, schema, language, options) {
       const stored = state.docStore.get(docId);
@@ -3634,6 +3535,12 @@ function createPartitionIndex(partitionId, trackPositions = true, vectorPromotio
       state.docStore.remove(docId);
       state.stats.removeDocument(fieldLengths, tokensByField);
     },
+    beginBatch() {
+      state.invertedIdx.beginBatch();
+    },
+    endBatch() {
+      state.invertedIdx.endBatch();
+    },
     update(docId, document, schema, language, options) {
       const stored = state.docStore.get(docId);
       if (!stored) {
@@ -3661,6 +3568,7 @@ function createPartitionIndex(partitionId, trackPositions = true, vectorPromotio
         );
         state.stats.removeDocument(oldFieldLengths, oldTokens);
         state.docStore.remove(docId);
+        state.docStore.ensureInternalId(docId);
         const { fieldLengths: newFieldLengths, tokensByField: newTokens } = indexDocument(
           state,
           docId,
@@ -3719,29 +3627,17 @@ function createPartitionIndex(partitionId, trackPositions = true, vectorPromotio
       bytes += docCount * state.booleanIndexes.size * FIELD_ENTRY_OVERHEAD;
       bytes += docCount * state.enumIndexes.size * FIELD_ENTRY_OVERHEAD;
       bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD;
-      for (const store of state.vectorStores.values()) {
-        bytes += store.estimateMemoryBytes();
-      }
       return bytes;
     },
-    vectorFieldCount() {
-      return state.vectorStores.size;
-    },
-    hasPromotedHnsw() {
-      for (const store of state.vectorStores.values()) {
-        if (store.isPromoted) return true;
-      }
-      return false;
-    },
     searchFulltext(params) {
       return searchFulltext(state, params);
     },
-    searchVector(params) {
-      return searchVector(state, params);
-    },
     applyFilters(filters, schema) {
       return applyPartitionFilters(state, filters, schema);
     },
+    applyFiltersBitset(filters, schema) {
+      return applyPartitionFiltersBitset(state, filters, schema);
+    },
     computeFacets(docIds, config, schema) {
       return computeFacets(state, docIds, config, schema);
     },
@@ -3763,14 +3659,31 @@ function createPartitionIndex(partitionId, trackPositions = true, vectorPromotio
 }
 
 // src/partitioning/manager.ts
-function createPartitionManager(indexName, config, language, router, initialPartitionCount) {
+function setNestedValue(obj, path, value) {
+  if (!path.includes(".")) {
+    obj[path] = value;
+    return;
+  }
+  const segments = path.split(".");
+  let current = obj;
+  for (let i = 0; i < segments.length - 1; i++) {
+    let next = current[segments[i]];
+    if (next === null || next === void 0 || typeof next !== "object") {
+      next = {};
+      current[segments[i]] = next;
+    }
+    current = next;
+  }
+  current[segments[segments.length - 1]] = value;
+}
+function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
   const count = initialPartitionCount ?? 1;
   const trackPositions = config.trackPositions ?? true;
-  const vectorPromotionConfig = config.vectorPromotion;
+  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
   let partitions = [];
   const docPartitionMap = /* @__PURE__ */ new Map();
   for (let i = 0; i < count; i++) {
-    partitions.push(createPartitionIndex(i, trackPositions, vectorPromotionConfig));
+    partitions.push(createPartitionIndex(i, trackPositions));
   }
   function validatePartitionId(partitionId) {
     if (partitionId < 0 || partitionId >= partitions.length) {
@@ -3838,7 +3751,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
           { maxPartitions: currentMaxPartitions, currentCount: partitions.length }
         );
       }
-      const partition = createPartitionIndex(partitions.length, trackPositions, vectorPromotionConfig);
+      const partition = createPartitionIndex(partitions.length, trackPositions);
       partitions.push(partition);
       return partition;
     },
@@ -3878,6 +3791,16 @@ function createPartitionManager(indexName, config, language, router, initialPart
       partitions[pid].remove(docId, config.schema, language);
       docPartitionMap.delete(docId);
     },
+    beginBatchRemove() {
+      for (let i = 0; i < partitions.length; i++) {
+        partitions[i].beginBatch();
+      }
+    },
+    endBatchRemove() {
+      for (let i = 0; i < partitions.length; i++) {
+        partitions[i].endBatch();
+      }
+    },
     update(docId, document, options) {
       const pid = docPartitionMap.get(docId);
       if (pid === void 0) {
@@ -3889,7 +3812,15 @@ function createPartitionManager(indexName, config, language, router, initialPart
     get(docId) {
       const pid = docPartitionMap.get(docId);
       if (pid === void 0) return void 0;
-      return partitions[pid].get(docId);
+      const doc = partitions[pid].get(docId);
+      if (!doc) return void 0;
+      for (const [fieldPath, vecIndex] of vecIndexes) {
+        const vector = vecIndex.getVector(docId);
+        if (vector) {
+          setNestedValue(doc, fieldPath, vector);
+        }
+      }
+      return doc;
     },
     getRef(docId) {
       const pid = docPartitionMap.get(docId);
@@ -3936,16 +3867,26 @@ function createPartitionManager(indexName, config, language, router, initialPart
       for (let i = 0; i < partitions.length; i++) {
         total += partitions[i].estimateMemoryBytes();
       }
+      for (const [, vecIndex] of vecIndexes) {
+        total += vecIndex.estimateMemoryBytes();
+      }
       return total;
     },
     getPartitionStats() {
       return partitions.map((p, i) => ({
         partitionId: i,
         documentCount: p.count(),
-        estimatedMemoryBytes: p.estimateMemoryBytes(),
-        vectorFieldCount: p.vectorFieldCount(),
-        isHnswPromoted: p.hasPromotedHnsw()
+        estimatedMemoryBytes: p.estimateMemoryBytes()
       }));
+    },
+    getVectorIndexes() {
+      return vecIndexes;
+    },
+    resetVectorIndexes(newIndexes) {
+      vecIndexes.clear();
+      for (const [key, value] of newIndexes) {
+        vecIndexes.set(key, value);
+      }
     }
   };
   return manager;
@@ -3993,6 +3934,957 @@ function createPartitionRouter() {
   return { route, routeBatch };
 }
 
+// src/runtime/detect.ts
+function detectRuntimeName() {
+  if ("Bun" in globalThis) {
+    return "bun";
+  }
+  if ("Deno" in globalThis) {
+    return "deno";
+  }
+  const proc = globalThis.process;
+  if (typeof proc === "object" && proc !== null && typeof proc.versions === "object" && typeof proc.versions?.node === "string") {
+    return "node";
+  }
+  return "browser";
+}
+function getCpuCount() {
+  const nav = globalThis.navigator;
+  if (nav && typeof nav.hardwareConcurrency === "number" && nav.hardwareConcurrency >= 1) {
+    return nav.hardwareConcurrency;
+  }
+  return 1;
+}
+function detectRuntime() {
+  const runtime = detectRuntimeName();
+  return {
+    runtime,
+    cpuCount: getCpuCount(),
+    supportsWorkerThreads: runtime === "node" || runtime === "bun",
+    supportsWebWorkers: runtime === "browser" || runtime === "deno" || runtime === "bun",
+    supportsFileSystem: runtime === "node" || runtime === "bun" || runtime === "deno",
+    supportsIndexedDB: runtime === "browser" || "indexedDB" in globalThis,
+    supportsBroadcastChannel: "BroadcastChannel" in globalThis
+  };
+}
+
+// src/vector/hnsw-worker-dispatch.ts
+var BUILD_TIMEOUT_MS = 12e4;
+function resolveWorkerEntryPoint() {
+  const base = import.meta.url;
+  const distIndex = base.lastIndexOf("/dist/");
+  if (distIndex !== -1) {
+    return new URL("vector/hnsw-build-worker.mjs", base.slice(0, distIndex + 6)).href;
+  }
+  return base.replace(/\/src\/vector\/[^/]+$/, "/dist/vector/hnsw-build-worker.mjs");
+}
+async function spawnWorker() {
+  const runtime = detectRuntime();
+  const entryPoint = resolveWorkerEntryPoint();
+  if (runtime.supportsWorkerThreads) {
+    try {
+      const workerThreads = await import('worker_threads');
+      const instance = new workerThreads.Worker(new URL(entryPoint));
+      return instance;
+    } catch {
+      return null;
+    }
+  }
+  if (runtime.supportsWebWorkers) {
+    try {
+      const WorkerCtor = globalThis.Worker;
+      if (typeof WorkerCtor !== "function") return null;
+      return new WorkerCtor(entryPoint, { type: "module" });
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}
+function listenForMessage(worker, handler) {
+  if (typeof worker.on === "function") {
+    worker.on("message", handler);
+    return () => {
+    };
+  }
+  if (typeof worker.addEventListener === "function") {
+    const wrappedHandler = (event) => {
+      handler(event.data);
+    };
+    worker.addEventListener("message", wrappedHandler);
+    return () => {
+      worker.removeEventListener?.("message", wrappedHandler);
+    };
+  }
+  return () => {
+  };
+}
+async function dispatchWorkerBuild(docIds, vectorData, dimension, config, timeoutMs, skipCopy = false) {
+  let worker;
+  try {
+    worker = await spawnWorker();
+  } catch {
+    return {
+      ok: false,
+      reason: "spawn-error",
+      message: "Failed to spawn HNSW build worker"
+    };
+  }
+  if (worker === null) {
+    return {
+      ok: false,
+      reason: "no-workers",
+      message: "No worker thread support available in the current runtime"
+    };
+  }
+  const effectiveTimeout = timeoutMs ?? BUILD_TIMEOUT_MS;
+  let transferBuffer;
+  if (skipCopy) {
+    transferBuffer = vectorData;
+  } else {
+    transferBuffer = new Float32Array(vectorData.length);
+    transferBuffer.set(vectorData);
+  }
+  const request = {
+    type: "build-binary",
+    docIds,
+    vectorData: transferBuffer,
+    dimension,
+    config
+  };
+  return new Promise((resolve) => {
+    let settled = false;
+    const timeoutId = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      resolve({
+        ok: false,
+        reason: "timeout",
+        message: `HNSW build timed out after ${effectiveTimeout}ms`
+      });
+    }, effectiveTimeout);
+    function cleanup() {
+      clearTimeout(timeoutId);
+      removeListener();
+      try {
+        worker?.terminate();
+      } catch {
+      }
+    }
+    function handleMessage(msg) {
+      if (settled) return;
+      const response = msg;
+      if (response.type === "success") {
+        settled = true;
+        cleanup();
+        resolve({ ok: true, graph: response.graph });
+        return;
+      }
+      if (response.type === "error") {
+        settled = true;
+        cleanup();
+        resolve({
+          ok: false,
+          reason: "build-error",
+          message: response.message
+        });
+      }
+    }
+    const removeListener = listenForMessage(worker, handleMessage);
+    try {
+      worker.postMessage(request, [transferBuffer.buffer]);
+    } catch (err) {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      resolve({
+        ok: false,
+        reason: "spawn-error",
+        message: err instanceof Error ? err.message : "Failed to post message to worker"
+      });
+    }
+  });
+}
+
+// src/vector/scalar-quantization.ts
+var PADDING_FACTOR = 0.01;
+function createScalarQuantizer(dimensions) {
+  const quantizedVectors = /* @__PURE__ */ new Map();
+  const vectorSums = /* @__PURE__ */ new Map();
+  const vectorSumSqs = /* @__PURE__ */ new Map();
+  const dequantizedMagnitudes = /* @__PURE__ */ new Map();
+  let alpha = 0;
+  let offset = 0;
+  let alphaSquared = 0;
+  let alphaTimesOffset = 0;
+  let dTimesOffsetSquared = 0;
+  let invAlpha = 0;
+  let calibrated = false;
+  function updateDerivedConstants() {
+    alphaSquared = alpha * alpha;
+    alphaTimesOffset = alpha * offset;
+    dTimesOffsetSquared = dimensions * offset * offset;
+    invAlpha = alpha > 0 ? 1 / alpha : 0;
+  }
+  function computeMagnitude(sumSq, sum) {
+    const val = alphaSquared * sumSq + 2 * alphaTimesOffset * sum + dTimesOffsetSquared;
+    return val > 0 ? Math.sqrt(val) : 0;
+  }
+  function quantizeVector(vector) {
+    const result = new Uint8Array(dimensions);
+    for (let d = 0; d < dimensions; d++) {
+      const normalized = (vector[d] - offset) * invAlpha;
+      const scaled = normalized + 0.5;
+      result[d] = scaled < 0 ? 0 : scaled > 255 ? 255 : scaled | 0;
+    }
+    return result;
+  }
+  function computeSumAndSumSq(quantized) {
+    let sum = 0;
+    let sumSq = 0;
+    for (let d = 0; d < dimensions; d++) {
+      const val = quantized[d];
+      sum += val;
+      sumSq += val * val;
+    }
+    return { sum, sumSq };
+  }
+  function storeQuantized(docId, quantized, sum, sumSq) {
+    quantizedVectors.set(docId, quantized);
+    vectorSums.set(docId, sum);
+    vectorSumSqs.set(docId, sumSq);
+    dequantizedMagnitudes.set(docId, computeMagnitude(sumSq, sum));
+  }
+  function calibrateFromVectors(vectors) {
+    let globalMin = Number.POSITIVE_INFINITY;
+    let globalMax = Number.NEGATIVE_INFINITY;
+    let count = 0;
+    for (const vec of vectors) {
+      for (let d = 0; d < dimensions; d++) {
+        const val = vec[d];
+        if (val < globalMin) globalMin = val;
+        if (val > globalMax) globalMax = val;
+      }
+      count++;
+    }
+    if (count === 0) return;
+    const range = globalMax - globalMin;
+    const pad = range * PADDING_FACTOR;
+    globalMin -= pad;
+    globalMax += pad;
+    if (globalMin === globalMax) {
+      globalMin -= 1e-3;
+      globalMax += 1e-3;
+    }
+    alpha = (globalMax - globalMin) / 255;
+    offset = globalMin;
+    updateDerivedConstants();
+    calibrated = true;
+  }
+  function isOutsideBounds(vector) {
+    const currentMin = offset;
+    const currentMax = offset + alpha * 255;
+    for (let d = 0; d < dimensions; d++) {
+      if (vector[d] < currentMin || vector[d] > currentMax) {
+        return true;
+      }
+    }
+    return false;
+  }
+  return {
+    get dimensions() {
+      return dimensions;
+    },
+    get size() {
+      return quantizedVectors.size;
+    },
+    isCalibrated() {
+      return calibrated;
+    },
+    calibrate(vectors) {
+      calibrateFromVectors(vectors);
+    },
+    needsRecalibration(vector) {
+      if (!calibrated) return false;
+      return isOutsideBounds(vector);
+    },
+    recalibrateAll(vectors) {
+      const collected = [];
+      const rawVectors = [];
+      for (const pair of vectors) {
+        collected.push(pair);
+        rawVectors.push(pair[1]);
+      }
+      calibrateFromVectors(rawVectors);
+      quantizedVectors.clear();
+      vectorSums.clear();
+      vectorSumSqs.clear();
+      dequantizedMagnitudes.clear();
+      for (const [docId, vec] of collected) {
+        const quantized = quantizeVector(vec);
+        const { sum, sumSq } = computeSumAndSumSq(quantized);
+        storeQuantized(docId, quantized, sum, sumSq);
+      }
+    },
+    quantize(docId, vector) {
+      if (!calibrated) {
+        calibrateFromVectors([vector]);
+      }
+      const quantized = quantizeVector(vector);
+      const { sum, sumSq } = computeSumAndSumSq(quantized);
+      storeQuantized(docId, quantized, sum, sumSq);
+    },
+    remove(docId) {
+      quantizedVectors.delete(docId);
+      vectorSums.delete(docId);
+      vectorSumSqs.delete(docId);
+      dequantizedMagnitudes.delete(docId);
+    },
+    getQuantized(docId) {
+      return quantizedVectors.get(docId);
+    },
+    prepareQuery(query) {
+      if (!calibrated) return null;
+      const quantized = quantizeVector(query);
+      const { sum, sumSq } = computeSumAndSumSq(quantized);
+      const mag = computeMagnitude(sumSq, sum);
+      return { quantized, sum, sumSq, magnitude: mag };
+    },
+    distanceFromPrepared(prepared, docId, metric) {
+      const vecQ = quantizedVectors.get(docId);
+      if (!vecQ) return Number.POSITIVE_INFINITY;
+      const vecSum = vectorSums.get(docId);
+      const vecSumSq = vectorSumSqs.get(docId);
+      if (vecSum === void 0 || vecSumSq === void 0) return Number.POSITIVE_INFINITY;
+      const qQ = prepared.quantized;
+      const dims = dimensions;
+      if (metric === "euclidean") {
+        let intSqDist = 0;
+        for (let d = 0; d < dims; d++) {
+          const diff = qQ[d] - vecQ[d];
+          intSqDist += diff * diff;
+        }
+        return alpha * Math.sqrt(intSqDist);
+      }
+      let intDot = 0;
+      for (let d = 0; d < dims; d++) {
+        intDot += qQ[d] * vecQ[d];
+      }
+      const realDot = alphaSquared * intDot + alphaTimesOffset * (prepared.sum + vecSum) + dTimesOffsetSquared;
+      if (metric === "dotProduct") {
+        return -realDot;
+      }
+      const vecMag = dequantizedMagnitudes.get(docId);
+      if (!vecMag || vecMag === 0 || prepared.magnitude === 0) return 1;
+      return 1 - realDot / (prepared.magnitude * vecMag);
+    },
+    restoreCalibration(restoredAlpha, restoredOffset) {
+      alpha = restoredAlpha;
+      offset = restoredOffset;
+      updateDerivedConstants();
+      calibrated = true;
+    },
+    restoreEntry(docId, quantized, sum, sumSq) {
+      storeQuantized(docId, quantized, sum, sumSq);
+    },
+    serialize() {
+      const serializedVectors = {};
+      const serializedSums = {};
+      const serializedSumSqs = {};
+      for (const [docId, qv] of quantizedVectors) {
+        serializedVectors[docId] = Array.from(qv);
+        const sum = vectorSums.get(docId);
+        const sumSq = vectorSumSqs.get(docId);
+        if (sum !== void 0) serializedSums[docId] = sum;
+        if (sumSq !== void 0) serializedSumSqs[docId] = sumSq;
+      }
+      return {
+        alpha,
+        offset,
+        quantizedVectors: serializedVectors,
+        vectorSums: serializedSums,
+        vectorSumSqs: serializedSumSqs
+      };
+    },
+    clear() {
+      quantizedVectors.clear();
+      vectorSums.clear();
+      vectorSumSqs.clear();
+      dequantizedMagnitudes.clear();
+      alpha = 0;
+      offset = 0;
+      alphaSquared = 0;
+      alphaTimesOffset = 0;
+      dTimesOffsetSquared = 0;
+      invAlpha = 0;
+      calibrated = false;
+    }
+  };
+}
+function deserializeScalarQuantizer(data, dimensions) {
+  const quantizer = createScalarQuantizer(dimensions);
+  if (data.alpha === 0 && data.offset === 0 && Object.keys(data.quantizedVectors).length === 0) {
+    return quantizer;
+  }
+  quantizer.restoreCalibration(data.alpha, data.offset);
+  for (const [docId, values] of Object.entries(data.quantizedVectors)) {
+    const quantized = new Uint8Array(values);
+    const sum = data.vectorSums[docId] ?? 0;
+    const sumSq = data.vectorSumSqs[docId] ?? 0;
+    quantizer.restoreEntry(docId, quantized, sum, sumSq);
+  }
+  return quantizer;
+}
+
+// src/vector/vector-index.ts
+var DEFAULT_PROMOTION_THRESHOLD = 1024;
+var DEFAULT_FILTER_THRESHOLD = 0.03;
+var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
+var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
+var WORKER_BUILD_SIZE_THRESHOLD = 5e3;
+function createVectorIndex(fieldName, dimension, config) {
+  if (!Number.isInteger(dimension) || dimension <= 0) {
+    throw new NarsilError(
+      ErrorCodes.VECTOR_DIMENSION_MISMATCH,
+      `Vector dimension must be a positive integer, got ${dimension}`,
+      { dimension }
+    );
+  }
+  const promotionThreshold = config?.threshold ?? DEFAULT_PROMOTION_THRESHOLD;
+  const filterThreshold = Math.max(0, Math.min(1, config?.filterThreshold ?? DEFAULT_FILTER_THRESHOLD));
+  const quantizationMode = config?.quantization ?? "sq8";
+  const rawHnswM = config?.hnswConfig?.m;
+  const hnswConfig = config?.hnswConfig ? { ...config.hnswConfig, m: rawHnswM !== void 0 ? Math.max(rawHnswM, 2) : void 0 } : void 0;
+  const dimensionScale = dimension / 256;
+  const store = createVectorStore();
+  const tombstones = /* @__PURE__ */ new Set();
+  let sq8 = quantizationMode === "sq8" ? createScalarQuantizer(dimension) : null;
+  let hnsw = null;
+  const buffer = /* @__PURE__ */ new Set();
+  let building = false;
+  let buildScheduled = false;
+  let pendingBuild = null;
+  let disposed = false;
+  function liveSize() {
+    return store.size - tombstones.size;
+  }
+  function yieldToEventLoop() {
+    return new Promise((resolve) => setTimeout(resolve, 0));
+  }
+  function mergeResults(hnswResults, bufferResults, k) {
+    const seen = /* @__PURE__ */ new Set();
+    const merged = [];
+    let hi = 0;
+    let bi = 0;
+    while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
+      const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
+      const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
+      let pick;
+      if (h && b) {
+        if (h.score > b.score || h.score === b.score && h.docId.localeCompare(b.docId) < 0) {
+          pick = h;
+          hi++;
+        } else {
+          pick = b;
+          bi++;
+        }
+      } else if (h) {
+        pick = h;
+        hi++;
+      } else if (b) {
+        pick = b;
+        bi++;
+      } else {
+        break;
+      }
+      if (seen.has(pick.docId)) continue;
+      seen.add(pick.docId);
+      merged.push(pick);
+    }
+    return merged;
+  }
+  function validateDimension(vector) {
+    if (vector.length !== dimension) {
+      throw new NarsilError(
+        ErrorCodes.VECTOR_DIMENSION_MISMATCH,
+        `Vector dimension mismatch: expected ${dimension}, got ${vector.length}`,
+        { expected: dimension, received: vector.length }
+      );
+    }
+  }
+  function calibrateAndQuantizeAll() {
+    if (!sq8) return;
+    if (store.size === 0) return;
+    function* vectorIterator() {
+      for (const [docId, entry] of store.entries()) {
+        if (tombstones.has(docId)) continue;
+        yield entry.vector;
+      }
+    }
+    sq8.calibrate(vectorIterator());
+    for (const [docId, entry] of store.entries()) {
+      if (tombstones.has(docId)) continue;
+      sq8.quantize(docId, entry.vector);
+    }
+  }
+  function recalibrateFromStore() {
+    if (!sq8) return;
+    function* storeVectors() {
+      for (const [docId, entry] of store.entries()) {
+        if (tombstones.has(docId)) continue;
+        yield [docId, entry.vector];
+      }
+    }
+    sq8.recalibrateAll(storeVectors());
+  }
+  function triggerBuild() {
+    if (building) return;
+    building = true;
+    const liveDocIds = [];
+    for (const [docId] of store.entries()) {
+      if (tombstones.has(docId)) continue;
+      liveDocIds.push(docId);
+    }
+    const bufferSnapshot = new Set(buffer);
+    const buildPromise = (async () => {
+      try {
+        if (liveDocIds.length === 0 || disposed) return;
+        if (sq8) {
+          calibrateAndQuantizeAll();
+        }
+        if (liveDocIds.length > WORKER_BUILD_SIZE_THRESHOLD) {
+          const workerResult = await tryWorkerBuild(liveDocIds, bufferSnapshot);
+          if (workerResult) return;
+        }
+        const newHnsw = createHNSWIndex(dimension, store, hnswConfig, sq8 ?? void 0);
+        const CHUNK_SIZE = 100;
+        for (let i = 0; i < liveDocIds.length; i++) {
+          const docId = liveDocIds[i];
+          if (!store.has(docId) || tombstones.has(docId)) continue;
+          newHnsw.insertNode(docId);
+          if ((i + 1) % CHUNK_SIZE === 0) {
+            if (disposed) return;
+            await yieldToEventLoop();
+          }
+        }
+        if (disposed) return;
+        hnsw = newHnsw;
+        for (const docId of tombstones) {
+          if (newHnsw.has(docId)) {
+            newHnsw.markTombstone(docId);
+          }
+        }
+        for (const docId of bufferSnapshot) {
+          buffer.delete(docId);
+        }
+      } finally {
+        building = false;
+        pendingBuild = null;
+      }
+    })();
+    pendingBuild = buildPromise;
+  }
+  async function tryWorkerBuild(liveDocIds, bufferSnapshot) {
+    const vectorData = new Float32Array(liveDocIds.length * dimension);
+    const validDocIds = [];
+    let offset = 0;
+    for (const docId of liveDocIds) {
+      const entry = store.get(docId);
+      if (!entry || tombstones.has(docId)) continue;
+      vectorData.set(entry.vector, offset);
+      validDocIds.push(docId);
+      offset += dimension;
+    }
+    if (validDocIds.length === 0) return false;
+    const packedData = offset < vectorData.length ? vectorData.subarray(0, offset) : vectorData;
+    const resolvedConfig = {
+      m: hnswConfig?.m,
+      efConstruction: hnswConfig?.efConstruction,
+      metric: hnswConfig?.metric
+    };
+    const timeoutMs = Math.max(1e4, liveDocIds.length * 2);
+    const outcome = await dispatchWorkerBuild(validDocIds, packedData, dimension, resolvedConfig, timeoutMs, true);
+    if (!outcome.ok) return false;
+    const newHnsw = createHNSWIndex(dimension, store, hnswConfig, sq8 ?? void 0);
+    newHnsw.deserialize(outcome.graph);
+    hnsw = newHnsw;
+    for (const docId of tombstones) {
+      if (newHnsw.has(docId)) {
+        newHnsw.markTombstone(docId);
+      }
+    }
+    for (const docId of bufferSnapshot) {
+      buffer.delete(docId);
+    }
+    return true;
+  }
+  async function awaitPendingBuild() {
+    if (pendingBuild) {
+      await pendingBuild;
+    }
+  }
+  function dispose() {
+    disposed = true;
+  }
+  function scheduleBuild() {
+    if (building || buildScheduled || disposed) return;
+    const thresholdMet = !hnsw && liveSize() >= promotionThreshold || hnsw !== null && buffer.size >= promotionThreshold;
+    if (!thresholdMet) return;
+    buildScheduled = true;
+    setTimeout(() => {
+      buildScheduled = false;
+      triggerBuild();
+    }, 0);
+  }
+  function bruteForceSearch(query, k, metric, minSimilarity, candidates) {
+    const queryMag = magnitude(query);
+    const highScoreFirst = (a, b) => b.score - a.score || a.docId.localeCompare(b.docId);
+    const heap = createBoundedMaxHeap(highScoreFirst, k);
+    for (const docId of candidates) {
+      if (tombstones.has(docId)) continue;
+      const entry = store.get(docId);
+      if (!entry) continue;
+      let score;
+      switch (metric) {
+        case "cosine":
+          score = cosineSimilarityWithMagnitudes(query, entry.vector, queryMag, entry.magnitude);
+          break;
+        case "dotProduct":
+          score = dotProduct(query, entry.vector);
+          break;
+        case "euclidean": {
+          const dist = euclideanDistance(query, entry.vector);
+          score = 1 / (1 + dist);
+          break;
+        }
+      }
+      if (score >= minSimilarity) {
+        heap.push({ docId, score });
+      }
+    }
+    return heap.toSortedArray().reverse();
+  }
+  function* allLiveDocIds() {
+    for (const [docId] of store.entries()) {
+      if (tombstones.has(docId)) continue;
+      yield docId;
+    }
+  }
+  function insert(docId, vector) {
+    validateDimension(vector);
+    tombstones.delete(docId);
+    store.insert(docId, vector);
+    buffer.add(docId);
+  }
+  function remove(docId) {
+    if (!store.has(docId)) return;
+    tombstones.add(docId);
+    buffer.delete(docId);
+    if (hnsw) {
+      hnsw.markTombstone(docId);
+    }
+  }
+  function* bufferCandidates(filterDocIds) {
+    if (filterDocIds) {
+      if (buffer.size <= filterDocIds.size) {
+        for (const docId of buffer) {
+          if (filterDocIds.has(docId) && !tombstones.has(docId)) yield docId;
+        }
+      } else {
+        for (const docId of filterDocIds) {
+          if (buffer.has(docId) && !tombstones.has(docId)) yield docId;
+        }
+      }
+    } else {
+      for (const docId of buffer) {
+        if (!tombstones.has(docId)) yield docId;
+      }
+    }
+  }
+  function search(query, k, options) {
+    validateDimension(query);
+    const currentLiveSize = liveSize();
+    if (currentLiveSize === 0) return [];
+    if (k <= 0) return [];
+    if (buffer.size > 0 && !building && !buildScheduled) {
+      scheduleBuild();
+    }
+    const { metric, minSimilarity, filterDocIds, efSearch } = options;
+    if (filterDocIds && filterDocIds.size === 0) return [];
+    if (!hnsw) {
+      const candidates = filterDocIds ?? allLiveDocIds();
+      return bruteForceSearch(query, k, metric, minSimilarity, candidates);
+    }
+    if (buffer.size === 0) {
+      if (filterDocIds) {
+        const hnswLiveSize = hnsw.size;
+        const selectivity = hnswLiveSize > 0 ? filterDocIds.size / hnswLiveSize : 1;
+        if (selectivity < filterThreshold) {
+          return bruteForceSearch(query, k, metric, minSimilarity, filterDocIds);
+        }
+      }
+      const hnswResults2 = hnsw.search(query, k, metric, minSimilarity, filterDocIds, efSearch);
+      return hnswResults2.map((r) => ({ docId: r.docId, score: r.score }));
+    }
+    if (filterDocIds) {
+      const hnswLiveSize = hnsw.size;
+      const selectivity = hnswLiveSize > 0 ? filterDocIds.size / hnswLiveSize : 1;
+      if (selectivity < filterThreshold) {
+        return bruteForceSearch(query, k, metric, minSimilarity, filterDocIds);
+      }
+    }
+    const hnswResults = hnsw.search(query, k, metric, minSimilarity, filterDocIds, efSearch).map((r) => ({ docId: r.docId, score: r.score }));
+    const bufferResults = bruteForceSearch(query, k, metric, minSimilarity, bufferCandidates(filterDocIds));
+    return mergeResults(hnswResults, bufferResults, k);
+  }
+  function getVector(docId) {
+    if (tombstones.has(docId)) return null;
+    const entry = store.get(docId);
+    if (!entry) return null;
+    return new Float32Array(entry.vector);
+  }
+  function has(docId) {
+    return store.has(docId) && !tombstones.has(docId);
+  }
+  function compact() {
+    if (tombstones.size === 0) return;
+    if (hnsw) {
+      hnsw.compactTombstones();
+    }
+    for (const docId of tombstones) {
+      store.remove(docId);
+      buffer.delete(docId);
+      if (sq8) {
+        sq8.remove(docId);
+      }
+    }
+    tombstones.clear();
+    if (sq8?.isCalibrated() && store.size > 0) {
+      recalibrateFromStore();
+    }
+  }
+  async function optimize() {
+    if (pendingBuild) {
+      await pendingBuild;
+    }
+    compact();
+    const live = liveSize();
+    if (live === 0) {
+      if (hnsw) {
+        hnsw.clear();
+        hnsw = null;
+      }
+      buffer.clear();
+      if (sq8) {
+        sq8.clear();
+      }
+      return;
+    }
+    if (sq8) {
+      calibrateAndQuantizeAll();
+    }
+    const newHnsw = createHNSWIndex(dimension, store, hnswConfig, sq8 ?? void 0);
+    const liveDocIds = [];
+    for (const [docId] of store.entries()) {
+      if (tombstones.has(docId)) continue;
+      liveDocIds.push(docId);
+    }
+    const CHUNK_SIZE = 100;
+    for (let i = 0; i < liveDocIds.length; i++) {
+      newHnsw.insertNode(liveDocIds[i]);
+      if ((i + 1) % CHUNK_SIZE === 0) {
+        await yieldToEventLoop();
+      }
+    }
+    hnsw = newHnsw;
+    buffer.clear();
+    if (sq8 && store.size > 0) {
+      recalibrateFromStore();
+    }
+  }
+  function maintenanceStatus() {
+    const storeSize = store.size;
+    const tombstoneRatio = storeSize > 0 ? tombstones.size / storeSize : 0;
+    const graphCount = hnsw ? 1 : 0;
+    const estimatedCompactMs = Math.round(tombstones.size * ESTIMATED_MS_PER_TOMBSTONE * dimensionScale);
+    const estimatedOptimizeMs = Math.round(storeSize * ESTIMATED_MS_PER_VECTOR_REBUILD * dimensionScale);
+    return {
+      tombstoneRatio,
+      graphCount,
+      bufferSize: buffer.size,
+      building,
+      estimatedCompactMs,
+      estimatedOptimizeMs
+    };
+  }
+  function serialize() {
+    const vectors = [];
+    for (const [docId, entry] of store.entries()) {
+      if (tombstones.has(docId)) continue;
+      vectors.push({ docId, vector: Array.from(entry.vector) });
+    }
+    const graphs = [];
+    if (hnsw) {
+      graphs.push(hnsw.serialize());
+    }
+    let sq8Data = null;
+    if (sq8?.isCalibrated() && sq8.size > 0) {
+      sq8Data = sq8.serialize();
+    }
+    return {
+      fieldName,
+      dimension,
+      vectors,
+      graphs,
+      sq8: sq8Data
+    };
+  }
+  function deserialize(payload) {
+    if (payload.dimension !== dimension) {
+      throw new NarsilError(
+        ErrorCodes.VECTOR_DIMENSION_MISMATCH,
+        `Payload dimension ${payload.dimension} does not match index dimension ${dimension}`,
+        { expected: dimension, received: payload.dimension }
+      );
+    }
+    for (const entry of payload.vectors) {
+      if (entry.vector.length !== dimension) {
+        throw new NarsilError(
+          ErrorCodes.VECTOR_DIMENSION_MISMATCH,
+          `Vector for doc "${entry.docId}" has dimension ${entry.vector.length}, expected ${dimension}`,
+          { docId: entry.docId, expected: dimension, received: entry.vector.length }
+        );
+      }
+    }
+    store.clear();
+    tombstones.clear();
+    buffer.clear();
+    if (hnsw) {
+      hnsw.clear();
+      hnsw = null;
+    }
+    if (sq8) {
+      sq8.clear();
+    }
+    for (const entry of payload.vectors) {
+      store.insert(entry.docId, new Float32Array(entry.vector));
+    }
+    if (payload.sq8) {
+      if (quantizationMode === "sq8") {
+        sq8 = deserializeScalarQuantizer(payload.sq8, dimension);
+      }
+    }
+    if (payload.graphs.length > 0) {
+      const graphData = payload.graphs[0];
+      const restoredHnsw = createHNSWIndex(
+        dimension,
+        store,
+        {
+          m: graphData.m ?? hnswConfig?.m,
+          efConstruction: graphData.efConstruction ?? hnswConfig?.efConstruction,
+          metric: graphData.metric ?? hnswConfig?.metric
+        },
+        sq8 ?? void 0
+      );
+      restoredHnsw.deserialize(graphData);
+      hnsw = restoredHnsw;
+      for (let i = 1; i < payload.graphs.length; i++) {
+        const additionalGraph = payload.graphs[i];
+        for (const [nodeDocId] of additionalGraph.nodes) {
+          if (!restoredHnsw.has(nodeDocId) && store.has(nodeDocId)) {
+            restoredHnsw.insertNode(nodeDocId);
+          }
+        }
+      }
+      for (const [docId] of store.entries()) {
+        if (tombstones.has(docId)) continue;
+        if (!restoredHnsw.has(docId)) {
+          buffer.add(docId);
+        }
+      }
+      if (buffer.size > restoredHnsw.size) {
+        hnsw.clear();
+        hnsw = null;
+        buffer.clear();
+        for (const [docId] of store.entries()) {
+          if (tombstones.has(docId)) continue;
+          buffer.add(docId);
+        }
+      }
+    } else {
+      for (const [docId] of store.entries()) {
+        if (tombstones.has(docId)) continue;
+        buffer.add(docId);
+      }
+    }
+  }
+  function estimateMemoryBytes() {
+    const count = store.size;
+    if (count === 0 && tombstones.size === 0 && buffer.size === 0) return 0;
+    let bytes = store.estimateMemory(dimension);
+    const TOMBSTONE_SET_OVERHEAD = 64;
+    const TOMBSTONE_ENTRY_COST = 72;
+    bytes += TOMBSTONE_SET_OVERHEAD + tombstones.size * TOMBSTONE_ENTRY_COST;
+    const BUFFER_SET_OVERHEAD = 64;
+    const BUFFER_ENTRY_COST = 72;
+    bytes += BUFFER_SET_OVERHEAD + buffer.size * BUFFER_ENTRY_COST;
+    if (hnsw) {
+      const HNSW_NODE_OBJ = 48;
+      const MAP_ENTRY = 72;
+      const MAP_OVERHEAD = 64;
+      const CONN_ARRAY_HEADER = 32;
+      const SET_OVERHEAD = 64;
+      const SET_ENTRY_COST = 72;
+      const m = hnsw.m;
+      const avgLayers = m > 1 ? m / (m - 1) : 1;
+      const avgConnsLayer0 = m;
+      const avgConnsUpper = Math.ceil(m / 2);
+      const connMemPerNode = CONN_ARRAY_HEADER + (SET_OVERHEAD + avgConnsLayer0 * SET_ENTRY_COST) + Math.max(0, avgLayers - 1) * (SET_OVERHEAD + avgConnsUpper * SET_ENTRY_COST);
+      const hnswNodeCount = hnsw.size + hnsw.tombstoneCount;
+      const perHnswNode = MAP_ENTRY + HNSW_NODE_OBJ + connMemPerNode;
+      bytes += MAP_OVERHEAD + hnswNodeCount * perHnswNode;
+    }
+    if (sq8?.isCalibrated()) {
+      const sqCount = sq8.size;
+      const MAP_OVERHEAD_SQ = 64;
+      const MAP_ENTRY_SQ = 72;
+      const UINT8_ARRAY_HEADER = 64;
+      const PER_VECTOR_METADATA = 8 * 3;
+      const GLOBAL_CALIBRATION = 8 * 5;
+      bytes += 4 * (MAP_OVERHEAD_SQ + sqCount * MAP_ENTRY_SQ);
+      bytes += sqCount * (UINT8_ARRAY_HEADER + dimension + PER_VECTOR_METADATA);
+      bytes += GLOBAL_CALIBRATION;
+    }
+    return Math.round(bytes);
+  }
+  return {
+    get size() {
+      return liveSize();
+    },
+    get dimension() {
+      return dimension;
+    },
+    get fieldName() {
+      return fieldName;
+    },
+    insert,
+    remove,
+    scheduleBuild,
+    awaitPendingBuild,
+    dispose,
+    search,
+    getVector,
+    has,
+    compact,
+    optimize,
+    maintenanceStatus,
+    estimateMemoryBytes,
+    serialize,
+    deserialize
+  };
+}
+
 // src/workers/direct-executor.ts
 function createDirectExecutor() {
   const indexes = /* @__PURE__ */ new Map();
@@ -4013,8 +4905,13 @@ function createDirectExecutor() {
     }
     const router = createPartitionRouter();
     const partitionCount = config.partitions?.maxPartitions ?? 1;
-    const manager = createPartitionManager(indexName, config, language, router, partitionCount);
-    indexes.set(indexName, { manager, config, language });
+    const vectorFields = extractVectorFieldsFromSchema(config.schema);
+    const vectorIndexes = /* @__PURE__ */ new Map();
+    for (const [fieldPath, dim] of vectorFields) {
+      vectorIndexes.set(fieldPath, createVectorIndex(fieldPath, dim, config.vectorPromotion));
+    }
+    const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
+    indexes.set(indexName, { manager, config, language, vectorIndexes });
   }
   function dropIndex(indexName) {
     const entry = requireIndex(indexName);
@@ -4098,6 +4995,13 @@ function createDirectExecutor() {
           partition.clear();
         }
         entry.manager.setPartitions(partitions);
+        const vectorFields = extractVectorFieldsFromSchema(entry.config.schema);
+        const newVectorIndexes = /* @__PURE__ */ new Map();
+        for (const [fieldPath, dim] of vectorFields) {
+          newVectorIndexes.set(fieldPath, createVectorIndex(fieldPath, dim, entry.config.vectorPromotion));
+        }
+        entry.manager.resetVectorIndexes(newVectorIndexes);
+        entry.vectorIndexes = entry.manager.getVectorIndexes();
         return void 0;
       }
       case "serialize": {
@@ -4137,4 +5041,4 @@ function createDirectExecutor() {
   };
 }
 
-export { createDirectExecutor, createPartitionIndex, createPartitionRouter, deserializePayloadV2, detectRuntime, fanOutQuery, fnv1a, getLanguage, kWayMerge, registerLanguage, tokenize, validateSchema };
+export { clearNormalizationCache, configureNormalizationCache, createDirectExecutor, createPartitionIndex, createPartitionRouter, deserializePayloadV2, detectRuntime, extractVectorFieldsFromSchema, fanOutQuery, flattenSchema, fnv1a, getLanguage, getNestedValue, getNormalizationCacheSize, kWayMerge, registerLanguage, resetNormalizationCache, tokenize, validateRequiredFields, validateSchema };