@delali/narsil 0.1.1 → 0.1.2

package/dist/index.mjs

@@ -1,9 +1,413 @@
-import { createDirectExecutor, fanOutQuery, tokenize, validateSchema, getLanguage } from './chunk-5HXMVMSM.mjs';
-export { getLanguage, registerLanguage } from './chunk-5HXMVMSM.mjs';
+import { createDirectExecutor, createPartitionRouter, getLanguage, validateSchema, deserializePayloadV2, tokenize, fanOutQuery, createPartitionIndex, detectRuntime, fnv1a, kWayMerge } from './chunk-LIRVYUQ5.mjs';
+export { getLanguage, registerLanguage } from './chunk-LIRVYUQ5.mjs';
+export { isSimdAvailable } from './chunk-GVN2W4VA.mjs';
 import './chunk-WYC635GI.mjs';
-import { NarsilError, ErrorCodes } from './chunk-JKIBUPAY.mjs';
-export { ErrorCodes, NarsilError } from './chunk-JKIBUPAY.mjs';
-import { generateId } from './chunk-N37VMC62.mjs';
+import { NarsilError, ErrorCodes } from './chunk-7GKPWGCN.mjs';
+export { ErrorCodes, NarsilError } from './chunk-7GKPWGCN.mjs';
+import { generateId, isValidWorkerResponse, createRequestId } from './chunk-HRABJLPB.mjs';
+import { decode } from '@msgpack/msgpack';
+
+// src/workers/worker-executor.ts
+var DEFAULT_BACKPRESSURE_LIMIT = 100;
+var DEFAULT_REQUEST_TIMEOUT = 3e4;
+var SHUTDOWN_TIMEOUT = 5e3;
+function createWorkerExecutor(worker, config) {
+  const backpressureLimit = DEFAULT_BACKPRESSURE_LIMIT;
+  const requestTimeout = DEFAULT_REQUEST_TIMEOUT;
+  const pending = /* @__PURE__ */ new Map();
+  function processResponse(msg) {
+    if (!isValidWorkerResponse(msg)) {
+      return;
+    }
+    const response = msg;
+    const entry = pending.get(response.requestId);
+    if (!entry) {
+      return;
+    }
+    clearTimeout(entry.timeoutId);
+    pending.delete(response.requestId);
+    if (response.type === "error") {
+      entry.reject(new NarsilError(response.code, response.message));
+    } else {
+      entry.resolve(response.data);
+    }
+  }
+  if (typeof worker.on === "function") {
+    worker.on("message", (msg) => processResponse(msg));
+  } else if (typeof worker.addEventListener === "function") {
+    worker.addEventListener("message", (event) => {
+      const msg = event.data;
+      processResponse(msg);
+    });
+  }
+  function execute(action) {
+    if (pending.size >= backpressureLimit) {
+      return Promise.reject(
+        new NarsilError(ErrorCodes.WORKER_BUSY, `Backpressure limit of ${backpressureLimit} pending requests reached`)
+      );
+    }
+    const requestId = createRequestId();
+    const taggedAction = { ...action, requestId };
+    return new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        pending.delete(requestId);
+        reject(new NarsilError(ErrorCodes.WORKER_TIMEOUT, `Request ${requestId} timed out after ${requestTimeout}ms`));
+      }, requestTimeout);
+      pending.set(requestId, {
+        resolve,
+        reject,
+        timeoutId
+      });
+      worker.postMessage(taggedAction);
+    });
+  }
+  async function shutdown() {
+    const requestId = createRequestId();
+    const shutdownAction = { type: "shutdown", requestId };
+    const shutdownPromise = new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        pending.delete(requestId);
+        reject(new NarsilError(ErrorCodes.WORKER_TIMEOUT, "Shutdown timed out"));
+      }, SHUTDOWN_TIMEOUT);
+      pending.set(requestId, {
+        resolve: () => resolve(),
+        reject,
+        timeoutId
+      });
+      worker.postMessage(shutdownAction);
+    });
+    try {
+      await shutdownPromise;
+    } finally {
+      for (const [id, entry] of pending) {
+        clearTimeout(entry.timeoutId);
+        entry.reject(new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker is shutting down"));
+        pending.delete(id);
+      }
+    }
+  }
+  return { execute, shutdown };
+}
+
+// src/workers/factory.ts
+function resolveEntryPoint() {
+  const base = import.meta.url;
+  if (base.endsWith(".mjs") || base.endsWith(".js")) {
+    return new URL("./workers/entry.mjs", base).href;
+  }
+  return base.replace(/\/src\/workers\/[^/]+$/, "/dist/workers/entry.mjs");
+}
+async function createWorkerFactory(entryPoint) {
+  const runtime = detectRuntime();
+  const resolvedEntry = resolveEntryPoint();
+  if (runtime.supportsWorkerThreads) {
+    const workerThreadsModule = await import('worker_threads');
+    return function nodeFactory(_workerId) {
+      const instance = new workerThreadsModule.Worker(new URL(resolvedEntry));
+      return createWorkerExecutor(instance);
+    };
+  }
+  if (runtime.supportsWebWorkers) {
+    return function webFactory(_workerId) {
+      const instance = new Worker(resolvedEntry, { type: "module" });
+      return createWorkerExecutor(instance);
+    };
+  }
+  throw new NarsilError(ErrorCodes.WORKER_CRASHED, `No worker support available on runtime "${runtime.runtime}"`);
+}
+
+// src/workers/pool.ts
+function resolveWorkerCount(requested) {
+  if (requested !== void 0 && requested > 0) {
+    return requested;
+  }
+  let cpuCount = 4;
+  try {
+    if (navigator?.hardwareConcurrency) {
+      cpuCount = navigator.hardwareConcurrency;
+    } else if (typeof process !== "undefined") {
+      const ap = process.availableParallelism;
+      if (typeof ap === "function") {
+        cpuCount = ap();
+      }
+    }
+  } catch {
+    cpuCount = 4;
+  }
+  return Math.max(2, Math.min(8, cpuCount - 1));
+}
+function createWorkerPool(config) {
+  const workerCount = resolveWorkerCount(config.count);
+  const workers = [];
+  const indexAssignment = /* @__PURE__ */ new Map();
+  let isShutdown = false;
+  function ensureWorker(slotIndex) {
+    if (!workers[slotIndex]) {
+      workers[slotIndex] = {
+        executor: config.workerFactory(slotIndex),
+        indexes: /* @__PURE__ */ new Set(),
+        memoryStats: { heapUsed: 0, heapTotal: 0, external: 0 }
+      };
+    }
+    return workers[slotIndex];
+  }
+  function assignSlot(indexName) {
+    return fnv1a(indexName) % workerCount;
+  }
+  function addIndex(indexName) {
+    if (isShutdown) {
+      throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
+    }
+    if (indexAssignment.has(indexName)) {
+      return;
+    }
+    const slotIndex = assignSlot(indexName);
+    const slot = ensureWorker(slotIndex);
+    slot.indexes.add(indexName);
+    indexAssignment.set(indexName, slotIndex);
+  }
+  function removeIndex(indexName) {
+    const slotIndex = indexAssignment.get(indexName);
+    if (slotIndex === void 0) {
+      return;
+    }
+    const slot = workers[slotIndex];
+    if (slot) {
+      slot.indexes.delete(indexName);
+    }
+    indexAssignment.delete(indexName);
+  }
+  function getExecutor(indexName) {
+    if (isShutdown) {
+      throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
+    }
+    const slotIndex = indexAssignment.get(indexName);
+    if (slotIndex === void 0) {
+      throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
+    }
+    return workers[slotIndex].executor;
+  }
+  function getMemoryStats() {
+    const stats = [];
+    for (let i = 0; i < workers.length; i++) {
+      const slot = workers[i];
+      if (slot) {
+        stats.push({
+          workerId: i,
+          heapUsed: slot.memoryStats.heapUsed,
+          heapTotal: slot.memoryStats.heapTotal,
+          external: slot.memoryStats.external
+        });
+      }
+    }
+    return stats;
+  }
+  async function shutdown() {
+    if (isShutdown) {
+      return;
+    }
+    isShutdown = true;
+    const shutdownPromises = workers.filter(Boolean).map((slot) => {
+      const timeoutPromise = new Promise((resolve) => {
+        setTimeout(resolve, 5e3);
+      });
+      return Promise.race([slot.executor.shutdown(), timeoutPromise]);
+    });
+    await Promise.allSettled(shutdownPromises);
+    indexAssignment.clear();
+  }
+  function getAllExecutors() {
+    return workers.filter(Boolean).map((slot) => slot.executor);
+  }
+  function addIndexToAll(indexName) {
+    if (isShutdown) {
+      throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
+    }
+    for (let i = 0; i < workerCount; i++) {
+      const slot = ensureWorker(i);
+      slot.indexes.add(indexName);
+    }
+    indexAssignment.set(indexName, 0);
+  }
+  return {
+    getExecutor,
+    getAllExecutors,
+    get workerCount() {
+      return workerCount;
+    },
+    addIndex,
+    addIndexToAll,
+    removeIndex,
+    getMemoryStats,
+    shutdown
+  };
+}
+
+// src/engine/orchestration.ts
+function createWorkerOrchestrator(config, executor, promoter, indexRegistry, callbacks) {
+  let workerPool = null;
+  let promotionInProgress = false;
+  const promotionBuffer = [];
+  const workersEnabled = config?.workers?.enabled === true;
+  async function checkPromotion() {
+    if (!workersEnabled || promotionInProgress || workerPool) return;
+    const indexMap = /* @__PURE__ */ new Map();
+    for (const [name] of indexRegistry) {
+      const mgr = executor.getManager(name);
+      indexMap.set(name, { documentCount: mgr?.countDocuments() ?? 0 });
+    }
+    const result = promoter.check(indexMap);
+    if (result.shouldPromote) {
+      promotionInProgress = true;
+      setTimeout(() => {
+        runPromotion(result.reason).catch((err) => {
+          console.warn("Worker promotion failed:", err);
+          promotionInProgress = false;
+        });
+      }, 0);
+    }
+  }
+  async function runPromotion(reason) {
+    try {
+      const factory = await createWorkerFactory();
+      const pool = createWorkerPool({
+        count: config?.workers?.count,
+        workerFactory: factory
+      });
+      for (const [name] of indexRegistry) {
+        pool.addIndexToAll(name);
+      }
+      const allExecutors = pool.getAllExecutors();
+      for (const [name, entry] of indexRegistry) {
+        await Promise.all(
+          allExecutors.map(
+            (workerExecutor) => workerExecutor.execute({
+              type: "createIndex",
+              indexName: name,
+              config: entry.config,
+              requestId: `promote-create-${name}`
+            })
+          )
+        );
+        const manager = executor.getManager(name);
+        if (manager) {
+          for (let i = 0; i < manager.partitionCount; i++) {
+            const serialized = manager.serializePartition(i);
+            await Promise.all(
+              allExecutors.map(
+                (workerExecutor) => workerExecutor.execute({
+                  type: "deserialize",
+                  indexName: name,
+                  partitionId: i,
+                  data: serialized,
+                  requestId: `promote-sync-${name}-${i}`
+                })
+              )
+            );
+          }
+        }
+      }
+      workerPool = pool;
+      promoter.markPromoted();
+      if (promotionBuffer.length > 0) {
+        const buffered = [...promotionBuffer];
+        promotionBuffer.length = 0;
+        for (const action of buffered) {
+          await replicateToWorkers(action);
+        }
+      }
+      callbacks?.onPromotion?.(pool.workerCount, reason);
+    } catch (err) {
+      promotionBuffer.length = 0;
+      throw err;
+    } finally {
+      promotionInProgress = false;
+    }
+  }
+  async function replicateToWorkers(action) {
+    if (promotionInProgress) {
+      promotionBuffer.push(action);
+      return;
+    }
+    if (!workerPool) return;
+    const allExecutors = workerPool.getAllExecutors();
+    const results = await Promise.allSettled(allExecutors.map((workerExecutor) => workerExecutor.execute(action)));
+    for (const result of results) {
+      if (result.status === "rejected") {
+        console.warn("Worker replication failed:", result.reason);
+      }
+    }
+  }
+  async function searchViaWorker(indexName, params) {
+    if (!workerPool) return null;
+    const manager = executor.getManager(indexName);
+    if (!manager) return null;
+    const allExecutors = workerPool.getAllExecutors();
+    const totalPartitions = manager.partitionCount;
+    const numWorkers = allExecutors.length;
+    if (numWorkers === 0) return null;
+    if (numWorkers === 1) {
+      try {
+        return await allExecutors[0].execute({
+          type: "query",
+          indexName,
+          params,
+          requestId: createRequestId()
+        });
+      } catch (err) {
+        console.warn("Worker search failed, falling back to local:", err);
+        return null;
+      }
+    }
+    try {
+      const workerAssignments = Array.from({ length: numWorkers }, () => []);
+      for (let p = 0; p < totalPartitions; p++) {
+        workerAssignments[p % numWorkers].push(p);
+      }
+      const results = await Promise.all(
+        allExecutors.map(
+          (workerExecutor, idx) => workerExecutor.execute({
+            type: "query",
+            indexName,
+            params,
+            requestId: createRequestId(),
+            partitionIds: workerAssignments[idx]
+          })
+        )
+      );
+      const allScored = results.map((r) => r.scored);
+      const merged = kWayMerge(allScored);
+      let totalMatched = 0;
+      for (const r of results) {
+        totalMatched += r.totalMatched;
+      }
+      return { scored: merged, totalMatched };
+    } catch (err) {
+      console.warn("Parallel worker search failed, falling back to local:", err);
+      return null;
+    }
+  }
+  function isPromoted() {
+    return workerPool !== null;
+  }
+  function getMemoryStats() {
+    const workers = workerPool ? workerPool.getMemoryStats().map((s) => ({
+      workerId: s.workerId,
+      heapUsed: s.heapUsed,
+      heapTotal: s.heapTotal,
+      external: s.external
+    })) : [];
+    const totalBytes = workers.reduce((sum, w) => sum + w.heapUsed, 0);
+    return { totalBytes, workers };
+  }
+  async function shutdown() {
+    if (workerPool) {
+      await workerPool.shutdown();
+      workerPool = null;
+    }
+  }
+  return { checkPromotion, replicateToWorkers, searchViaWorker, isPromoted, getMemoryStats, shutdown };
+}
 
 // src/highlighting/highlighter.ts
 function buildTokenCharOffsets(text, originalTokens) {
@@ -139,222 +543,66 @@ function highlightField(text, queryTokens, language, options) {
   return { snippet: result, positions: mergedPositions };
 }
 
-// src/persistence/flush-manager.ts
-var DEFAULT_INTERVAL = 5e3;
-var DEFAULT_MUTATION_THRESHOLD = 1e3;
-var DEFAULT_MAX_RETRIES = 3;
-var DEFAULT_BASE_RETRY_DELAY = 100;
-function createFlushManager(config, getPartitionData, getInstanceId) {
-  const {
-    persistence,
-    invalidation,
-    interval = DEFAULT_INTERVAL,
-    mutationThreshold = DEFAULT_MUTATION_THRESHOLD,
-    maxRetries = DEFAULT_MAX_RETRIES,
-    baseRetryDelay = DEFAULT_BASE_RETRY_DELAY,
-    onError
-  } = config;
-  const dirtyPartitions = /* @__PURE__ */ new Map();
-  let mutationCount = 0;
-  let flushTimerId = null;
-  let isShuttingDown = false;
-  let flushPromise = null;
-  function startTimer() {
-    if (flushTimerId !== null || isShuttingDown) {
-      return;
+// src/search/grouping.ts
+function getNestedValue(obj, path) {
+  const segments = path.split(".");
+  let current = obj;
+  for (const segment of segments) {
+    if (current === null || current === void 0 || typeof current !== "object") {
+      return void 0;
     }
-    flushTimerId = setInterval(() => {
-      flush();
-    }, interval);
-  }
-  function addToDirtySet(indexName, partitionId) {
-    let partitions = dirtyPartitions.get(indexName);
-    if (partitions === void 0) {
-      partitions = /* @__PURE__ */ new Set();
-      dirtyPartitions.set(indexName, partitions);
+    if (!Object.hasOwn(current, segment)) {
+      return void 0;
     }
-    partitions.add(partitionId);
+    current = current[segment];
   }
-  function snapshotAndClearDirty() {
-    const snapshot = /* @__PURE__ */ new Map();
-    for (const [indexName, partitions] of dirtyPartitions) {
-      snapshot.set(indexName, new Set(partitions));
-    }
-    dirtyPartitions.clear();
-    return snapshot;
+  return current;
+}
+function applyGrouping(hits, group, getDocument) {
+  if (!group.fields || group.fields.length === 0) {
+    return [{ values: {}, hits }];
   }
-  async function saveWithRetry(indexName, partitionId) {
-    const key = `${indexName}/${partitionId}`;
-    const data = getPartitionData(indexName, partitionId);
-    for (let attempt = 0; attempt < maxRetries; attempt++) {
-      try {
-        await persistence.save(key, data);
-        return true;
-      } catch (err) {
-        const isLastAttempt = attempt === maxRetries - 1;
-        if (isLastAttempt) {
-          onError?.(indexName, partitionId, err, true);
-          return false;
-        }
-        const cappedAttempt = Math.min(attempt, 20);
-        const delay = baseRetryDelay * 2 ** cappedAttempt * (0.5 + Math.random() * 0.5);
-        await new Promise((resolve) => setTimeout(resolve, delay));
-      }
+  const groupMap = /* @__PURE__ */ new Map();
+  for (const hit of hits) {
+    const doc = getDocument(hit.id);
+    const fieldValues = [];
+    const valuesRecord = {};
+    for (const field of group.fields) {
+      const value = doc ? getNestedValue(doc, field) : void 0;
+      fieldValues.push(value);
+      valuesRecord[field] = value;
+    }
+    const compositeKey = JSON.stringify(fieldValues);
+    const existing = groupMap.get(compositeKey);
+    if (existing) {
+      existing.hits.push(hit);
+    } else {
+      groupMap.set(compositeKey, { values: valuesRecord, hits: [hit] });
     }
-    return false;
   }
-  async function performFlush() {
-    const snapshot = snapshotAndClearDirty();
-    for (const [indexName, partitions] of snapshot) {
-      const succeededPartitions = [];
-      for (const partitionId of partitions) {
-        const saved = await saveWithRetry(indexName, partitionId);
-        if (saved) {
-          succeededPartitions.push(partitionId);
-        } else {
-          addToDirtySet(indexName, partitionId);
+  const groups = [];
+  for (const entry of groupMap.values()) {
+    let groupHits = entry.hits;
+    if (group.maxPerGroup !== void 0 && group.maxPerGroup > 0) {
+      groupHits = groupHits.slice(0, group.maxPerGroup);
+    }
+    const result = { values: entry.values, hits: groupHits };
+    if (group.reduce) {
+      try {
+        let accumulator = group.reduce.initialValue();
+        for (const hit of groupHits) {
+          const doc = getDocument(hit.id);
+          if (doc) {
+            accumulator = group.reduce.reducer(accumulator, doc, hit.score);
+          }
         }
-      }
-      if (succeededPartitions.length > 0) {
-        await invalidation.publish({
-          type: "partition",
-          indexName,
-          partitions: succeededPartitions,
-          timestamp: Date.now(),
-          sourceInstanceId: getInstanceId()
-        });
+        ;
+        result.reduced = accumulator;
+      } catch (err) {
+        result.reducerError = err instanceof Error ? err.message : String(err);
       }
     }
-    mutationCount = 0;
-  }
-  function flush() {
-    if (flushPromise !== null) {
-      return flushPromise;
-    }
-    flushPromise = performFlush().finally(() => {
-      flushPromise = null;
-    });
-    return flushPromise;
-  }
-  function markDirty(indexName, partitionId) {
-    if (isShuttingDown) return;
-    addToDirtySet(indexName, partitionId);
-    mutationCount++;
-    if (mutationCount >= mutationThreshold) {
-      flush();
-    }
-    startTimer();
-  }
-  async function shutdown() {
-    isShuttingDown = true;
-    if (flushTimerId !== null) {
-      clearInterval(flushTimerId);
-      flushTimerId = null;
-    }
-    await flush();
-  }
-  return {
-    markDirty,
-    flush,
-    shutdown
-  };
-}
-
-// src/plugins/registry.ts
-function isThenable(value) {
-  return value !== null && typeof value === "object" && typeof value.then === "function";
-}
-async function continueAsync(pending, hooks, startIndex, context) {
-  await pending;
-  for (let i = startIndex; i < hooks.length; i++) {
-    await hooks[i](context);
-  }
-}
-function createPluginRegistry() {
-  const plugins = [];
-  return {
-    register(plugin) {
-      plugins.push(plugin);
-    },
-    runHook(hookName, context) {
-      const hooks = [];
-      for (const plugin of plugins) {
-        const hook = plugin[hookName];
-        if (typeof hook === "function") {
-          hooks.push(hook.bind(plugin));
-        }
-      }
-      if (hooks.length === 0) return;
-      for (let i = 0; i < hooks.length; i++) {
-        const result = hooks[i](context);
-        if (isThenable(result)) {
-          return continueAsync(result, hooks, i + 1, context);
-        }
-      }
-    }
-  };
-}
-
-// src/search/grouping.ts
-function getNestedValue(obj, path) {
-  const segments = path.split(".");
-  let current = obj;
-  for (const segment of segments) {
-    if (current === null || current === void 0 || typeof current !== "object") {
-      return void 0;
-    }
-    if (!Object.hasOwn(current, segment)) {
-      return void 0;
-    }
-    current = current[segment];
-  }
-  return current;
-}
-function applyGrouping(hits, group, getDocument) {
-  if (!group.fields || group.fields.length === 0) {
-    return [{ values: {}, hits }];
-  }
-  const groupMap = /* @__PURE__ */ new Map();
-  for (const hit of hits) {
-    const doc = getDocument(hit.id);
-    const fieldValues = [];
-    const valuesRecord = {};
-    for (const field of group.fields) {
-      const value = doc ? getNestedValue(doc, field) : void 0;
-      fieldValues.push(value);
-      valuesRecord[field] = value;
-    }
-    const compositeKey = JSON.stringify(fieldValues);
-    const existing = groupMap.get(compositeKey);
-    if (existing) {
-      existing.hits.push(hit);
-    } else {
-      groupMap.set(compositeKey, { values: valuesRecord, hits: [hit] });
-    }
-  }
-  const groups = [];
-  for (const entry of groupMap.values()) {
-    let groupHits = entry.hits;
-    if (group.maxPerGroup !== void 0 && group.maxPerGroup > 0) {
-      groupHits = groupHits.slice(0, group.maxPerGroup);
-    }
-    const result = { values: entry.values, hits: groupHits };
-    if (group.reduce) {
-      try {
-        let accumulator = group.reduce.initialValue();
-        for (const hit of groupHits) {
-          const doc = getDocument(hit.id);
-          if (doc) {
-            accumulator = group.reduce.reducer(accumulator, doc, hit.score);
-          }
-        }
-        ;
-        result.reduced = accumulator;
-      } catch (err) {
-        result.reducerError = err instanceof Error ? err.message : String(err);
-      }
-    }
-    groups.push(result);
+    groups.push(result);
   }
   groups.sort((a, b) => {
     const aScore = a.hits.length > 0 ? a.hits[0].score : 0;
@@ -543,13 +791,583 @@ function applySorting(hits, sort, getDocument) {
       } else if (typeof aVal === "boolean" && typeof bVal === "boolean") {
         comparison = aVal === bVal ? 0 : aVal ? 1 : -1;
       }
-      if (comparison !== 0) {
-        return direction === "desc" ? -comparison : comparison;
+      if (comparison !== 0) {
+        return direction === "desc" ? -comparison : comparison;
+      }
+    }
+    return 0;
+  });
+  return sorted;
+}
+
+// src/engine/validation.ts
+var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
+var MAX_INDEX_NAME_LENGTH = 256;
+var MAX_DOC_ID_LENGTH = 512;
+var BATCH_CHUNK_SIZE = 1e3;
+var MAX_LIMIT = 1e4;
+var MAX_OFFSET = 1e5;
+var DEFAULT_LIMIT = 10;
+var DEFAULT_OFFSET = 0;
+function now() {
+  if (typeof performance !== "undefined" && typeof performance.now === "function") {
+    return performance.now();
+  }
+  return Date.now();
+}
+function validateIndexName(name) {
+  if (!name || name.length === 0) {
+    throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
+  }
+  if (name.length > MAX_INDEX_NAME_LENGTH) {
+    throw new NarsilError(
+      ErrorCodes.INDEX_NOT_FOUND,
+      `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
+      {
+        indexName: name,
+        length: name.length
+      }
+    );
+  }
+  if (!INDEX_NAME_PATTERN.test(name)) {
+    throw new NarsilError(
+      ErrorCodes.INDEX_NOT_FOUND,
+      `Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
+      { indexName: name }
+    );
+  }
+  if (name.includes("..")) {
+    throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
+  }
+}
+function validateDocId(docId) {
+  if (!docId || docId.length === 0) {
+    throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
+  }
+  if (docId.length > MAX_DOC_ID_LENGTH) {
+    throw new NarsilError(
+      ErrorCodes.DOC_VALIDATION_FAILED,
+      `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
+      {
+        docId,
+        length: docId.length
+      }
+    );
+  }
+  if (docId.includes("\0")) {
+    throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
+  }
+}
+function clampLimit(limit) {
+  if (limit === void 0) return DEFAULT_LIMIT;
+  return Math.max(0, Math.min(limit, MAX_LIMIT));
+}
+function clampOffset(offset) {
+  if (offset === void 0) return DEFAULT_OFFSET;
+  return Math.max(0, Math.min(offset, MAX_OFFSET));
+}
+
+// src/engine/query.ts
+async function executeQuery(params, context) {
+  const { manager, language, config, workerSearch, indexName } = context;
+  const startTime = now();
+  const limit = clampLimit(params.limit);
+  const offset = clampOffset(params.offset);
+  let fanOutResult;
+  const workerResult = workerSearch ? await workerSearch(indexName, params) : null;
+  if (workerResult) {
+    fanOutResult = workerResult;
+  } else {
+    const searchOptions = {
+      bm25Params: config.bm25,
+      stopWords: config.stopWords,
+      customTokenizer: config.tokenizer
+    };
+    fanOutResult = await fanOutQuery(
+      manager,
+      params,
+      language,
+      config.schema,
+      { scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
+      searchOptions
+    );
+  }
+  const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
+  let hits;
+  if (needsFullHits) {
+    hits = fanOutResult.scored.map((scored) => ({
+      id: scored.docId,
+      score: scored.score,
+      document: void 0,
+      scoreComponents: params.includeScoreComponents ? { termFrequencies: scored.termFrequencies, fieldLengths: scored.fieldLengths, idf: scored.idf } : void 0
+    }));
+  } else {
+    const end = Math.min(offset + limit + 1, fanOutResult.scored.length);
+    hits = new Array(end);
+    for (let i = 0; i < end; i++) {
+      const scored = fanOutResult.scored[i];
+      hits[i] = {
+        id: scored.docId,
+        score: scored.score,
+        document: void 0,
+        scoreComponents: params.includeScoreComponents ? { termFrequencies: scored.termFrequencies, fieldLengths: scored.fieldLengths, idf: scored.idf } : void 0
+      };
+    }
+  }
+  if (params.sort) {
+    hits = applySorting(hits, params.sort, (docId) => manager.getRef(docId));
+  }
+  let groups;
+  if (params.group) {
+    groups = applyGrouping(hits, params.group, (docId) => manager.getRef(docId));
+  }
+  if (params.pinned) {
+    hits = applyPinning(hits, params.pinned, (docId) => {
+      const doc = manager.getRef(docId);
+      if (!doc) return void 0;
+      return { id: docId, score: 0, document: doc };
+    });
+  }
+  const { paginated, nextCursor } = applyPagination(hits, limit, offset, params.searchAfter);
+  for (const hit of paginated) {
+    hit.document = manager.get(hit.id) ?? {};
+  }
+  if (groups) {
+    for (const group of groups) {
+      for (const hit of group.hits) {
+        hit.document = manager.get(hit.id) ?? {};
+      }
+    }
+  }
+  if (params.highlight) {
+    applyHighlights(paginated, params, language, config);
+  }
+  const elapsed = now() - startTime;
+  return {
+    hits: paginated,
+    count: fanOutResult.totalMatched,
+    elapsed,
+    cursor: nextCursor,
+    facets: fanOutResult.facets,
+    groups
+  };
+}
+async function executePreflight(params, context) {
+  const { manager, language, config, workerSearch, indexName } = context;
+  const startTime = now();
+  let totalMatched;
+  const workerResult = workerSearch ? await workerSearch(indexName, params) : null;
+  if (workerResult) {
+    totalMatched = workerResult.totalMatched;
+  } else {
+    const searchOptions = {
+      bm25Params: config.bm25,
+      stopWords: config.stopWords,
+      customTokenizer: config.tokenizer
+    };
+    const fanOutResult = await fanOutQuery(
+      manager,
+      params,
+      language,
+      config.schema,
+      { scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
+      searchOptions
+    );
+    totalMatched = fanOutResult.totalMatched;
+  }
+  const elapsed = now() - startTime;
+  return { count: totalMatched, elapsed };
+}
+function applyHighlights(hits, params, language, config) {
+  if (!params.highlight) return;
+  const queryTokenResult = tokenize(params.term ?? "", language, {
+    stem: true,
+    removeStopWords: true,
+    customTokenizer: config.tokenizer,
+    stopWordOverride: config.stopWords
+  });
+  for (const hit of hits) {
+    const highlights = {};
+    for (const field of params.highlight.fields) {
+      const doc = hit.document;
+      const fieldValue = doc[field];
+      if (typeof fieldValue === "string") {
+        highlights[field] = highlightField(fieldValue, queryTokenResult.tokens, language, {
+          preTag: params.highlight.preTag,
+          postTag: params.highlight.postTag,
+          maxSnippetLength: params.highlight.maxSnippetLength
+        });
+      }
+    }
+    hit.highlights = highlights;
+  }
+}
+
+// src/partitioning/rebalancer.ts
+var CHUNK_SIZE = 1e3;
+function createRebalancer() {
+  let rebalancing = false;
+  async function rebalance(manager, newPartitionCount, router, onProgress) {
+    if (rebalancing) {
+      throw new NarsilError(
+        ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
+        "A rebalance operation is already in progress"
+      );
+    }
+    if (newPartitionCount <= 0) {
+      throw new NarsilError(
+        ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
+        `New partition count must be greater than 0, received ${newPartitionCount}`,
+        { newPartitionCount }
+      );
+    }
+    if (newPartitionCount === manager.partitionCount) {
+      throw new NarsilError(
+        ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
+        `New partition count (${newPartitionCount}) is the same as the current count`,
+        { newPartitionCount, currentPartitionCount: manager.partitionCount }
+      );
+    }
+    rebalancing = true;
+    try {
+      const collectedDocs = [];
+      const currentPartitions = manager.getAllPartitions();
+      for (const partition of currentPartitions) {
+        for (const docId of partition.docIds()) {
+          const document = partition.get(docId);
+          if (document) {
+            collectedDocs.push({ docId, document });
+          }
+        }
+      }
+      const documentsTotal = collectedDocs.length;
+      onProgress?.({
+        phase: "scanning",
+        documentsProcessed: 0,
+        documentsTotal
+      });
+      const newPartitions = [];
+      for (let i = 0; i < newPartitionCount; i++) {
+        newPartitions.push(createPartitionIndex(i));
+      }
+      let documentsProcessed = 0;
+      for (let chunkStart = 0; chunkStart < collectedDocs.length; chunkStart += CHUNK_SIZE) {
+        const chunkEnd = Math.min(chunkStart + CHUNK_SIZE, collectedDocs.length);
+        for (let i = chunkStart; i < chunkEnd; i++) {
+          const { docId, document } = collectedDocs[i];
+          const targetPartitionId = router.route(docId, newPartitionCount);
+          const targetPartition = newPartitions[targetPartitionId];
+          if (targetPartition.has(docId)) {
+            continue;
+          }
+          targetPartition.insert(docId, document, manager.schema, manager.language, { validate: false });
+          documentsProcessed++;
+        }
+        onProgress?.({
+          phase: "moving",
+          documentsProcessed,
+          documentsTotal
+        });
+        await new Promise((resolve) => setTimeout(resolve, 0));
+      }
+      onProgress?.({
+        phase: "swapping",
+        documentsProcessed,
+        documentsTotal
+      });
+      manager.setPartitions(newPartitions);
+      onProgress?.({
+        phase: "complete",
+        documentsProcessed,
+        documentsTotal
+      });
+    } finally {
+      rebalancing = false;
+    }
+  }
+  return {
+    rebalance,
+    isRebalancing() {
+      return rebalancing;
+    }
+  };
+}
+
+// src/partitioning/write-ahead-queue.ts
+function createWriteAheadQueue(maxSize = 1e4) {
+  const entries = [];
+  let nextSequence = 1;
+  return {
+    push(entry) {
+      if (entries.length >= maxSize) {
+        throw new NarsilError(
+          ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
+          `Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
+          { maxSize, currentSize: entries.length }
+        );
+      }
+      const seq = nextSequence++;
+      entries.push({ ...entry, sequenceNumber: seq });
+      return seq;
+    },
+    drain() {
+      const result = entries.splice(0);
+      result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
+      return result;
+    },
+    clear() {
+      entries.length = 0;
+    },
+    get size() {
+      return entries.length;
+    },
+    get isFull() {
+      return entries.length >= maxSize;
+    }
+  };
+}
+
+// src/persistence/flush-manager.ts
+var DEFAULT_INTERVAL = 5e3;
+var DEFAULT_MUTATION_THRESHOLD = 1e3;
+var DEFAULT_MAX_RETRIES = 3;
+var DEFAULT_BASE_RETRY_DELAY = 100;
+function createFlushManager(config, getPartitionData, getInstanceId) {
+  const {
+    persistence,
+    invalidation,
+    interval = DEFAULT_INTERVAL,
+    mutationThreshold = DEFAULT_MUTATION_THRESHOLD,
+    maxRetries = DEFAULT_MAX_RETRIES,
+    baseRetryDelay = DEFAULT_BASE_RETRY_DELAY,
+    onError
+  } = config;
+  const dirtyPartitions = /* @__PURE__ */ new Map();
+  let mutationCount = 0;
+  let flushTimerId = null;
+  let isShuttingDown = false;
+  let flushPromise = null;
+  function startTimer() {
+    if (flushTimerId !== null || isShuttingDown) {
+      return;
+    }
+    flushTimerId = setInterval(() => {
+      flush();
+    }, interval);
+  }
+  function addToDirtySet(indexName, partitionId) {
+    let partitions = dirtyPartitions.get(indexName);
+    if (partitions === void 0) {
+      partitions = /* @__PURE__ */ new Set();
+      dirtyPartitions.set(indexName, partitions);
+    }
+    partitions.add(partitionId);
+  }
+  function snapshotAndClearDirty() {
+    const snapshot = /* @__PURE__ */ new Map();
+    for (const [indexName, partitions] of dirtyPartitions) {
+      snapshot.set(indexName, new Set(partitions));
+    }
+    dirtyPartitions.clear();
+    return snapshot;
+  }
+  async function saveWithRetry(indexName, partitionId) {
+    const key = `${indexName}/${partitionId}`;
+    const data = getPartitionData(indexName, partitionId);
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      try {
+        await persistence.save(key, data);
+        return true;
+      } catch (err) {
+        const isLastAttempt = attempt === maxRetries - 1;
+        if (isLastAttempt) {
+          onError?.(indexName, partitionId, err, true);
+          return false;
+        }
+        const cappedAttempt = Math.min(attempt, 20);
+        const delay = baseRetryDelay * 2 ** cappedAttempt * (0.5 + Math.random() * 0.5);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+    }
+    return false;
+  }
+  async function performFlush() {
+    const snapshot = snapshotAndClearDirty();
+    for (const [indexName, partitions] of snapshot) {
+      const succeededPartitions = [];
+      for (const partitionId of partitions) {
+        const saved = await saveWithRetry(indexName, partitionId);
+        if (saved) {
+          succeededPartitions.push(partitionId);
+        } else {
+          addToDirtySet(indexName, partitionId);
+        }
+      }
+      if (succeededPartitions.length > 0) {
+        await invalidation.publish({
+          type: "partition",
+          indexName,
+          partitions: succeededPartitions,
+          timestamp: Date.now(),
+          sourceInstanceId: getInstanceId()
+        });
+      }
+    }
+    mutationCount = 0;
+  }
+  function flush() {
+    if (flushPromise !== null) {
+      return flushPromise;
+    }
+    flushPromise = performFlush().finally(() => {
+      flushPromise = null;
+    });
+    return flushPromise;
+  }
+  function markDirty(indexName, partitionId) {
+    if (isShuttingDown) return;
+    addToDirtySet(indexName, partitionId);
+    mutationCount++;
+    if (mutationCount >= mutationThreshold) {
+      flush();
+    }
+    startTimer();
+  }
+  async function shutdown() {
+    isShuttingDown = true;
+    if (flushTimerId !== null) {
+      clearInterval(flushTimerId);
+      flushTimerId = null;
+    }
+    await flush();
+  }
+  return {
+    markDirty,
+    flush,
+    shutdown
+  };
+}
+
+// src/plugins/registry.ts
+function isThenable(value) {
+  return value !== null && typeof value === "object" && typeof value.then === "function";
+}
+async function continueAsync(pending, hooks, startIndex, context) {
+  await pending;
+  for (let i = startIndex; i < hooks.length; i++) {
+    await hooks[i](context);
+  }
+}
+function createPluginRegistry() {
+  const plugins = [];
+  return {
+    register(plugin) {
+      plugins.push(plugin);
+    },
+    hasHooks(hookName) {
+      return plugins.some((p) => typeof p[hookName] === "function");
+    },
+    runHook(hookName, context) {
+      const hooks = [];
+      for (const plugin of plugins) {
+        const hook = plugin[hookName];
+        if (typeof hook === "function") {
+          hooks.push(hook.bind(plugin));
+        }
+      }
+      if (hooks.length === 0) return;
+      for (let i = 0; i < hooks.length; i++) {
+        const result = hooks[i](context);
+        if (isThenable(result)) {
+          return continueAsync(result, hooks, i + 1, context);
+        }
       }
     }
-    return 0;
-  });
-  return sorted;
+  };
+}
+var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
+function validateHnswMetric(value) {
+  if (value === void 0 || value === null) return void 0;
+  if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
+    return value;
+  }
+  return void 0;
+}
+function wireToPartition(raw) {
+  const documents = {};
+  for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
+    documents[docId] = {
+      fields: doc.fields,
+      fieldLengths: doc.field_lengths ?? {}
+    };
+  }
+  const invertedIndex = {};
+  for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
+    invertedIndex[token] = {
+      docFrequency: list.doc_freq,
+      postings: (list.postings ?? []).map((p) => ({
+        docId: p.doc_id,
+        termFrequency: p.term_freq,
+        field: p.field,
+        positions: p.positions ?? []
+      }))
+    };
+  }
+  const numeric = {};
+  for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
+    numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
+  }
+  const boolean = {};
+  for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
+    boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
+  }
+  const geopoint = {};
+  for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
+    geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
+  }
+  const vectorData = {};
+  for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
+    vectorData[field] = {
+      dimension: data.dimension,
+      vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
+      hnswGraph: data.hnsw_graph ? {
+        entryPoint: data.hnsw_graph.entry_point,
+        maxLayer: data.hnsw_graph.max_layer,
+        m: data.hnsw_graph.m,
+        efConstruction: data.hnsw_graph.ef_construction,
+        metric: validateHnswMetric(data.hnsw_graph.metric),
+        nodes: data.hnsw_graph.nodes
+      } : null
+    };
+  }
+  return {
+    indexName: raw.index_name,
+    partitionId: raw.partition_id,
+    totalPartitions: raw.total_partitions,
+    language: raw.language ?? "english",
+    schema: raw.schema ?? {},
+    docCount: raw.doc_count ?? 0,
+    avgDocLength: raw.avg_doc_length ?? 0,
+    documents,
+    invertedIndex,
+    fieldIndexes: {
+      numeric,
+      boolean,
+      enum: raw.field_indexes?.enum ?? {},
+      geopoint
+    },
+    vectorData,
+    statistics: {
+      totalDocuments: raw.statistics?.total_documents ?? 0,
+      totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
+      averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
+      docFrequencies: raw.statistics?.doc_frequencies ?? {}
+    }
+  };
+}
+function deserializePayloadV1(data) {
+  const raw = decode(data);
+  return wireToPartition(raw);
 }
 
 // src/workers/promoter.ts
@@ -591,65 +1409,6 @@ function createExecutionPromoter(config) {
 }
 
 // src/narsil.ts
-var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
-var MAX_INDEX_NAME_LENGTH = 256;
-var MAX_DOC_ID_LENGTH = 512;
-var BATCH_CHUNK_SIZE = 1e3;
-var MAX_LIMIT = 1e4;
-var MAX_OFFSET = 1e5;
-var DEFAULT_LIMIT = 10;
-var DEFAULT_OFFSET = 0;
-function now() {
-  if (typeof performance !== "undefined" && typeof performance.now === "function") {
-    return performance.now();
-  }
-  return Date.now();
-}
-function validateIndexName(name) {
-  if (!name || name.length === 0) {
-    throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
-  }
-  if (name.length > MAX_INDEX_NAME_LENGTH) {
-    throw new NarsilError(
-      ErrorCodes.INDEX_NOT_FOUND,
-      `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
-      { indexName: name, length: name.length }
-    );
-  }
-  if (!INDEX_NAME_PATTERN.test(name)) {
-    throw new NarsilError(
-      ErrorCodes.INDEX_NOT_FOUND,
-      `Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
-      { indexName: name }
-    );
-  }
-  if (name.includes("..")) {
-    throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
-  }
-}
-function validateDocId(docId) {
-  if (!docId || docId.length === 0) {
-    throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
-  }
-  if (docId.length > MAX_DOC_ID_LENGTH) {
-    throw new NarsilError(
-      ErrorCodes.DOC_VALIDATION_FAILED,
-      `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
-      { docId, length: docId.length }
-    );
-  }
-  if (docId.includes("\0")) {
-    throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
-  }
-}
-function clampLimit(limit) {
-  if (limit === void 0) return DEFAULT_LIMIT;
-  return Math.max(0, Math.min(limit, MAX_LIMIT));
-}
-function clampOffset(offset) {
-  if (offset === void 0) return DEFAULT_OFFSET;
-  return Math.max(0, Math.min(offset, MAX_OFFSET));
-}
 async function createNarsil(config) {
   const executor = createDirectExecutor();
   const promoter = createExecutionPromoter({
@@ -687,6 +1446,21 @@ async function createNarsil(config) {
   const indexRegistry = /* @__PURE__ */ new Map();
   const eventHandlers = /* @__PURE__ */ new Map();
   let isShutdown = false;
+  const orchestrator = createWorkerOrchestrator(config, executor, promoter, indexRegistry, {
+    onPromotion(workerCount, reason) {
+      const handlers = eventHandlers.get("workerPromote");
+      if (handlers) {
+        for (const handler of handlers) {
+          handler({ workerCount, reason });
+        }
+      }
+    }
+  });
+  const rebalancer = createRebalancer();
+  const rebalanceRouter = createPartitionRouter();
+  const rebalancingIndexes = /* @__PURE__ */ new Set();
+  const waqMap = /* @__PURE__ */ new Map();
+  const lastAppliedSeqMap = /* @__PURE__ */ new Map();
   function guardShutdown() {
     if (isShutdown) {
       throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "This Narsil instance has been shut down");
@@ -699,6 +1473,20 @@ async function createNarsil(config) {
     }
     return entry;
   }
+  function bufferIfRebalancing(indexName, entry) {
+    if (!rebalancingIndexes.has(indexName)) return false;
+    const waq = waqMap.get(indexName);
+    if (!waq) return false;
+    waq.push(entry);
+    return true;
+  }
+  function requireManager(indexName) {
+    const manager = executor.getManager(indexName);
+    if (!manager) {
+      throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" manager not found`, { indexName });
+    }
+    return manager;
+  }
   const narsil = {
     async createIndex(name, indexConfig) {
       guardShutdown();
@@ -717,10 +1505,7 @@ async function createNarsil(config) {
       const entry = requireIndex(name);
       executor.dropIndex(name);
       indexRegistry.delete(name);
-      await pluginRegistry.runHook("onIndexDrop", {
-        indexName: name,
-        config: entry.config
-      });
+      await pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
     },
     listIndexes() {
       const infos = [];
@@ -739,61 +1524,90 @@ async function createNarsil(config) {
       guardShutdown();
       const entry = requireIndex(indexName);
       const manager = executor.getManager(indexName);
+      const estimatedMemory = manager?.estimateMemoryBytes() ?? 0;
       return {
         documentCount: manager?.countDocuments() ?? 0,
         partitionCount: manager?.partitionCount ?? 0,
-        memoryBytes: 0,
-        indexSizeBytes: 0,
+        memoryBytes: estimatedMemory,
+        indexSizeBytes: estimatedMemory,
         language: entry.language.name,
         schema: entry.config.schema
       };
     },
-    async insert(indexName, document, docId) {
+    getPartitionStats(indexName) {
+      guardShutdown();
+      requireIndex(indexName);
+      const manager = executor.getManager(indexName);
+      return manager?.getPartitionStats() ?? [];
+    },
+    async insert(indexName, document, docId, options) {
       guardShutdown();
       requireIndex(indexName);
       const resolvedDocId = docId ?? idGenerator();
       validateDocId(resolvedDocId);
-      await pluginRegistry.runHook("beforeInsert", {
-        indexName,
-        docId: resolvedDocId,
-        document
-      });
+      if (bufferIfRebalancing(indexName, { action: "insert", docId: resolvedDocId, document, indexName })) {
+        return resolvedDocId;
+      }
+      await pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document });
       await executor.execute({
         type: "insert",
         indexName,
         docId: resolvedDocId,
         document,
-        requestId: resolvedDocId
+        requestId: resolvedDocId,
+        skipClone: options?.skipClone
       });
       try {
-        await pluginRegistry.runHook("afterInsert", {
-          indexName,
-          docId: resolvedDocId,
-          document
-        });
+        await pluginRegistry.runHook("afterInsert", { indexName, docId: resolvedDocId, document });
       } catch (err) {
         console.warn("afterInsert plugin hook error:", err);
       }
       flushManager?.markDirty(indexName, 0);
-      const indexMap = /* @__PURE__ */ new Map();
-      for (const [name] of indexRegistry) {
-        const mgr = executor.getManager(name);
-        indexMap.set(name, { documentCount: mgr?.countDocuments() ?? 0 });
-      }
-      promoter.check(indexMap);
+      await orchestrator.replicateToWorkers({
+        type: "insert",
+        indexName,
+        docId: resolvedDocId,
+        document,
+        requestId: `replicate-insert-${resolvedDocId}`,
+        skipClone: options?.skipClone
+      });
+      await orchestrator.checkPromotion();
       return resolvedDocId;
     },
-    async insertBatch(indexName, documents) {
+    async insertBatch(indexName, documents, options) {
       guardShutdown();
       requireIndex(indexName);
       const succeeded = [];
       const failed = [];
+      const hasBeforeHook = pluginRegistry.hasHooks("beforeInsert");
+      const hasAfterHook = pluginRegistry.hasHooks("afterInsert");
       for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
         const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
         for (let i = chunkStart; i < chunkEnd; i++) {
           const docId = idGenerator();
           try {
-            await narsil.insert(indexName, documents[i], docId);
+            validateDocId(docId);
+            if (hasBeforeHook) {
+              await pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
+            }
+            const result = executor.execute({
+              type: "insert",
+              indexName,
+              docId,
+              document: documents[i],
+              requestId: docId,
+              skipClone: options?.skipClone
+            });
+            if (result && typeof result.then === "function") {
+              await result;
+            }
+            if (hasAfterHook) {
+              try {
+                await pluginRegistry.runHook("afterInsert", { indexName, docId, document: documents[i] });
+              } catch (err) {
+                console.warn("afterInsert plugin hook error:", err);
+              }
+            }
             succeeded.push(docId);
           } catch (err) {
             failed.push({
@@ -806,25 +1620,31 @@ async function createNarsil(config) {
           await new Promise((r) => setTimeout(r, 0));
         }
       }
+      flushManager?.markDirty(indexName, 0);
+      await orchestrator.checkPromotion();
       return { succeeded, failed };
     },
     async remove(indexName, docId) {
       guardShutdown();
       requireIndex(indexName);
       validateDocId(docId);
+      if (bufferIfRebalancing(indexName, { action: "remove", docId, indexName })) {
+        return;
+      }
       await pluginRegistry.runHook("beforeRemove", { indexName, docId });
-      await executor.execute({
-        type: "remove",
-        indexName,
-        docId,
-        requestId: docId
-      });
+      await executor.execute({ type: "remove", indexName, docId, requestId: docId });
       try {
         await pluginRegistry.runHook("afterRemove", { indexName, docId });
       } catch (err) {
         console.warn("afterRemove plugin hook error:", err);
       }
       flushManager?.markDirty(indexName, 0);
+      await orchestrator.replicateToWorkers({
+        type: "remove",
+        indexName,
+        docId,
+        requestId: `replicate-remove-${docId}`
+      });
     },
     async removeBatch(indexName, docIds) {
       guardShutdown();
@@ -854,6 +1674,9 @@ async function createNarsil(config) {
       guardShutdown();
       requireIndex(indexName);
       validateDocId(docId);
+      if (bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
+        return;
+      }
       const manager = executor.getManager(indexName);
       const oldDocument = manager?.get(docId);
       await pluginRegistry.runHook("beforeUpdate", {
@@ -862,13 +1685,7 @@ async function createNarsil(config) {
         oldDocument: oldDocument ?? {},
         newDocument: document
       });
-      await executor.execute({
-        type: "update",
-        indexName,
-        docId,
-        document,
-        requestId: docId
-      });
+      await executor.execute({ type: "update", indexName, docId, document, requestId: docId });
       try {
         await pluginRegistry.runHook("afterUpdate", {
           indexName,
@@ -880,6 +1697,13 @@ async function createNarsil(config) {
         console.warn("afterUpdate plugin hook error:", err);
       }
       flushManager?.markDirty(indexName, 0);
+      await orchestrator.replicateToWorkers({
+        type: "update",
+        indexName,
+        docId,
+        document,
+        requestId: `replicate-update-${docId}`
+      });
     },
     async updateBatch(indexName, updates) {
       guardShutdown();
@@ -935,86 +1759,16 @@ async function createNarsil(config) {
     async query(indexName, params) {
       guardShutdown();
       const entry = requireIndex(indexName);
-      const startTime = now();
-      const limit = clampLimit(params.limit);
-      const offset = clampOffset(params.offset);
+      const manager = requireManager(indexName);
       await pluginRegistry.runHook("beforeSearch", { indexName, params });
-      const manager = executor.getManager(indexName);
-      if (!manager) {
-        throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" manager not found`, { indexName });
-      }
-      const searchOptions = {
-        bm25Params: entry.config.bm25,
-        stopWords: entry.config.stopWords,
-        customTokenizer: entry.config.tokenizer
-      };
-      const fanOutResult = fanOutQuery(
+      const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
+      const result = await executeQuery(params, {
         manager,
-        params,
-        entry.language,
-        entry.config.schema,
-        {
-          scoringMode: params.scoring ?? entry.config.defaultScoring ?? "local"
-        },
-        searchOptions
-      );
-      let hits = fanOutResult.scored.map((scored) => ({
-        id: scored.docId,
-        score: scored.score,
-        document: manager.get(scored.docId) ?? {},
-        scoreComponents: {
-          termFrequencies: scored.termFrequencies,
-          fieldLengths: scored.fieldLengths,
-          idf: scored.idf
-        }
-      }));
-      if (params.highlight) {
-        const queryTokenResult = tokenize(params.term ?? "", entry.language, {
-          stem: true,
-          removeStopWords: true,
-          customTokenizer: entry.config.tokenizer,
-          stopWordOverride: entry.config.stopWords
-        });
-        for (const hit of hits) {
-          const highlights = {};
-          for (const field of params.highlight.fields) {
-            const doc = hit.document;
-            const fieldValue = doc[field];
-            if (typeof fieldValue === "string") {
-              highlights[field] = highlightField(fieldValue, queryTokenResult.tokens, entry.language, {
-                preTag: params.highlight.preTag,
-                postTag: params.highlight.postTag,
-                maxSnippetLength: params.highlight.maxSnippetLength
-              });
-            }
-          }
-          hit.highlights = highlights;
-        }
-      }
-      if (params.sort) {
-        hits = applySorting(hits, params.sort, (docId) => manager.get(docId));
-      }
-      let groups;
-      if (params.group) {
-        groups = applyGrouping(hits, params.group, (docId) => manager.get(docId));
-      }
-      if (params.pinned) {
-        hits = applyPinning(hits, params.pinned, (docId) => {
-          const doc = manager.get(docId);
-          if (!doc) return void 0;
-          return { id: docId, score: 0, document: doc };
-        });
-      }
-      const { paginated, nextCursor } = applyPagination(hits, limit, offset, params.searchAfter);
-      const elapsed = now() - startTime;
-      const result = {
-        hits: paginated,
-        count: fanOutResult.totalMatched,
-        elapsed,
-        cursor: nextCursor,
-        facets: fanOutResult.facets,
-        groups
-      };
+        language: entry.language,
+        config: entry.config,
+        workerSearch,
+        indexName
+      });
       try {
         await pluginRegistry.runHook("afterSearch", {
           indexName,
@@ -1029,36 +1783,233 @@ async function createNarsil(config) {
     async preflight(indexName, params) {
       guardShutdown();
       const entry = requireIndex(indexName);
-      const startTime = now();
-      const manager = executor.getManager(indexName);
-      if (!manager) {
-        throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" manager not found`, { indexName });
-      }
-      const searchOptions = {
-        bm25Params: entry.config.bm25,
-        stopWords: entry.config.stopWords,
-        customTokenizer: entry.config.tokenizer
-      };
-      const fanOutResult = fanOutQuery(
+      const manager = requireManager(indexName);
+      const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
+      return executePreflight(params, {
         manager,
-        params,
-        entry.language,
-        entry.config.schema,
-        {
-          scoringMode: params.scoring ?? entry.config.defaultScoring ?? "local"
-        },
-        searchOptions
-      );
-      const elapsed = now() - startTime;
-      return { count: fanOutResult.totalMatched, elapsed };
+        language: entry.language,
+        config: entry.config,
+        workerSearch,
+        indexName
+      });
+    },
+    async suggest(indexName, params) {
+      guardShutdown();
+      const entry = requireIndex(indexName);
+      const manager = requireManager(indexName);
+      const t0 = performance.now();
+      const limit = Math.max(1, Math.min(params.limit ?? 10, 100));
+      const rawPrefix = params.prefix.trim();
+      if (rawPrefix.length === 0) {
+        return { terms: [], elapsed: performance.now() - t0 };
+      }
+      const unstemmed = tokenize(rawPrefix, entry.language, { stem: false, removeStopWords: false });
+      const lastToken = unstemmed.tokens.length > 0 ? unstemmed.tokens[unstemmed.tokens.length - 1].token : rawPrefix.toLowerCase();
+      if (lastToken.length === 0) {
+        return { terms: [], elapsed: performance.now() - t0 };
+      }
+      const stemmed = entry.language.stemmer ? entry.language.stemmer(lastToken) : lastToken;
+      const prefixes = stemmed !== lastToken ? [lastToken, stemmed] : [lastToken];
+      const partitions = manager.getAllPartitions();
+      const merged = /* @__PURE__ */ new Map();
+      const perPartitionLimit = limit * 2;
+      for (const partition of partitions) {
+        const seen = /* @__PURE__ */ new Set();
+        for (const prefix of prefixes) {
+          const suggestions = partition.suggestTerms(prefix, perPartitionLimit);
+          for (const s of suggestions) {
+            if (seen.has(s.term)) continue;
+            seen.add(s.term);
+            merged.set(s.term, (merged.get(s.term) ?? 0) + s.documentFrequency);
+          }
+        }
+      }
+      const terms = Array.from(merged.entries()).map(([term, documentFrequency]) => ({ term, documentFrequency })).sort((a, b) => b.documentFrequency - a.documentFrequency);
+      if (terms.length > limit) terms.length = limit;
+      return { terms, elapsed: performance.now() - t0 };
+    },
+    async snapshot(indexName) {
+      guardShutdown();
+      const entry = requireIndex(indexName);
+      const manager = requireManager(indexName);
+      const partitionBuffers = [];
+      for (let i = 0; i < manager.partitionCount; i++) {
+        partitionBuffers.push(manager.serializePartitionToBytes(i));
+      }
+      const { encode: encode2 } = await import('@msgpack/msgpack');
+      return encode2({
+        version: 2,
+        schema: entry.config.schema,
+        language: entry.language.name,
+        partitions: partitionBuffers
+      });
+    },
+    async restore(indexName, data) {
+      guardShutdown();
+      if (!(data instanceof Uint8Array)) {
+        throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data must be a Uint8Array");
+      }
+      const { decode: decode2 } = await import('@msgpack/msgpack');
+      const envelope = decode2(data);
+      if (envelope.version !== 1 && envelope.version !== 2) {
+        throw new NarsilError(
+          ErrorCodes.DOC_VALIDATION_FAILED,
+          `Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`,
+          { version: envelope.version }
+        );
+      }
+      if (!envelope.schema || typeof envelope.schema !== "object") {
+        throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing or invalid schema");
+      }
+      if (!envelope.language || typeof envelope.language !== "string") {
+        throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing or invalid language");
+      }
+      if (!Array.isArray(envelope.partitions)) {
+        throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing partitions");
+      }
+      const language = getLanguage(envelope.language);
+      const schema = envelope.schema;
+      validateSchema(schema);
+      if (indexRegistry.has(indexName)) {
+        await narsil.dropIndex(indexName);
+      }
+      const indexConfig = { schema, language: envelope.language };
+      executor.createIndex(indexName, indexConfig, language);
+      indexRegistry.set(indexName, { config: indexConfig, language });
+      try {
+        const manager = requireManager(indexName);
+        while (manager.partitionCount < envelope.partitions.length) {
+          manager.addPartition();
+        }
+        const deserializePartitionPayload = envelope.version === 2 ? deserializePayloadV2 : deserializePayloadV1;
+        for (let i = 0; i < envelope.partitions.length; i++) {
+          const partition = deserializePartitionPayload(envelope.partitions[i]);
+          manager.deserializePartition(i, partition);
+        }
+      } catch (err) {
+        try {
+          executor.dropIndex(indexName);
+          indexRegistry.delete(indexName);
+        } catch (_) {
+        }
+        throw err;
+      }
     },
     async clear(indexName) {
       guardShutdown();
       requireIndex(indexName);
       await executor.execute({ type: "clear", indexName, requestId: indexName });
+      await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
+    },
+    async rebalance(indexName, targetPartitionCount) {
+      guardShutdown();
+      requireIndex(indexName);
+      const manager = requireManager(indexName);
+      if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
+        throw new NarsilError(
+          ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
+          `Target partition count must be a positive integer, got ${targetPartitionCount}`,
+          { targetPartitionCount }
+        );
+      }
+      if (rebalancingIndexes.has(indexName)) {
+        throw new NarsilError(
+          ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
+          `Index "${indexName}" is already being rebalanced`
+        );
+      }
+      if (targetPartitionCount === manager.partitionCount) {
+        return;
+      }
+      const waq = createWriteAheadQueue();
+      waqMap.set(indexName, waq);
+      rebalancingIndexes.add(indexName);
+      try {
+        const oldCount = manager.partitionCount;
+        await rebalancer.rebalance(manager, targetPartitionCount, rebalanceRouter, (progress) => {
+          if (progress.phase === "complete") {
+            const handlers = eventHandlers.get("partitionRebalance");
+            if (handlers) {
+              for (const handler of handlers) {
+                handler({ indexName, oldCount, newCount: targetPartitionCount });
+              }
+            }
+          }
+        });
+        const entries = waq.drain();
+        const appliedSeqs = lastAppliedSeqMap.get(indexName) ?? /* @__PURE__ */ new Map();
+        for (const entry of entries) {
+          const partitionLastSeq = appliedSeqs.get(0) ?? 0;
+          if (entry.sequenceNumber <= partitionLastSeq) continue;
+          try {
+            if (entry.action === "insert" && entry.document) {
+              manager.insert(entry.docId, entry.document);
+            } else if (entry.action === "remove") {
+              manager.remove(entry.docId);
+            } else if (entry.action === "update" && entry.document) {
+              manager.update(entry.docId, entry.document);
+            }
+          } catch (replayErr) {
+            const isDuplicate = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_ALREADY_EXISTS;
+            const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
+            if (!isDuplicate && !isMissing) {
+              console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
+            }
+          }
+          appliedSeqs.set(0, entry.sequenceNumber);
+        }
+        lastAppliedSeqMap.set(indexName, appliedSeqs);
+      } finally {
+        rebalancingIndexes.delete(indexName);
+        waqMap.delete(indexName);
+      }
+    },
+    async updatePartitionConfig(indexName, partitionConfig) {
+      guardShutdown();
+      const entry = requireIndex(indexName);
+      const manager = requireManager(indexName);
+      if (rebalancingIndexes.has(indexName)) {
+        throw new NarsilError(
+          ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
+          `Index "${indexName}" is currently being rebalanced`
+        );
+      }
+      const currentDocCount = manager.countDocuments();
+      const newMaxDocs = partitionConfig.maxDocsPerPartition ?? entry.config.partitions?.maxDocsPerPartition;
+      const newMaxPartitions = partitionConfig.maxPartitions ?? entry.config.partitions?.maxPartitions ?? manager.partitionCount;
+      if (newMaxDocs !== void 0) {
+        const newTotalCapacity = newMaxDocs * newMaxPartitions;
+        if (newTotalCapacity < currentDocCount) {
+          throw new NarsilError(
+            ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
+            `New capacity (${newTotalCapacity}) is less than current document count (${currentDocCount})`,
+            { newTotalCapacity, currentDocCount }
+          );
+        }
+      }
+      if (!entry.config.partitions) {
+        entry.config.partitions = {};
+      }
+      if (partitionConfig.maxDocsPerPartition !== void 0) {
+        entry.config.partitions.maxDocsPerPartition = partitionConfig.maxDocsPerPartition;
+      }
+      if (partitionConfig.maxPartitions !== void 0) {
+        entry.config.partitions.maxPartitions = partitionConfig.maxPartitions;
+      }
     },
     getMemoryStats() {
-      return { totalBytes: 0, workers: [] };
+      const workerStats = orchestrator.getMemoryStats();
+      let mainThreadBytes = 0;
+      for (const [name] of indexRegistry) {
+        const manager = executor.getManager(name);
+        if (manager) {
+          mainThreadBytes += manager.estimateMemoryBytes();
+        }
+      }
+      return {
+        totalBytes: mainThreadBytes + workerStats.totalBytes,
+        workers: workerStats.workers
+      };
     },
     on(event, handler) {
       const key = event;
@@ -1085,6 +2036,7 @@ async function createNarsil(config) {
       if (flushManager) {
         await flushManager.shutdown();
       }
+      await orchestrator.shutdown();
       await executor.shutdown();
       eventHandlers.clear();
       indexRegistry.clear();