@delali/narsil 0.2.3 → 0.2.4-next.43

package/dist/index.browser.mjs

@@ -3,7 +3,6 @@ import { encode, decode } from '@msgpack/msgpack';
 // src/errors.ts
 var ErrorCodes = {
   SCHEMA_INVALID_TYPE: "SCHEMA_INVALID_TYPE",
-  SCHEMA_MISSING_FIELD: "SCHEMA_MISSING_FIELD",
   SCHEMA_DEPTH_EXCEEDED: "SCHEMA_DEPTH_EXCEEDED",
   SCHEMA_INVALID_VECTOR_DIMENSION: "SCHEMA_INVALID_VECTOR_DIMENSION",
   SCHEMA_INVALID_GEOPOINT: "SCHEMA_INVALID_GEOPOINT",
@@ -25,7 +24,6 @@ var ErrorCodes = {
   PERSISTENCE_WAL_CORRUPT: "PERSISTENCE_WAL_CORRUPT",
   PERSISTENCE_FSYNC_FAILED: "PERSISTENCE_FSYNC_FAILED",
   SEARCH_INVALID_FIELD: "SEARCH_INVALID_FIELD",
-  SEARCH_INVALID_VECTOR_SIZE: "SEARCH_INVALID_VECTOR_SIZE",
   VECTOR_DIMENSION_MISMATCH: "VECTOR_DIMENSION_MISMATCH",
   SEARCH_INVALID_FILTER: "SEARCH_INVALID_FILTER",
   SEARCH_INVALID_MODE: "SEARCH_INVALID_MODE",
@@ -49,8 +47,6 @@ var ErrorCodes = {
   ALLOCATION_NO_DATA_NODES: "ALLOCATION_NO_DATA_NODES",
   ALLOCATION_INVALID_CONFIG: "ALLOCATION_INVALID_CONFIG",
   ALLOCATION_FAILED: "ALLOCATION_FAILED",
-  CONTROLLER_LEASE_LOST: "CONTROLLER_LEASE_LOST",
-  CONTROLLER_NOT_ACTIVE: "CONTROLLER_NOT_ACTIVE",
   CONTROLLER_METADATA_INVALID: "CONTROLLER_METADATA_INVALID",
   NODE_BOOTSTRAP_FAILED: "NODE_BOOTSTRAP_FAILED",
   NODE_ALREADY_JOINED: "NODE_ALREADY_JOINED",
@@ -106,6 +102,7 @@ var ServerErrorCodes = {
   TASK_NOT_FOUND: "TASK_NOT_FOUND",
   TASK_NOT_CANCELLABLE: "TASK_NOT_CANCELLABLE",
   TASK_OWNED_BY_ANOTHER_INSTANCE: "TASK_OWNED_BY_ANOTHER_INSTANCE",
+  TASK_INTERRUPTED: "TASK_INTERRUPTED",
   TOO_MANY_REQUESTS: "TOO_MANY_REQUESTS",
   HOOK_ERROR: "HOOK_ERROR",
   INTERNAL_ERROR: "INTERNAL_ERROR"
@@ -2622,7 +2619,7 @@ function wireToMetadata(raw) {
     if (isPositiveInteger2(promotion.threshold)) {
       restored.threshold = promotion.threshold;
     }
-    if (isPositiveInteger2(promotion.filter_threshold)) {
+    if (isSelectivityRatio(promotion.filter_threshold)) {
       restored.filterThreshold = promotion.filter_threshold;
     }
     if (Object.keys(hnswConfig).length > 0) {
@@ -2649,6 +2646,9 @@ function isNonNegativeInteger(value) {
 function isPositiveInteger2(value) {
   return typeof value === "number" && Number.isInteger(value) && value >= 1;
 }
+function isSelectivityRatio(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+}
 function isWatermarkFraction(value) {
   return typeof value === "number" && value > 0 && value <= 1;
 }
@@ -8079,7 +8079,7 @@ function createRebalancer() {
         skipClone: true,
         stopWordOverride: manager.analysis.stopWords,
         customTokenizer: manager.analysis.customTokenizer,
-        collectSurfaces: manager.config.surfaceForms === true
+        collectSurfaces: manager.config.surfaceForms !== false
       };
       for (const partition of currentPartitions) {
         const docIds = [...partition.docIds()];
@@ -8140,8 +8140,7 @@ function createRebalancer() {
 var FNV_OFFSET_BASIS = 2166136261;
 var FNV_PRIME = 16777619;
 var encoder = new TextEncoder();
-function fnv1a(input) {
-  const bytes = encoder.encode(input);
+function fnv1aBytes(bytes) {
   let hash = FNV_OFFSET_BASIS;
   for (let i = 0; i < bytes.length; i++) {
     hash ^= bytes[i];
@@ -8149,6 +8148,9 @@ function fnv1a(input) {
   }
   return hash >>> 0;
 }
+function fnv1a(input) {
+  return fnv1aBytes(encoder.encode(input));
+}
 
 // src/partitioning/router.ts
 function createPartitionRouter() {
@@ -10189,7 +10191,7 @@ function applySorting(hits, sort, sortKeyOf) {
 }
 
 // src/search/cursor.ts
-var CURSOR_VERSION = 2;
+var CURSOR_VERSION = 3;
 var MAX_CURSOR_LENGTH = 40960;
 var MAX_SORT_FIELDS = 8;
 var MAX_SORT_FIELD_NAME_LENGTH = 255;
@@ -10244,6 +10246,7 @@ function encodePageCursor(cursor) {
   } else if (cursor.score !== null) {
     payload.s = cursor.score;
   }
+  payload.q = cursor.binding;
   return encodeCursorText(JSON.stringify(payload));
 }
 function decodeSortKey(value, cursor) {
@@ -10288,8 +10291,11 @@ function decodePageCursor(cursor) {
   if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
     throw invalidCursor(cursor, "expected an object");
   }
-  const { v, a, s, k, o } = parsed;
+  const { v, a, s, k, o, q } = parsed;
   if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
+  if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
+    throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
+  }
   if (typeof a !== "string" || a.length === 0) throw invalidCursor(cursor, '"a" must be a non-empty string');
   if (exceedsCodePoints(a, MAX_ANCHOR_CODE_POINTS)) {
     throw invalidCursor(cursor, '"a" is too long to be a document id');
@@ -10307,10 +10313,14 @@ function decodePageCursor(cursor) {
     anchor: a,
     score: typeof s === "number" ? s : null,
     sortKey,
-    sortSignature: typeof o === "string" ? o : null
+    sortSignature: typeof o === "string" ? o : null,
+    binding: q
   };
 }
-function requireMatchingCursor(cursor, encoded, signature, scoreAnchored) {
+function requireMatchingCursor(cursor, encoded, signature, scoreAnchored, binding) {
+  if (cursor.binding !== binding) {
+    throw invalidCursor(encoded, "the cursor belongs to a different query");
+  }
   if (signature !== null) {
     if (cursor.sortSignature !== signature) {
       throw invalidCursor(encoded, "the cursor belongs to a different sort order");
@@ -10344,6 +10354,12 @@ function requireWithinResultWindow(limit, offset) {
     { limit, offset, window: RESULT_WINDOW }
   );
 }
+function lastOrganicResult(sliced, pinnedIds) {
+  for (let i = sliced.length - 1; i >= 0; i--) {
+    if (pinnedIds === void 0 || !pinnedIds.has(sliced[i].id)) return sliced[i];
+  }
+  return void 0;
+}
 function ordersAfterAnchor(result, anchor, sort) {
   if (sort !== void 0 && anchor.sortKey !== null) {
     const comparison = compareSortValues(sort.sortKeyOf(result.id), anchor.sortKey, sort.directions) || compareCodePoints(result.id, anchor.anchor);
@@ -10354,10 +10370,10 @@ function ordersAfterAnchor(result, anchor, sort) {
   if (result.score < anchor.score) return true;
   return result.score === anchor.score && compareCodePoints(result.id, anchor.anchor) > 0;
 }
-function applyPagination(results, limit, offset, cursor, sort) {
+function applyPagination(results, limit, offset, binding, cursor, sort, pinnedIds) {
   const decoded = cursor ? decodePageCursor(cursor) : null;
   if (decoded !== null && cursor !== void 0) {
-    requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true);
+    requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding);
   }
   if (limit <= 0) {
     return { paginated: [] };
@@ -10376,18 +10392,20 @@ function applyPagination(results, limit, offset, cursor, sort) {
   const sliced = results.slice(afterOffset, afterOffset + limit);
   let nextCursor;
   const hasMore = afterOffset + limit < results.length;
-  if (hasMore && sliced.length > 0) {
-    const lastResult = sliced[sliced.length - 1];
+  const lastResult = lastOrganicResult(sliced, pinnedIds);
+  if (hasMore && lastResult !== void 0) {
     nextCursor = sort !== void 0 ? encodePageCursor({
       anchor: lastResult.id,
       score: null,
       sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
-      sortSignature: sort.signature
+      sortSignature: sort.signature,
+      binding
     }) : encodePageCursor({
       anchor: lastResult.id,
       score: lastResult.score ?? null,
       sortKey: null,
-      sortSignature: null
+      sortSignature: null,
+      binding
     });
   }
   return { paginated: sliced, nextCursor };
@@ -10587,9 +10605,24 @@ function queryTermSatisfied(queryToken, indexTokens, tolerance, exact) {
   return false;
 }
 
+// src/partitioning/partition-selection.ts
+function partitionsIn(manager, partitionIds) {
+  if (partitionIds === void 0) {
+    return manager.getAllPartitions();
+  }
+  const partitions = [];
+  for (const partitionId of partitionIds) {
+    const partition = manager.partitionAt(partitionId);
+    if (partition !== void 0) {
+      partitions.push(partition);
+    }
+  }
+  return partitions;
+}
+
 // src/partitioning/fan-out.ts
 async function fanOutQuery(manager, params, language, schema, config, searchOptions) {
-  const partitions = config.partitionIds ? config.partitionIds.map((id) => manager.partitionAt(id)).filter((p) => p !== void 0) : manager.getAllPartitions();
+  const partitions = partitionsIn(manager, config.partitionIds);
   if (partitions.length === 0) {
     return { scored: [], totalMatched: 0 };
   }
@@ -11018,6 +11051,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
   return manager;
 }
 
+// src/partitioning/match-count.ts
+function countsWithoutScores(params) {
+  return params.minScore === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
+}
+function fanOutMatchCount(manager, params, language, schema, options) {
+  let count = 0;
+  for (const partition of partitionsIn(manager, options?.partitionIds)) {
+    const matches = fulltextMatches(partition, params, language, schema, options?.searchOptions);
+    if (matches !== null) count += matches.count;
+  }
+  return count;
+}
+
 // src/vector/scalar-quantization-calibration.ts
 var PADDING_FACTOR = 0.01;
 function computeCalibrationBounds(vectors, dimensions) {
@@ -11247,7 +11293,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
   let liveCount = 0;
   let constants = deriveSq8Constants(0, 0, dimensions);
   let calibrated = false;
-  function ensureCapacity2(needed) {
+  function ensureCapacity3(needed) {
     if (needed <= capacity) return;
     let newCap = capacity === 0 ? INITIAL_CAPACITY2 : capacity;
     while (newCap < needed) newCap *= 2;
@@ -11377,7 +11423,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
       liveCount = 0;
       for (const [docId, vec] of collected) {
         const ord = resolveOrdinal(docId);
-        ensureCapacity2(ord + 1);
+        ensureCapacity3(ord + 1);
         quantizeInto(ord, vec);
         storeEntry(docId, ord);
       }
@@ -11387,7 +11433,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
         calibrateFromVectors([vector]);
       }
       const ord = resolveOrdinal(docId);
-      ensureCapacity2(ord + 1);
+      ensureCapacity3(ord + 1);
       quantizeInto(ord, vector);
       storeEntry(docId, ord);
     },
@@ -11454,7 +11500,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
     },
     restoreEntry(docId, quantized, sum, sumSq) {
       const ord = resolveOrdinal(docId);
-      ensureCapacity2(ord + 1);
+      ensureCapacity3(ord + 1);
       const base = ord * dimensions;
       quantizedArena.set(quantized.subarray(0, dimensions), base);
       sums[ord] = sum;
@@ -11605,7 +11651,7 @@ function createVectorStore() {
       arena = new Float32Array(simd.memory.buffer);
     }
   }
-  function ensureCapacity2(needed) {
+  function ensureCapacity3(needed) {
     if (needed <= capacity) return;
     let newCap = capacity === 0 ? INITIAL_CAPACITY3 : capacity;
     while (newCap < needed) newCap *= 2;
@@ -11687,7 +11733,7 @@ function createVectorStore() {
         recycledOrds.delete(docId);
         docToOrd.set(docId, recycled);
         ordToDoc[recycled] = docId;
-        ensureCapacity2(recycled + 1);
+        ensureCapacity3(recycled + 1);
         writeVector(recycled, vector);
         recordPartition(recycled, partitionId, false);
         liveCount++;
@@ -11696,7 +11742,7 @@ function createVectorStore() {
       const ord = ordToDoc.length;
       docToOrd.set(docId, ord);
       ordToDoc.push(docId);
-      ensureCapacity2(ord + 1);
+      ensureCapacity3(ord + 1);
       writeVector(ord, vector);
       recordPartition(ord, partitionId, false);
       liveCount++;
@@ -11862,7 +11908,7 @@ function createVectorStore() {
       }
       if (snapshot.dimension === 0 || snapshot.slots === 0) return;
       initStorage(snapshot.dimension);
-      ensureCapacity2(snapshot.slots);
+      ensureCapacity3(snapshot.slots);
       arena.set(snapshot.vectors.subarray(0, snapshot.slots * snapshot.dimension), scratchFloatLength);
       mags.set(snapshot.magnitudes.subarray(0, Math.min(snapshot.slots, mags.length)));
       ensurePartitionSlots(snapshot.slots);
@@ -13015,20 +13061,43 @@ function resolveWorkerCount(requested) {
 }
 function createWorkerPool(config) {
   const workerCount = resolveWorkerCount(config.count);
-  const workers = [];
+  const workers = /* @__PURE__ */ new Map();
+  const deadSlots = /* @__PURE__ */ new Set();
   const indexAssignment = /* @__PURE__ */ new Map();
   let isShutdown = false;
+  function handleWorkerDeath(slotIndex, error) {
+    if (isShutdown || deadSlots.has(slotIndex)) {
+      return;
+    }
+    deadSlots.add(slotIndex);
+    const slot = workers.get(slotIndex);
+    const indexNames = slot ? [...slot.indexes].sort() : [];
+    workers.delete(slotIndex);
+    config.onWorkerCrash?.(slotIndex, indexNames, error);
+  }
   function ensureWorker(slotIndex) {
-    if (!workers[slotIndex]) {
-      workers[slotIndex] = {
-        executor: config.workerFactory(slotIndex),
+    if (deadSlots.has(slotIndex)) {
+      return void 0;
+    }
+    let slot = workers.get(slotIndex);
+    if (!slot) {
+      slot = {
+        executor: config.workerFactory(slotIndex, (error) => handleWorkerDeath(slotIndex, error)),
         indexes: /* @__PURE__ */ new Set()
       };
+      workers.set(slotIndex, slot);
     }
-    return workers[slotIndex];
+    return slot;
   }
   function assignSlot(indexName) {
-    return fnv1a(indexName) % workerCount;
+    const preferred = fnv1a(indexName) % workerCount;
+    for (let probe = 0; probe < workerCount; probe++) {
+      const candidate = (preferred + probe) % workerCount;
+      if (!deadSlots.has(candidate)) {
+        return candidate;
+      }
+    }
+    throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker in the pool has crashed`);
   }
   function addIndex(indexName) {
     if (isShutdown) {
@@ -13039,6 +13108,9 @@ function createWorkerPool(config) {
     }
     const slotIndex = assignSlot(indexName);
     const slot = ensureWorker(slotIndex);
+    if (!slot) {
+      throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Worker ${slotIndex} for index "${indexName}" has crashed`);
+    }
     slot.indexes.add(indexName);
     indexAssignment.set(indexName, slotIndex);
   }
@@ -13047,10 +13119,7 @@ function createWorkerPool(config) {
     if (slotIndex === void 0) {
       return;
     }
-    const slot = workers[slotIndex];
-    if (slot) {
-      slot.indexes.delete(indexName);
-    }
+    workers.get(slotIndex)?.indexes.delete(indexName);
     indexAssignment.delete(indexName);
   }
   function getExecutor(indexName) {
@@ -13061,15 +13130,22 @@ function createWorkerPool(config) {
     if (slotIndex === void 0) {
       throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
     }
-    return workers[slotIndex].executor;
+    const slot = workers.get(slotIndex);
+    if (slot) {
+      return slot.executor;
+    }
+    for (const survivor of workers.values()) {
+      if (survivor.indexes.has(indexName)) {
+        return survivor.executor;
+      }
+    }
+    throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker holding index "${indexName}" has crashed`);
   }
   async function getMemoryStats() {
     const indices = [];
     const pending = [];
-    for (let i = 0; i < workers.length; i++) {
-      const slot = workers[i];
-      if (!slot) continue;
-      indices.push(i);
+    for (const [slotIndex, slot] of workers) {
+      indices.push(slotIndex);
       pending.push(
         slot.executor.execute({ type: "memoryReport", requestId: createRequestId() }).catch(() => null)
       );
@@ -13092,7 +13168,7 @@ function createWorkerPool(config) {
       return;
     }
     isShutdown = true;
-    const shutdownPromises = workers.filter(Boolean).map((slot) => {
+    const shutdownPromises = [...workers.values()].map((slot) => {
       const timeoutPromise = new Promise((resolve) => {
         setTimeout(resolve, 5e3);
       });
@@ -13102,17 +13178,16 @@ function createWorkerPool(config) {
     indexAssignment.clear();
   }
   function getAllExecutors() {
-    return workers.filter(Boolean).map((slot) => slot.executor);
+    return [...workers.values()].map((slot) => slot.executor);
   }
   function addIndexToAll(indexName) {
     if (isShutdown) {
       throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
     }
     for (let i = 0; i < workerCount; i++) {
-      const slot = ensureWorker(i);
-      slot.indexes.add(indexName);
+      ensureWorker(i)?.indexes.add(indexName);
     }
-    indexAssignment.set(indexName, 0);
+    indexAssignment.set(indexName, assignSlot(indexName));
   }
   return {
     getExecutor,
@@ -14365,6 +14440,12 @@ function createDirectExecutor() {
       }
       case "preflight": {
         const entry = requireIndex(action.indexName);
+        if (countsWithoutScores(action.params)) {
+          const count = fanOutMatchCount(entry.manager, action.params, entry.language, entry.config.schema, {
+            searchOptions: entry.searchOptions
+          });
+          return { count };
+        }
         const result = await fanOutQuery(
           entry.manager,
           action.params,
@@ -15178,6 +15259,9 @@ function writeHeader(header) {
   }
   return buffer;
 }
+function hasNrslMagic(data) {
+  return data.length >= MAGIC_BYTES.length && data[0] === MAGIC_BYTES[0] && data[1] === MAGIC_BYTES[1] && data[2] === MAGIC_BYTES[2] && data[3] === MAGIC_BYTES[3];
+}
 function readHeader(data) {
   if (data.length < HEADER_SIZE) {
     throw new NarsilError(
@@ -15213,6 +15297,7 @@ function readHeader(data) {
 // src/serialization/envelope.ts
 var CURRENT_ENVELOPE_VERSION = 1;
 var SNAPSHOT_ENVELOPE_VERSION = 2;
+var INDEX_SNAPSHOT_ENVELOPE_VERSION = 3;
 function parseEngineVersion(version) {
   const parts = version.split(".");
   return [parseInt(parts[0] ?? "0", 10), parseInt(parts[1] ?? "0", 10), parseInt(parts[2] ?? "0", 10)];
@@ -15357,6 +15442,19 @@ function concatEnvelopeParts(parts) {
 async function unpackEnvelopeBytes(data) {
   return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
 }
+async function packIndexSnapshotEnvelope(payloadBytes) {
+  return packEnvelope(payloadBytes, { checksum: true, envelopeFormatVersion: INDEX_SNAPSHOT_ENVELOPE_VERSION });
+}
+async function unpackIndexSnapshotEnvelope(data) {
+  const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
+  if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
+    throw new NarsilError(
+      ErrorCodes.ENVELOPE_VERSION_MISMATCH,
+      `Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
+    );
+  }
+  return payloadBytes;
+}
 async function writeMetadataEnvelope(meta, options = {}) {
   const payloadBytes = serializeMetadata(meta);
   return packEnvelope(payloadBytes, options);
@@ -18933,14 +19031,34 @@ async function awaitCompactions(state) {
 var DEFAULT_BACKPRESSURE_LIMIT = 100;
 var DEFAULT_REQUEST_TIMEOUT = 3e4;
 var SHUTDOWN_TIMEOUT = 5e3;
+var CONSECUTIVE_TIMEOUTS_BEFORE_DEATH = 3;
+function errorFromEventLike(event) {
+  if (event instanceof Error) {
+    return event;
+  }
+  if (typeof event === "object" && event !== null) {
+    const detail = event;
+    if (detail.error instanceof Error) {
+      return detail.error;
+    }
+    if (typeof detail.message === "string" && detail.message.length > 0) {
+      return new Error(detail.message);
+    }
+  }
+  return new Error("The worker reported an error event carrying no detail");
+}
 function createWorkerExecutor(worker, config) {
-  const backpressureLimit = DEFAULT_BACKPRESSURE_LIMIT;
-  const requestTimeout = DEFAULT_REQUEST_TIMEOUT;
+  const backpressureLimit = config?.backpressureLimit ?? DEFAULT_BACKPRESSURE_LIMIT;
+  const requestTimeout = config?.requestTimeout ?? DEFAULT_REQUEST_TIMEOUT;
   const pending = /* @__PURE__ */ new Map();
+  let deathError = null;
+  let shutdownRequested = false;
+  let consecutiveTimeouts = 0;
   function processResponse(msg) {
     if (!isValidWorkerResponse(msg)) {
       return;
     }
+    consecutiveTimeouts = 0;
     const response = msg;
     const entry = pending.get(response.requestId);
     if (!entry) {
@@ -18954,15 +19072,36 @@ function createWorkerExecutor(worker, config) {
       entry.resolve(response.data);
     }
   }
+  function handleDeath(cause) {
+    if (deathError !== null) {
+      return;
+    }
+    const message = cause instanceof Error ? cause.message : String(cause);
+    deathError = new NarsilError(ErrorCodes.WORKER_CRASHED, `Worker died: ${message}`);
+    for (const [id, entry] of pending) {
+      clearTimeout(entry.timeoutId);
+      entry.reject(deathError);
+      pending.delete(id);
+    }
+    if (!shutdownRequested) {
+      config?.onDeath?.(cause instanceof Error ? cause : new Error(message));
+    }
+  }
   if (typeof worker.on === "function") {
     worker.on("message", (msg) => processResponse(msg));
+    worker.on("error", (cause) => handleDeath(cause));
+    worker.on("exit", (code) => handleDeath(new Error(`Worker exited with code ${String(code)}`)));
   } else if (typeof worker.addEventListener === "function") {
     worker.addEventListener("message", (event) => {
       const msg = event.data;
       processResponse(msg);
     });
+    worker.addEventListener("error", (event) => handleDeath(errorFromEventLike(event)));
   }
   function execute(action) {
+    if (deathError !== null) {
+      return Promise.reject(deathError);
+    }
     if (pending.size >= backpressureLimit) {
       return Promise.reject(
         new NarsilError(ErrorCodes.WORKER_BUSY, `Backpressure limit of ${backpressureLimit} pending requests reached`)
@@ -18974,6 +19113,10 @@ function createWorkerExecutor(worker, config) {
       const timeoutId = setTimeout(() => {
         pending.delete(requestId);
         reject2(new NarsilError(ErrorCodes.WORKER_TIMEOUT, `Request ${requestId} timed out after ${requestTimeout}ms`));
+        consecutiveTimeouts += 1;
+        if (consecutiveTimeouts >= CONSECUTIVE_TIMEOUTS_BEFORE_DEATH) {
+          handleDeath(new Error(`no answer to ${CONSECUTIVE_TIMEOUTS_BEFORE_DEATH} consecutive requests`));
+        }
       }, requestTimeout);
       pending.set(requestId, {
         resolve,
@@ -18984,6 +19127,10 @@ function createWorkerExecutor(worker, config) {
     });
   }
   async function shutdown() {
+    shutdownRequested = true;
+    if (deathError !== null) {
+      return;
+    }
     const requestId = createRequestId();
     const shutdownAction = { type: "shutdown", requestId };
     const shutdownPromise = new Promise((resolve, reject2) => {
@@ -19024,9 +19171,9 @@ async function createWorkerFactory(entryPoint) {
   const runtime = detectRuntime();
   const resolvedEntry = resolveEntryPoint();
   if (runtime.supportsWebWorkers) {
-    return function webFactory(_workerId) {
+    return function webFactory(_workerId, onDeath) {
       const instance = new Worker(resolvedEntry, { type: "module" });
-      return createWorkerExecutor(instance);
+      return createWorkerExecutor(instance, { onDeath });
     };
   }
   throw new NarsilError(ErrorCodes.WORKER_CRASHED, `No worker support available on runtime "${runtime.runtime}"`);
@@ -19082,7 +19229,13 @@ async function runPromotion(state, reason) {
       return;
     }
     const factory = await createWorkerFactory();
-    const pool = createWorkerPool({ count: state.config?.workers?.count, workerFactory: factory });
+    const pool = createWorkerPool({
+      count: state.config?.workers?.count,
+      workerFactory: factory,
+      onWorkerCrash(workerId, indexNames, error) {
+        state.callbacks?.onWorkerCrash?.(workerId, indexNames, error);
+      }
+    });
     for (const name of promotable) {
       pool.addIndexToAll(name);
     }
@@ -19403,6 +19556,9 @@ function createEngineCore(config) {
       if (handlers) {
         for (const handler of handlers) handler({ workerCount, reason });
       }
+      void Promise.resolve(pluginRegistry.runHook("onWorkerPromote", { workerCount, reason })).catch((err) => {
+        console.warn("onWorkerPromote plugin hook failed:", err instanceof Error ? err.message : String(err));
+      });
     },
     onPromotionFailure(reason, error, retryable) {
       const handlers = eventHandlers.get("workerPromoteFailure");
@@ -19411,6 +19567,14 @@ function createEngineCore(config) {
         return;
       }
       for (const handler of handlers) handler({ reason, error, retryable });
+    },
+    onWorkerCrash(workerId, indexNames, error) {
+      const handlers = eventHandlers.get("workerCrash");
+      if (!handlers || handlers.size === 0) {
+        console.warn(`Worker ${workerId} crashed:`, error);
+        return;
+      }
+      for (const handler of handlers) handler({ workerId, indexNames, error });
     }
   });
   const rebalancer = createRebalancer();
@@ -21370,6 +21534,15 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
         }
       }
     });
+    try {
+      await ctx.pluginRegistry.runHook("onPartitionSplit", {
+        indexName,
+        oldPartitionCount: oldCount,
+        newPartitionCount: targetPartitionCount
+      });
+    } catch (hookErr) {
+      console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
+    }
     await replayQueued(manager, indexName, waq, ctx);
     if (ctx.durabilityManager) {
       await ctx.durabilityManager.persistMetadata(indexName);
@@ -21584,7 +21757,7 @@ async function createSnapshot(manager, entry) {
   const config = entry.config;
   const bm25 = config.bm25;
   const { encode: encode14 } = await import('@msgpack/msgpack');
-  return encode14({
+  const payload = encode14({
     version: 2,
     schema: config.schema,
     language: entry.language.name,
@@ -21614,15 +21787,17 @@ async function createSnapshot(manager, entry) {
     partitions: partitionBuffers,
     vectorIndexes: vectorPayloads
   });
+  return packIndexSnapshotEnvelope(payload);
 }
 async function restoreFromSnapshot(indexName, data, deps) {
   if (!(data instanceof Uint8Array)) {
     throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data must be a Uint8Array");
   }
+  const payloadBytes = hasNrslMagic(data) ? await unpackIndexSnapshotEnvelope(data) : data;
   const { decode: decode14 } = await import('@msgpack/msgpack');
   let decoded;
   try {
-    decoded = decode14(data);
+    decoded = decode14(payloadBytes);
   } catch (err) {
     throw new NarsilError(
       ErrorCodes.DOC_VALIDATION_FAILED,
@@ -21817,6 +21992,137 @@ function readProcessMemory() {
   }
 }
 
+// src/search/cursor-binding.ts
+var ABSENT_TAG = 0;
+var NULL_TAG = 1;
+var FALSE_TAG = 2;
+var TRUE_TAG = 3;
+var NUMBER_TAG = 4;
+var STRING_TAG = 5;
+var LIST_TAG = 6;
+var MAP_TAG = 7;
+var encoder4 = new TextEncoder();
+function ensureCapacity2(stream, extra) {
+  if (stream.length + extra <= stream.bytes.length) return;
+  const grown = new Uint8Array(Math.max(stream.bytes.length * 2, stream.length + extra));
+  grown.set(stream.bytes.subarray(0, stream.length));
+  stream.bytes = grown;
+}
+function writeByte(stream, value) {
+  ensureCapacity2(stream, 1);
+  stream.bytes[stream.length] = value;
+  stream.length += 1;
+}
+function writeUint32(stream, value) {
+  ensureCapacity2(stream, 4);
+  stream.bytes[stream.length] = value >>> 24 & 255;
+  stream.bytes[stream.length + 1] = value >>> 16 & 255;
+  stream.bytes[stream.length + 2] = value >>> 8 & 255;
+  stream.bytes[stream.length + 3] = value & 255;
+  stream.length += 4;
+}
+var numberBytes = new DataView(new ArrayBuffer(8));
+function writeNumber(stream, value) {
+  numberBytes.setFloat64(0, value, false);
+  ensureCapacity2(stream, 8);
+  for (let i = 0; i < 8; i++) {
+    stream.bytes[stream.length + i] = numberBytes.getUint8(i);
+  }
+  stream.length += 8;
+}
+function writeString(stream, value) {
+  const encoded = encoder4.encode(value);
+  writeUint32(stream, encoded.length);
+  ensureCapacity2(stream, encoded.length);
+  stream.bytes.set(encoded, stream.length);
+  stream.length += encoded.length;
+}
+function writeValue(stream, value) {
+  if (value === void 0) {
+    writeByte(stream, ABSENT_TAG);
+    return;
+  }
+  if (value === null) {
+    writeByte(stream, NULL_TAG);
+    return;
+  }
+  if (typeof value === "boolean") {
+    writeByte(stream, value ? TRUE_TAG : FALSE_TAG);
+    return;
+  }
+  if (typeof value === "number") {
+    writeByte(stream, NUMBER_TAG);
+    writeNumber(stream, value);
+    return;
+  }
+  if (typeof value === "string") {
+    writeByte(stream, STRING_TAG);
+    writeString(stream, value);
+    return;
+  }
+  if (Array.isArray(value)) {
+    writeByte(stream, LIST_TAG);
+    writeUint32(stream, value.length);
+    for (const member of value) {
+      writeValue(stream, member ?? null);
+    }
+    return;
+  }
+  const record = value;
+  const keys = Object.keys(record).sort(compareCodePoints);
+  writeByte(stream, MAP_TAG);
+  writeUint32(stream, keys.length);
+  for (const key of keys) {
+    writeByte(stream, STRING_TAG);
+    writeString(stream, key);
+    writeValue(stream, record[key] ?? null);
+  }
+}
+function boundVectorOf(vector) {
+  if (vector === void 0) return void 0;
+  const bound = { ...vector };
+  if (vector.value !== void 0) {
+    bound.value = Array.from(vector.value, Math.fround);
+  }
+  if (vector.text !== void 0) {
+    delete bound.value;
+  }
+  return bound;
+}
+var FILTERS_SLOT = 2;
+var BOUND_SLOT_COUNT = 15;
+function bindingOf(values) {
+  const stream = { bytes: new Uint8Array(256), length: 0 };
+  for (const value of values) {
+    writeValue(stream, value);
+  }
+  return fnv1aBytes(stream.bytes.subarray(0, stream.length)).toString(16).padStart(8, "0");
+}
+function queryBindingOf(params) {
+  return bindingOf([
+    params.term,
+    params.fields,
+    params.filters,
+    params.boost,
+    params.minScore,
+    params.termMatch,
+    params.tolerance,
+    params.prefixLength,
+    params.prefix,
+    params.exact,
+    params.pinned,
+    params.mode,
+    params.hybrid,
+    boundVectorOf(params.vector),
+    params.scoring ?? "local"
+  ]);
+}
+function listBindingOf(filters) {
+  const values = new Array(BOUND_SLOT_COUNT).fill(void 0);
+  values[FILTERS_SLOT] = filters;
+  return bindingOf(values);
+}
+
 // src/engine/list-documents/index.ts
 function clampListLimit(limit) {
   const clamped = Math.max(1, clampLimit(limit));
@@ -21851,7 +22157,7 @@ function collectCandidates(partition, anchor, matches, wanted, out) {
     taken++;
   }
 }
-function pageInIdOrder(partitions, cursor, filtered, limit) {
+function pageInIdOrder(partitions, cursor, filtered, limit, binding) {
   const anchor = cursor === null ? null : cursor.anchor;
   const candidates = [];
   for (let index = 0; index < partitions.length; index++) {
@@ -21866,10 +22172,10 @@ function pageInIdOrder(partitions, cursor, filtered, limit) {
   candidates.sort(compareCodePoints);
   const hasMore = candidates.length > limit;
   const ids = hasMore ? candidates.slice(0, limit) : candidates;
-  const nextCursor = hasMore && ids.length > 0 ? { anchor: ids[ids.length - 1], score: null, sortKey: null, sortSignature: null } : null;
+  const nextCursor = hasMore && ids.length > 0 ? { anchor: ids[ids.length - 1], score: null, sortKey: null, sortSignature: null, binding } : null;
   return { ids, nextCursor };
 }
-function pageInSortOrder(partitions, cursor, filtered, limit, sort, signature, schema) {
+function pageInSortOrder(partitions, cursor, filtered, limit, sort, signature, schema, binding) {
   const normalized = normalizeSort(sort);
   const fields = normalized.map((entry) => entry.field);
   const directions = normalized.map((entry) => entry.direction);
@@ -21892,7 +22198,7 @@ function pageInSortOrder(partitions, cursor, filtered, limit, sort, signature, s
   const hasMore = merged.length > limit;
   const page = hasMore ? merged.slice(0, limit) : merged;
   const last = page[page.length - 1];
-  const nextCursor = hasMore && last !== void 0 ? { anchor: last.id, score: null, sortKey: last.key, sortSignature: signature } : null;
+  const nextCursor = hasMore && last !== void 0 ? { anchor: last.id, score: null, sortKey: last.key, sortSignature: signature, binding } : null;
   return { ids: page.map((entry) => entry.id), nextCursor };
 }
 function countAcrossPartitions(manager, partitionIds, partitions) {
@@ -21911,15 +22217,16 @@ function executeListDocuments(params, context) {
   const limit = clampListLimit(params.limit);
   const signature = sortSignatureOf(params.sort);
   requireSortableFields(params.sort, schema);
+  const binding = listBindingOf(params.filters);
   let cursor = null;
   if (params.cursor !== void 0) {
     cursor = decodePageCursor(params.cursor);
-    requireMatchingCursor(cursor, params.cursor, signature, false);
+    requireMatchingCursor(cursor, params.cursor, signature, false, binding);
   }
   const partitions = context.partitionIds === void 0 ? manager.getAllPartitions() : context.partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
   const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema);
   const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
-  const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema);
+  const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
   const projection = resolveProjection(params.document);
   const documents = [];
   for (const docId of page.ids) {
@@ -21987,18 +22294,14 @@ function applyGrouping(hits, group, getDocument) {
     }
     const result = { values: entry.values, hits: groupHits };
     if (group.reduce) {
-      try {
-        let accumulator = group.reduce.initialValue();
-        for (const hit of groupHits) {
-          const doc = getDocument(hit.id);
-          if (doc) {
-            accumulator = group.reduce.reducer(accumulator, doc, hit.score ?? 0);
-          }
-        }
-        ;
-        result.reduced = accumulator;
-      } catch (err) {
-        result.reducerError = err instanceof Error ? err.message : String(err);
+      const folded = foldGroupReducer(
+        group.reduce,
+        groupHits.map((hit) => ({ document: getDocument(hit.id), score: hit.score ?? 0 }))
+      );
+      if (folded.reducerError !== void 0) {
+        result.reducerError = folded.reducerError;
+      } else {
+        result.reduced = folded.reduced;
       }
     }
     groups.push(result);
@@ -22011,19 +22314,49 @@ function applyGrouping(hits, group, getDocument) {
     const bId = b.hits.length > 0 ? b.hits[0].id : "";
     return compareCodePoints(aId, bId);
   });
+  if (group.limit !== void 0 && Number.isFinite(group.limit) && group.limit >= 0) {
+    const cap = Math.floor(group.limit);
+    if (groups.length > cap) {
+      groups.length = cap;
+    }
+  }
   return groups;
 }
+function foldGroupReducer(reduce, rows) {
+  try {
+    let accumulator = reduce.initialValue();
+    for (const row of rows) {
+      if (row.document !== void 0) {
+        accumulator = reduce.reducer(accumulator, row.document, row.score);
+      }
+    }
+    return { reduced: accumulator };
+  } catch (err) {
+    return { reducerError: err instanceof Error ? err.message : String(err) };
+  }
+}
 
 // src/search/pinning.ts
+function dedupePinnedEntries(pinned) {
+  const deduped = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const entry of pinned) {
+    if (seen.has(entry.docId)) continue;
+    seen.add(entry.docId);
+    deduped.push(entry);
+  }
+  return deduped;
+}
 function applyPinning(hits, pinned, resolveDoc) {
   const result = hits.slice();
-  const pinnedDocIds = new Set(pinned.map((p) => p.docId));
+  const deduped = dedupePinnedEntries(pinned);
+  const pinnedDocIds = new Set(deduped.map((entry) => entry.docId));
   for (let i = result.length - 1; i >= 0; i--) {
     if (pinnedDocIds.has(result[i].id)) {
       result.splice(i, 1);
     }
   }
-  const sorted = pinned.slice().sort((a, b) => a.position - b.position);
+  const sorted = deduped.sort((a, b) => a.position - b.position);
   for (const entry of sorted) {
     const doc = resolveDoc(entry.docId);
     if (!doc) continue;
@@ -22217,17 +22550,7 @@ function applyHighlights(hits, params, language, analysis) {
 
 // src/engine/query/shared.ts
 function partitionsFor(manager, partitionIds) {
-  if (partitionIds === void 0) {
-    return manager.getAllPartitions();
-  }
-  const partitions = [];
-  for (const partitionId of partitionIds) {
-    const partition = manager.partitionAt(partitionId);
-    if (partition !== void 0) {
-      partitions.push(partition);
-    }
-  }
-  return partitions;
+  return partitionsIn(manager, partitionIds);
 }
 function coverageFor(manager, partitionIds) {
   if (partitionIds === void 0) {
@@ -22321,7 +22644,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
   let anchorId = null;
   if (params.searchAfter !== void 0) {
     const decoded = decodePageCursor(params.searchAfter);
-    requireMatchingCursor(decoded, params.searchAfter, sortSignature, true);
+    requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding);
     anchorKey = decoded.sortKey;
     anchorId = decoded.anchor;
   }
@@ -22366,7 +22689,13 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
   const last = page[page.length - 1];
   let cursor;
   if (hasMore && last !== void 0) {
-    cursor = encodePageCursor({ anchor: last.id, score: null, sortKey: last.key, sortSignature });
+    cursor = encodePageCursor({
+      anchor: last.id,
+      score: null,
+      sortKey: last.key,
+      sortSignature,
+      binding: context.cursorBinding
+    });
   }
   return {
     hits: page.map((entry) => ({ id: entry.id, document: void 0 })),
@@ -22652,7 +22981,7 @@ async function executeQuery(params, context) {
     if (params.group) {
       groups = applyGrouping(hits, params.group, (docId) => manager.getRef(docId));
     }
-    if (params.pinned) {
+    if (params.pinned && params.searchAfter === void 0 && context.partitionIds === void 0) {
       hits = applyPinning(hits, params.pinned, (docId) => {
         const doc = manager.getRef(docId);
         if (!doc) return void 0;
@@ -22667,7 +22996,16 @@ async function executeQuery(params, context) {
         sortKeyOf
       };
     }
-    const paged = applyPagination(hits, limit, offset, params.searchAfter, sortContext);
+    const pinnedIds = params.pinned !== void 0 && params.searchAfter === void 0 ? new Set(params.pinned.map((entry) => entry.docId)) : void 0;
+    const paged = applyPagination(
+      hits,
+      limit,
+      offset,
+      context.cursorBinding,
+      params.searchAfter,
+      sortContext,
+      pinnedIds
+    );
     paginated = paged.paginated;
     nextCursor = paged.nextCursor;
     count = fanOutResult.totalMatched;
@@ -22736,6 +23074,11 @@ async function executePreflight(params, context) {
   } else if (isHybridMode && hasGlobalVectorIndex) {
     const result = await executeHybridSearch(params, context, preflightLimit, preflightOffset);
     totalMatched = result.totalMatched;
+  } else if (countsWithoutScores(params)) {
+    totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
+      searchOptions: searchOptionsFor(manager),
+      partitionIds: context.partitionIds
+    });
   } else {
     const scoring = scoringConfigFor(params, context);
     const workerResult = workerSearch ? await workerSearch(indexName, params, broadcastStatsForWorker(params, context, scoring), context.partitionIds) : null;
@@ -22881,7 +23224,8 @@ async function runEngineQuery(core, indexName, params, options) {
     workerSearch,
     indexName,
     broadcastStats: broadcastStatsFor(core),
-    partitionIds: options?.partitionIds
+    partitionIds: options?.partitionIds,
+    cursorBinding: queryBindingOf(params)
   });
   try {
     await core.pluginRegistry.runHook("afterSearch", {
@@ -22915,7 +23259,8 @@ async function runEnginePreflight(core, indexName, params, options) {
     workerSearch,
     indexName,
     broadcastStats: broadcastStatsFor(core),
-    partitionIds: options?.partitionIds
+    partitionIds: options?.partitionIds,
+    cursorBinding: queryBindingOf(params)
   });
   if (core.analysisRebuild.isStale(indexName)) {
     result.analysisStale = true;