@delali/narsil 0.3.1-next.69 → 0.3.1-next.70

package/dist/{chunk-YA52AC2V.mjs → chunk-FVLQ63B7.mjs}

@@ -1,4 +1,4 @@
-import { validateRequiredFields } from './chunk-6TGZG2TV.mjs';
+import { validateRequiredFields } from './chunk-653BNMAF.mjs';
 import { ErrorCodes, NarsilError } from './chunk-USKFU7J5.mjs';
 
 // src/narsil/internals.ts
@@ -245,15 +245,43 @@ function providedDocId(document) {
 function asBatchInsertError(err) {
   return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
 }
-function admitInsert(ctx, indexName, manager, docId, pendingAdmitted = 0) {
-  if (!ctx.isRebalancing(indexName)) {
-    manager.assertCapacity(pendingAdmitted);
+function createPendingAdmissions() {
+  return { total: 0, byPartition: /* @__PURE__ */ new Map() };
+}
+function capsEachPartition(ctx, manager) {
+  return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
+}
+function recordPendingAdmission(ctx, pending, manager, docId) {
+  pending.total += 1;
+  if (!capsEachPartition(ctx, manager)) return;
+  const partitionId = manager.routePartition(docId);
+  pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
+}
+function admitInsert(ctx, indexName, manager, docId, pending) {
+  const rebalancing = ctx.isRebalancing(indexName);
+  if (rebalancing) {
+    const bufferedState = ctx.bufferedDocState(indexName, docId);
+    const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
+    if (exists) {
+      throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
+    }
+  }
+  assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing);
+}
+function admitBufferedUpsert(ctx, indexName, manager, docId) {
+  assertRoomFor(ctx, indexName, manager, docId, void 0, true);
+}
+function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
+  if (ctx.capacityScope === "partition") {
+    if (!capsEachPartition(ctx, manager)) return;
+    const partitionId = manager.routePartition(docId);
+    manager.assertPartitionCapacity(partitionId, pending?.byPartition.get(partitionId) ?? 0);
     return;
   }
-  const bufferedState = ctx.bufferedDocState(indexName, docId);
-  const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
-  if (exists) {
-    throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
+  const pendingAdmitted = pending?.total ?? 0;
+  if (!rebalancing) {
+    manager.assertCapacity(pendingAdmitted);
+    return;
   }
   manager.assertCapacity(
     ctx.pendingRebalanceWrites(indexName) + pendingAdmitted,
@@ -372,4 +400,170 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
   return { ...params, vector: resolved };
 }
 
-export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, resolveVectorText };
+// src/engine/snapshot/restore-config.ts
+function rejectSnapshot(message) {
+  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
+}
+function isPlainObject(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function restoredBm25(raw, reject) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!isPlainObject(raw)) {
+    reject("bm25 must be an object");
+  }
+  const { k1, b } = raw;
+  if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
+    reject("bm25 k1 must be a finite number");
+  }
+  if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
+    reject("bm25 b must be a finite number");
+  }
+  return {
+    ...k1 !== void 0 ? { k1 } : {},
+    ...b !== void 0 ? { b } : {}
+  };
+}
+function restoredStopWords(name, list, reject) {
+  if (name !== void 0 && typeof name !== "string") {
+    reject("stopWords must be a name");
+  }
+  if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
+    reject("stopWordList must be a list of words");
+  }
+  if (typeof name === "string" && Array.isArray(list)) {
+    reject("stopWords and stopWordList cannot both be present");
+  }
+  if (typeof name === "string") {
+    return name;
+  }
+  if (Array.isArray(list)) {
+    return new Set(list);
+  }
+  return void 0;
+}
+function restoredPartitionConfig(raw, reject = rejectSnapshot) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!isPlainObject(raw)) {
+    reject("partitionConfig must be an object");
+  }
+  const { maxDocsPerPartition, maxPartitions, watermark } = raw;
+  if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
+    reject("partitionConfig maxDocsPerPartition must be a positive integer");
+  }
+  if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
+    reject("partitionConfig maxPartitions must be a positive integer");
+  }
+  if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
+    reject("partitionConfig watermark must be above 0 and at most 1");
+  }
+  return {
+    ...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
+    ...typeof maxPartitions === "number" ? { maxPartitions } : {},
+    ...typeof watermark === "number" ? { watermark } : {}
+  };
+}
+function restoredScoring(raw, reject) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
+    reject("defaultScoring must be 'local', 'dfs', or 'broadcast'");
+  }
+  return raw;
+}
+function restoredBoolean(raw, field, reject) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (typeof raw !== "boolean") {
+    reject(`${field} must be a boolean`);
+  }
+  return raw;
+}
+function restoredRequired(raw, reject) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
+    reject("required must be a list of field paths");
+  }
+  return raw;
+}
+function restoredVectorPromotion(raw, reject) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!isPlainObject(raw) || raw.hnswConfig !== void 0 && !isPlainObject(raw.hnswConfig)) {
+    reject("vectorPromotion must be an object");
+  }
+  return raw;
+}
+function restoredEmbedding(raw, reject = rejectSnapshot) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!isPlainObject(raw)) {
+    reject("embedding must be an object");
+  }
+  const { fields, adapter } = raw;
+  if (adapter !== void 0 && typeof adapter !== "string") {
+    reject("embedding adapter must be a name");
+  }
+  if (!isPlainObject(fields)) {
+    reject("embedding fields must be an object");
+  }
+  for (const value of Object.values(fields)) {
+    const isPath = typeof value === "string";
+    const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
+    if (!isPath && !isPathList) {
+      reject("embedding fields must map vector fields to source paths");
+    }
+  }
+  return {
+    fields,
+    ...adapter !== void 0 ? { adapter } : {}
+  };
+}
+function readSharedConfigFields(source, reject) {
+  if (source.tokenizer !== void 0 && typeof source.tokenizer !== "string") {
+    reject("tokenizer must be a name");
+  }
+  const stopWords = restoredStopWords(source.stopWords, source.stopWordList, reject);
+  const bm25 = restoredBm25(source.bm25, reject);
+  const surfaceForms = restoredBoolean(source.surfaceForms, "surfaceForms", reject);
+  const defaultScoring = restoredScoring(source.defaultScoring, reject);
+  const trackPositions = restoredBoolean(source.trackPositions, "trackPositions", reject);
+  const strict = restoredBoolean(source.strict, "strict", reject);
+  const required = restoredRequired(source.required, reject);
+  const vectorPromotion = restoredVectorPromotion(source.vectorPromotion, reject);
+  if (source.patternValueLimit !== void 0 && typeof source.patternValueLimit !== "number") {
+    reject("patternValueLimit must be a number");
+  }
+  return {
+    ...typeof source.tokenizer === "string" ? { tokenizer: source.tokenizer } : {},
+    ...stopWords !== void 0 ? { stopWords } : {},
+    ...bm25 !== void 0 ? { bm25 } : {},
+    ...surfaceForms !== void 0 ? { surfaceForms } : {},
+    ...defaultScoring !== void 0 ? { defaultScoring } : {},
+    ...trackPositions !== void 0 ? { trackPositions } : {},
+    ...strict !== void 0 ? { strict } : {},
+    ...required !== void 0 ? { required } : {},
+    ...vectorPromotion !== void 0 ? { vectorPromotion } : {},
+    ...typeof source.patternValueLimit === "number" ? { patternValueLimit: source.patternValueLimit } : {}
+  };
+}
+function restoredConfigFields(envelope) {
+  const shared = readSharedConfigFields(envelope, rejectSnapshot);
+  const partitions = restoredPartitionConfig(envelope.partitionConfig);
+  return {
+    ...shared,
+    ...partitions !== void 0 ? { partitions } : {}
+  };
+}
+
+export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };

package/dist/{chunk-5JLSTECC.mjs → chunk-GIVAGD3Q.mjs}

@@ -1,4 +1,4 @@
-import { DEFAULT_PAGE_SIZE } from './chunk-6TGZG2TV.mjs';
+import { DEFAULT_PAGE_SIZE } from './chunk-653BNMAF.mjs';
 
 // src/engine/constants.ts
 var MIN_DOCUMENTS_FOR_SEGMENTS = 64;

package/dist/{chunk-AV4IJKD2.mjs → chunk-HWQLTLNG.mjs}

@@ -1,3 +1,20 @@
+import { english } from './chunk-TGP5M5R7.mjs';
+import { NarsilError, ErrorCodes } from './chunk-USKFU7J5.mjs';
+
+// src/languages/registry.ts
+var languages = /* @__PURE__ */ new Map();
+languages.set("english", english);
+function registerLanguage(module) {
+  languages.set(module.name, module);
+}
+function getLanguage(name) {
+  const lang = languages.get(name);
+  if (!lang) {
+    throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, `Language "${name}" is not registered`, { language: name });
+  }
+  return lang;
+}
+
 // src/runtime/detect.ts
 function detectRuntimeName() {
   if ("Bun" in globalThis) {
@@ -32,4 +49,4 @@ function detectRuntime() {
   };
 }
 
-export { detectRuntime };
+export { detectRuntime, getLanguage, registerLanguage };

package/dist/{chunk-4VU5TWCR.mjs → chunk-LMO5TJUS.mjs}

@@ -1,5 +1,5 @@
-import { isTextFieldType, flattenSchema, SORTABLE_TEXT_FIELD_TYPE, isGeopointOrVectorType, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PROTOTYPE_POLLUTION_KEYS, PREFIX_MAX_EXPANSIONS, MAX_CURSOR_ANCHOR_CODE_POINTS } from './chunk-6TGZG2TV.mjs';
-import { compareCodePoints, sortModeOf, defaultSortMode, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, compareSortValues } from './chunk-YMZCMH5M.mjs';
+import { isWordIndexedBase, flattenSchema, parsedTypeOf, isStringValuedBase, withSortableOption, isGeopointOrVectorType, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PROTOTYPE_POLLUTION_KEYS, isWordIndexedFieldType, PREFIX_MAX_EXPANSIONS } from './chunk-653BNMAF.mjs';
+import { compareCodePoints, sortModeOf, defaultSortMode, SORT_VALUE_MAX_CODE_POINTS, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, exceedsCodePoints, isSortMode, SORT_MODES, compareSortValues } from './chunk-OI42GWLR.mjs';
 import { createNarsilError, ErrorCodes, NarsilError } from './chunk-USKFU7J5.mjs';
 
 // src/core/tokenizer/constants.ts
@@ -762,12 +762,12 @@ function stringArraysEqual(a, b) {
   }
   return true;
 }
-function textFieldsChanged(oldDoc, newDoc, flatSchema) {
-  for (const [path, fieldType] of Object.entries(flatSchema)) {
-    if (!isTextFieldType(fieldType) && fieldType !== "string[]") continue;
+function textFieldsChanged(oldDoc, newDoc, fields) {
+  for (const { path, base } of fields) {
+    if (!isWordIndexedBase(base)) continue;
     const oldVal = getNestedValue(oldDoc, path);
     const newVal = getNestedValue(newDoc, path);
-    if (fieldType !== "string[]") {
+    if (base === "string") {
       if (oldVal !== newVal) return true;
     } else {
       if (!stringArraysEqual(oldVal, newVal)) return true;
@@ -1192,6 +1192,16 @@ function requireFieldKind(fieldPath, operator, type, accepted) {
     operator
   );
 }
+function requireTextRangeField(fieldPath, operator, type) {
+  const parsed = parsedTypeOf(type);
+  if (parsed?.sortable && (parsed.base === "string" || parsed.base === "verbatim")) return;
+  const declared = type === void 0 ? `the schema has no field "${fieldPath}"` : `field "${fieldPath}" has type "${type}"`;
+  invalid(
+    `The engine applies "${operator}" with a text bound to a "string" or "verbatim" field whose type includes "sortable", but ${declared}`,
+    fieldPath,
+    operator
+  );
+}
 function requireRadius(fieldPath, radius) {
   const circle = radius;
   if (circle === null || typeof circle !== "object") {
@@ -1252,22 +1262,32 @@ function requireValidFilter(expression, fieldTypes, declaredOnly = false) {
 }
 function requireValidOperands(fieldPath, filter, type) {
   for (const operator of RANGE_OPERATORS) {
-    if (filter[operator] === void 0) continue;
-    if (!isFiniteNumber(filter[operator])) {
-      invalid(`"${operator}" on field "${fieldPath}" must be a finite number`, fieldPath, operator);
+    const bound = filter[operator];
+    if (bound === void 0) continue;
+    if (typeof bound === "string") {
+      requireTextRangeField(fieldPath, operator, type);
+      continue;
+    }
+    if (!isFiniteNumber(bound)) {
+      invalid(`"${operator}" on field "${fieldPath}" must be a finite number or a string`, fieldPath, operator);
     }
     requireFieldKind(fieldPath, operator, type, ["number"]);
   }
   if (filter.between !== void 0) {
     const bounds = filter.between;
-    if (!Array.isArray(bounds) || bounds.length !== 2 || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
-      invalid(
-        `"between" on field "${fieldPath}" must be two finite numbers, the lower bound first`,
-        fieldPath,
-        "between"
-      );
+    const isPair = Array.isArray(bounds) && bounds.length === 2;
+    if (isPair && typeof bounds[0] === "string" && typeof bounds[1] === "string") {
+      requireTextRangeField(fieldPath, "between", type);
+    } else {
+      if (!isPair || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
+        invalid(
+          `"between" on field "${fieldPath}" must be two finite numbers or two strings, the lower bound first`,
+          fieldPath,
+          "between"
+        );
+      }
+      requireFieldKind(fieldPath, "between", type, ["number"]);
     }
-    requireFieldKind(fieldPath, "between", type, ["number"]);
   }
   for (const operator of TEXT_OPERATORS) {
     if (filter[operator] === void 0) continue;
@@ -1365,10 +1385,11 @@ function requireSortableFields(sort, schema, strict = false) {
       );
     }
     requireSortMode(entry, fieldType);
-    if (fieldType === "string") {
+    const parsed = parsedTypeOf(fieldType);
+    if (parsed !== void 0 && isStringValuedBase(parsed.base) && !parsed.sortable) {
       throw new NarsilError(
         ErrorCodes.SEARCH_INVALID_FIELD,
-        `The engine sorts by text field "${entry.field}" only where the schema declares it "${SORTABLE_TEXT_FIELD_TYPE}", because ordering text takes far more memory per document than ordering a number`,
+        `The engine sorts by text field "${entry.field}" only where its type includes "sortable", as in "${withSortableOption(parsed)}", because it keeps up to the first ${SORT_VALUE_MAX_CODE_POINTS} code points of every document's value in memory to sort by a text field`,
         { field: entry.field, fieldType }
       );
     }
@@ -1406,22 +1427,6 @@ function invalidCursor(cursor, reason) {
   const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
   return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
 }
-function exceedsCodePoints(value, maximum) {
-  if (value.length <= maximum) return false;
-  let counted = 0;
-  let i = 0;
-  while (i < value.length) {
-    const unit = value.charCodeAt(i);
-    if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
-      const low = value.charCodeAt(i + 1);
-      if (low >= 56320 && low < 57344) i++;
-    }
-    i++;
-    counted++;
-    if (counted > maximum) return true;
-  }
-  return false;
-}
 function sortSignatureOf(sort) {
   const fields = normalizeSort(sort);
   if (fields.length === 0) return null;
@@ -1750,7 +1755,7 @@ function validateSearchFields(fields, flatSchema) {
     if (!fieldType) {
       throw new NarsilError(ErrorCodes.SEARCH_INVALID_FIELD, `Field "${field}" does not exist in the schema`, { field });
     }
-    if (!isTextFieldType(fieldType) && fieldType !== "string[]") {
+    if (!isWordIndexedFieldType(fieldType)) {
       throw new NarsilError(
         ErrorCodes.SEARCH_INVALID_FIELD,
         `Field "${field}" has type "${fieldType}" which cannot be used for full-text search`,