@delali/narsil 0.3.1-next.68 → 0.3.1-next.70

package/dist/{chunk-YMZCMH5M.mjs → chunk-OI42GWLR.mjs}

@@ -1,4 +1,20 @@
 // src/core/ordering/code-points.ts
+function exceedsCodePoints(value, maximum) {
+  if (value.length <= maximum) return false;
+  let counted = 0;
+  let i = 0;
+  while (i < value.length) {
+    const unit = value.charCodeAt(i);
+    if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
+      const low = value.charCodeAt(i + 1);
+      if (low >= 56320 && low < 57344) i++;
+    }
+    i++;
+    counted++;
+    if (counted > maximum) return true;
+  }
+  return false;
+}
 function adjustUtf16Unit(unit) {
   if (unit < 55296) return unit;
   if (unit < 57344) return unit + 8192;
@@ -228,4 +244,4 @@ function compareSortValues(aValues, bValues, directions) {
   return 0;
 }
 
-export { SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, compareCodePoints, compareComparableKeys, compareComparableValues, compareSortValues, defaultSortMode, isSortMode, readSortField, sortModeOf, toComparableSortValue, toReducedSortValue };
+export { SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, compareCodePoints, compareComparableKeys, compareComparableValues, compareSortValues, defaultSortMode, exceedsCodePoints, isSortMode, readSortField, sortModeOf, toComparableSortValue, toReducedSortValue, truncateSortString };

package/dist/{chunk-2BDG2IWP.mjs → chunk-P4DSQ6M4.mjs}

@@ -1,8 +1,8 @@
-import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-5JLSTECC.mjs';
-import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, requireValidFilter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex, fuzzyTermMatches } from './chunk-4VU5TWCR.mjs';
-import { DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isTextFieldType, isGeopointOrVectorType, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-6TGZG2TV.mjs';
+import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-GIVAGD3Q.mjs';
+import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, requireValidFilter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex, fuzzyTermMatches } from './chunk-LMO5TJUS.mjs';
+import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isWordIndexedFieldType, isGeopointOrVectorType, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-653BNMAF.mjs';
 import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
-import { compareCodePoints, compareComparableKeys } from './chunk-YMZCMH5M.mjs';
+import { compareCodePoints, compareComparableKeys } from './chunk-OI42GWLR.mjs';
 import { NarsilError, ErrorCodes } from './chunk-USKFU7J5.mjs';
 
 // src/search/cursor-binding.ts
@@ -167,7 +167,7 @@ function listBindingOf(filters) {
 function requireBoostableFields(boost, flatSchema) {
   for (const [field, weight] of Object.entries(boost)) {
     const fieldType = flatSchema[field];
-    if (fieldType === void 0 || !isTextFieldType(fieldType) && fieldType !== "string[]") {
+    if (fieldType === void 0 || !isWordIndexedFieldType(fieldType)) {
       throw new NarsilError(
         ErrorCodes.SEARCH_INVALID_FIELD,
         fieldType === void 0 ? `The engine boosts a text field that the schema declares, and the schema declares no field "${field}"` : `The engine boosts a text field alone, and the schema declares field "${field}" as "${fieldType}"`,
@@ -414,6 +414,16 @@ function validatePartitionConfig(partitions) {
     });
   }
 }
+function validatePatternValueLimit(limit) {
+  if (limit === void 0) return;
+  if (typeof limit !== "number" || !Number.isInteger(limit) || limit < MIN_PATTERN_VALUE_LIMIT || limit > MAX_PATTERN_VALUE_LIMIT) {
+    throw new NarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      `patternValueLimit must be a whole number of code points from ${MIN_PATTERN_VALUE_LIMIT} to ${MAX_PATTERN_VALUE_LIMIT}`,
+      { patternValueLimit: typeof limit === "number" ? limit : String(limit) }
+    );
+  }
+}
 function requirePositiveSafeInteger(field, value) {
   if (value === void 0) return;
   if (!Number.isSafeInteger(value) || value < 1) {
@@ -1507,4 +1517,4 @@ function pickDisplaySurface(surfaceOccurrences) {
   return best;
 }
 
-export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, resolveHybridFusion, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validateWorkerConfig };
+export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, resolveHybridFusion, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };

package/dist/{chunk-6IT3MH5M.mjs → chunk-VNMTB3KC.mjs}

@@ -1,18 +1,18 @@
 import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
-import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, admitInsert, asBatchInsertError, embedDocumentFields, assertDocumentCarriesMappedVectors } from './chunk-YA52AC2V.mjs';
-import { generateId, getLanguage, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, createCompositePartition, deserializePayloadV2, createSharedFrozenSegment, createPartitionManager, buildSegmentPayload, resolvePartitionInsertOptions, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, resolveIndexAnalysis, deserializePayloadV1, decodeVectorIndexParts, nodeThreadId, resolveRequestThreadCount, createPartitionIndex, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, isCompositePartition, workerResourceLimits, deserializeMetadata, isNodeMainThread, spawnNodeWorker, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, serializeMetadata, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, decodeVectorFilePayload, documentsOfPayloadV2, countDead, decodeVectorGraphPayload, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createFrozenSegment } from './chunk-BEWOA4VV.mjs';
-import { detectRuntime } from './chunk-AV4IJKD2.mjs';
-import { validateWorkerConfig, validateIndexName, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-2BDG2IWP.mjs';
-import { BATCH_CHUNK_SIZE, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION } from './chunk-5JLSTECC.mjs';
-import { getNestedValue, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, kWayMerge, mergeFacets } from './chunk-4VU5TWCR.mjs';
-import { extractVectorFieldsFromSchema, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema, isTextFieldType, validateRequiredFields, validateDocument, validateDocumentStrict } from './chunk-6TGZG2TV.mjs';
-import { releaseLocksHeldBy } from './chunk-THYRHHO2.mjs';
+import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, admitInsert, asBatchInsertError, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-FVLQ63B7.mjs';
+import { generateId, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, createCompositePartition, deserializePayloadV2, createSharedFrozenSegment, createPartitionManager, buildSegmentPayload, resolvePartitionInsertOptions, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, resolveIndexAnalysis, deserializePayloadV1, decodeVectorIndexParts, nodeThreadId, resolveRequestThreadCount, createPartitionIndex, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, isCompositePartition, workerResourceLimits, deserializeMetadata, isNodeMainThread, spawnNodeWorker, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, serializeMetadata, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, decodeVectorFilePayload, documentsOfPayloadV2, countDead, decodeVectorGraphPayload, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createFrozenSegment } from './chunk-M76V3OGJ.mjs';
+import { getLanguage, detectRuntime } from './chunk-HWQLTLNG.mjs';
+import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-P4DSQ6M4.mjs';
+import { BATCH_CHUNK_SIZE, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION } from './chunk-GIVAGD3Q.mjs';
+import { getNestedValue, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, kWayMerge, mergeFacets } from './chunk-LMO5TJUS.mjs';
+import { extractVectorFieldsFromSchema, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema, isSingleStringFieldType, requireValidFieldTypes, validateRequiredFields, validateDocument, validateDocumentStrict } from './chunk-653BNMAF.mjs';
+import { releaseLocksHeldBy } from './chunk-YJDQ7NLA.mjs';
 import { requireSearchBackendSettings } from './chunk-YNVIUGF5.mjs';
 import { crc32, nativeCrc32, crc32Final, crc32Init, crc32Update } from './chunk-3L4U6EJQ.mjs';
 import { CHECKSUM_TIMEOUT_MS, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-IFBKLBGB.mjs';
-import { compareCodePoints } from './chunk-YMZCMH5M.mjs';
+import { compareCodePoints } from './chunk-OI42GWLR.mjs';
 import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
-import { NarsilError, ErrorCodes } from './chunk-USKFU7J5.mjs';
+import { ErrorCodes, NarsilError } from './chunk-USKFU7J5.mjs';
 import { encode, decode } from '@msgpack/msgpack';
 
 // src/engine/config-keys.ts
@@ -356,7 +356,7 @@ function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
           { field: source, targetField }
         );
       }
-      if (!isTextFieldType(sourceType)) {
+      if (!isSingleStringFieldType(sourceType)) {
         throw new NarsilError(
           ErrorCodes.EMBEDDING_CONFIG_INVALID,
           `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
@@ -1574,6 +1574,7 @@ async function decodeSnapshotBundle(data) {
   if (!raw.schema || typeof raw.schema !== "object") {
     throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle missing schema");
   }
+  requireValidFieldTypes(raw.schema);
   if (typeof raw.language !== "string") {
     throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle missing language");
   }
@@ -2533,6 +2534,7 @@ function validateManifestCommon(raw) {
   if (typeof raw.language !== "string") {
     throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest is missing language");
   }
+  requireValidFieldTypes(raw.schema);
   return { schema: raw.schema, language: raw.language };
 }
 function normalizeCheckpoint2(raw) {
@@ -2953,6 +2955,9 @@ function reconstructSchemaFromMetadata(metadata) {
   if (metadata.vectorPromotion !== void 0) {
     config.vectorPromotion = metadata.vectorPromotion;
   }
+  if (metadata.patternValueLimit !== void 0) {
+    config.patternValueLimit = metadata.patternValueLimit;
+  }
   return config;
 }
 function createEntryMessage(entry, sourceId) {
@@ -5667,6 +5672,9 @@ function buildMetadata(indexName, hooks, documentCount) {
   if (config.vectorPromotion !== void 0) {
     metadata.vectorPromotion = config.vectorPromotion;
   }
+  if (config.patternValueLimit !== void 0) {
+    metadata.patternValueLimit = config.patternValueLimit;
+  }
   if (config.indexUuid !== void 0) {
     metadata.indexUuid = config.indexUuid;
   }
@@ -5780,7 +5788,8 @@ function createDurabilityFromTier(tier, wiring) {
         ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
         ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
         ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
-        ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {}
+        ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
+        ...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {}
       };
     },
     createIndexFromMetadata: wiring.createIndexFromMetadata,
@@ -5795,6 +5804,11 @@ function getVectorFieldPaths(schema) {
 }
 
 // src/engine/index-from-metadata.ts
+var RECOVERY_CONFIG_ERROR_CODES = /* @__PURE__ */ new Set([
+  ErrorCodes.CONFIG_INVALID,
+  ErrorCodes.SCHEMA_INVALID_TYPE,
+  ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION
+]);
 function resolveEmbeddingAdapter(deps, metadata, schema) {
   const name = metadata.embedding?.adapter ?? null;
   if (!metadata.embedding) return { adapter: null, name };
@@ -5813,6 +5827,15 @@ function resolveEmbeddingAdapter(deps, metadata, schema) {
   }
   return { adapter: candidate, name };
 }
+function recoveryFailure(metadata, err) {
+  if (!(err instanceof NarsilError) || !RECOVERY_CONFIG_ERROR_CODES.has(err.code)) return err;
+  return new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
+    ...err.details,
+    indexName: metadata.indexName,
+    tokenizer: metadata.tokenizer,
+    stopWords: metadata.stopWords
+  });
+}
 async function createIndexFromMetadata(deps, metadata, loadData) {
   const existing = deps.indexRegistry.get(metadata.indexName);
   if (existing !== void 0) {
@@ -5823,19 +5846,18 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
   }
   const indexConfig = reconstructSchemaFromMetadata(metadata);
   const language = getLanguage(indexConfig.language ?? "english");
+  try {
+    indexConfig.schema = validateSchema(indexConfig.schema);
+    validatePatternValueLimit(indexConfig.patternValueLimit);
+  } catch (err) {
+    throw recoveryFailure(metadata, err);
+  }
   const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
   try {
     validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
     if (loadData) deps.executor.createIndex(metadata.indexName, indexConfig, language);
   } catch (err) {
-    if (err instanceof NarsilError && err.code === ErrorCodes.CONFIG_INVALID) {
-      throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
-        indexName: metadata.indexName,
-        tokenizer: metadata.tokenizer,
-        stopWords: metadata.stopWords
-      });
-    }
-    throw err;
+    throw recoveryFailure(metadata, err);
   }
   deps.indexRegistry.set(metadata.indexName, {
     config: indexConfig,
@@ -6442,21 +6464,10 @@ function createHeapPressureNotifier(deps) {
 }
 
 // src/engine/watermark.ts
-function createWatermarkNotifier(deps) {
+function createWatermarkNotifier(deps, scope = "index") {
   const latchedCapacity = /* @__PURE__ */ new Map();
-  function check(indexName) {
-    const partitionConfig = deps.getPartitionConfig(indexName);
-    const maxDocs = partitionConfig?.maxDocsPerPartition;
-    const watermark = partitionConfig?.watermark;
-    if (maxDocs === void 0 || watermark === void 0) {
-      latchedCapacity.delete(indexName);
-      return;
-    }
-    const manager = deps.getManager(indexName);
-    if (!manager) {
-      latchedCapacity.delete(indexName);
-      return;
-    }
+  const latchedPartitions = /* @__PURE__ */ new Map();
+  function checkIndex(indexName, manager, maxDocs, watermark) {
     const partitionCount = manager.partitionCount;
     const capacity = maxDocs * partitionCount;
     const documentCount = manager.countDocuments();
@@ -6469,19 +6480,63 @@ function createWatermarkNotifier(deps) {
     latchedCapacity.set(indexName, capacity);
     deps.emit({ indexName, documentCount, capacity, partitionCount });
   }
+  function checkPartitions(indexName, manager, maxDocs, watermark, insertedDocIds) {
+    const threshold = watermark * maxDocs;
+    const latched = latchedPartitions.get(indexName) ?? /* @__PURE__ */ new Set();
+    latchedPartitions.set(indexName, latched);
+    const checked = /* @__PURE__ */ new Set();
+    for (const docId of insertedDocIds) {
+      const partitionId = manager.partitionIdOf(docId) ?? manager.routePartition(docId);
+      if (checked.has(partitionId)) continue;
+      checked.add(partitionId);
+      const documentCount = manager.getPartition(partitionId).count();
+      if (documentCount < threshold) {
+        latched.delete(partitionId);
+        continue;
+      }
+      if (latched.has(partitionId)) continue;
+      latched.add(partitionId);
+      deps.emit({ indexName, documentCount, capacity: maxDocs, partitionCount: manager.partitionCount, partitionId });
+    }
+  }
+  function check(indexName, inserted) {
+    const partitionConfig = deps.getPartitionConfig(indexName);
+    const maxDocs = partitionConfig?.maxDocsPerPartition;
+    const watermark = partitionConfig?.watermark;
+    if (maxDocs === void 0 || watermark === void 0) {
+      forget(indexName);
+      return;
+    }
+    const manager = deps.getManager(indexName);
+    if (manager === void 0) {
+      forget(indexName);
+      return;
+    }
+    if (scope === "index") {
+      checkIndex(indexName, manager, maxDocs, watermark);
+      return;
+    }
+    if (inserted !== void 0) {
+      checkPartitions(indexName, manager, maxDocs, watermark, typeof inserted === "string" ? [inserted] : inserted);
+    }
+  }
   function forget(indexName) {
     latchedCapacity.delete(indexName);
+    latchedPartitions.delete(indexName);
   }
   return { check, forget };
 }
 
 // src/engine/notifiers.ts
 function wireWatermarkNotifier(wiring) {
-  return createWatermarkNotifier({
-    getManager: wiring.getManager,
-    getPartitionConfig: (indexName) => wiring.indexRegistry.get(indexName)?.config.partitions,
-    emit: (payload) => emitEngineEvent(wiring.eventHandlers, "partitionWatermark", payload)
-  });
+  return createWatermarkNotifier(
+    {
+      getManager: wiring.getManager,
+      getPartitionConfig: (indexName) => wiring.indexRegistry.get(indexName)?.config.partitions,
+      emit: (payload) => emitEngineEvent(wiring.eventHandlers, "partitionWatermark", payload)
+    },
+    wiring.capacityScope
+  );
 }
 function warnUnheardHeapPressure(payload) {
   console.warn(
@@ -8453,7 +8508,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
 }
 
 // src/engine/core.ts
-function createEngineCore(config, hooks) {
+function createEngineCore(config, hooks, capacityScope = "index") {
   validateWorkerConfig(config?.workers, config?.lifecycle);
   const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
   const vectorCopyPolicy = {
@@ -8601,7 +8656,8 @@ function createEngineCore(config, hooks) {
   const notifierWiring = {
     eventHandlers,
     indexRegistry,
-    getManager: (indexName) => executor.getManager(indexName)
+    getManager: (indexName) => executor.getManager(indexName),
+    capacityScope
   };
   const watermarkNotifier = wireWatermarkNotifier(notifierWiring);
   const heapPressureNotifier = wireHeapPressureNotifier(notifierWiring);
@@ -8645,6 +8701,7 @@ function createEngineCore(config, hooks) {
     pendingRebalanceWrites: (indexName) => waqMap.get(indexName)?.size ?? 0,
     rebalanceTargetPartitionCount: (indexName) => rebalanceTargets.get(indexName),
     bufferedDocState: (indexName, docId) => waqMap.get(indexName)?.bufferedDocState(docId),
+    capacityScope,
     checkWatermark: watermarkNotifier.check,
     checkHeapPressure: heapPressureNotifier.check
   };
@@ -8778,13 +8835,16 @@ async function runLifecycleHook(core, hookName, context) {
 async function announceIndexCreated(core, name, config) {
   await runLifecycleHook(core, "onIndexCreate", { indexName: name, config });
 }
-async function createEngineIndex(core, config, name, indexConfig, indexUuid) {
+async function createEngineIndex(core, config, name, requestedConfig, creation = {}) {
+  const { indexUuid, adapterBinding = "required" } = creation;
   core.guardShutdown();
   validateIndexName(name);
   if (core.indexRegistry.has(name)) {
     throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
   }
-  validateSchema(indexConfig.schema);
+  const schema = validateSchema(requestedConfig.schema);
+  const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
+  validatePatternValueLimit(indexConfig.patternValueLimit);
   validateVectorPromotion(indexConfig.vectorPromotion);
   validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
   validateBM25Params(indexConfig.bm25);
@@ -8810,24 +8870,31 @@ async function createEngineIndex(core, config, name, indexConfig, indexUuid) {
   let resolvedEmbeddingAdapter = null;
   let embeddingAdapterName = null;
   if (indexConfig.embedding) {
-    let configuredAdapter = indexConfig.embedding.adapter;
+    const configuredAdapter = indexConfig.embedding.adapter;
+    const fields = indexConfig.embedding.fields;
     if (typeof configuredAdapter === "string") {
       embeddingAdapterName = configuredAdapter;
       const registered = core.embeddingAdapters.get(configuredAdapter);
-      if (!registered) {
+      if (registered !== void 0) {
+        resolvedEmbeddingAdapter = validateEmbeddingConfig(
+          { fields, adapter: registered },
+          indexConfig.schema,
+          config?.embedding
+        );
+      } else if (adapterBinding === "required") {
         throw new NarsilError(
           ErrorCodes.EMBEDDING_CONFIG_INVALID,
           `Embedding adapter "${configuredAdapter}" is not registered on this engine`,
           { adapter: configuredAdapter, available: [...core.embeddingAdapters.keys()] }
         );
       }
-      configuredAdapter = registered;
+    } else if (adapterBinding === "required" || configuredAdapter !== void 0 || config?.embedding !== void 0) {
+      resolvedEmbeddingAdapter = validateEmbeddingConfig(
+        { fields, adapter: configuredAdapter },
+        indexConfig.schema,
+        config?.embedding
+      );
     }
-    resolvedEmbeddingAdapter = validateEmbeddingConfig(
-      { fields: indexConfig.embedding.fields, adapter: configuredAdapter },
-      indexConfig.schema,
-      config?.embedding
-    );
   }
   if (indexConfig.required && indexConfig.required.length > 0) {
     validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
@@ -9042,7 +9109,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
     await applyInsert();
   }
   if (buffered) {
-    ctx.checkWatermark(indexName);
+    ctx.checkWatermark(indexName, resolvedDocId);
     ctx.checkHeapPressure(indexName);
     if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
     return resolvedDocId;
@@ -9066,7 +9133,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
       vecIndex.scheduleBuild();
     }
   }
-  ctx.checkWatermark(indexName);
+  ctx.checkWatermark(indexName, resolvedDocId);
   ctx.checkHeapPressure(indexName);
   await ctx.orchestrator.scaleOutReadyIndexes();
   if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
@@ -9177,6 +9244,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
   const required = entry.config.required;
   const admitted = [];
   const admittedIds = /* @__PURE__ */ new Set();
+  const pending = createPendingAdmissions();
   for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
     if (ctx.abortController.signal.aborted) break;
     const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
@@ -9209,15 +9277,16 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
         if (extractedVectors.size > 0) {
           validateVectorDimensions(extractedVectors, vecIndexes);
         }
-        admitInsert(ctx, indexName, manager, docId, admitted.length);
+        admitInsert(ctx, indexName, manager, docId, pending);
         if (admittedIds.has(docId) || manager.has(docId)) {
           throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
         }
-        validateDocument(partitionDoc, entry.config.schema);
+        validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
         if (strict) {
           validateDocumentStrict(partitionDoc, entry.config.schema);
         }
         admittedIds.add(docId);
+        recordPendingAdmission(ctx, pending, manager, docId);
         admitted.push({ docId, document: documents[i], partitionDoc, extractedVectors });
       } catch (err) {
         failed.push({ docId, error: asBatchInsertError(err) });
@@ -9616,7 +9685,7 @@ async function insertBatchViaSegments(ctx, indexName, documents, options) {
   for (const fieldPath of outcome.touchedVectorFields) {
     vecIndexes.get(fieldPath)?.scheduleBuild();
   }
-  ctx.checkWatermark(indexName);
+  ctx.checkWatermark(indexName, outcome.succeeded);
   ctx.checkHeapPressure(indexName);
   await ctx.orchestrator.scaleOutReadyIndexes();
   if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
@@ -9749,7 +9818,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
       vecIndex.scheduleBuild();
     }
   }
-  ctx.checkWatermark(indexName);
+  ctx.checkWatermark(indexName, succeeded);
   ctx.checkHeapPressure(indexName);
   await ctx.orchestrator.scaleOutReadyIndexes();
   if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
@@ -9982,7 +10051,7 @@ function applyOfUpdate(ctx, indexName, prepared) {
       const bufferedState = ctx.bufferedDocState(indexName, docId);
       const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
       if (!exists) {
-        manager.assertCapacity(ctx.pendingRebalanceWrites(indexName), ctx.rebalanceTargetPartitionCount(indexName));
+        admitBufferedUpsert(ctx, indexName, manager, docId);
       }
     }
     if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
@@ -10341,162 +10410,6 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
   ctx.checkWatermark(indexName);
 }
 
-// src/engine/snapshot/restore-config.ts
-function invalid2(message) {
-  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
-}
-function isPlainObject(value) {
-  return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-function restoredBm25(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!isPlainObject(raw)) {
-    invalid2("bm25 must be an object");
-  }
-  const { k1, b } = raw;
-  if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
-    invalid2("bm25 k1 must be a finite number");
-  }
-  if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
-    invalid2("bm25 b must be a finite number");
-  }
-  return {
-    ...k1 !== void 0 ? { k1 } : {},
-    ...b !== void 0 ? { b } : {}
-  };
-}
-function restoredStopWords(name, list) {
-  if (name !== void 0 && typeof name !== "string") {
-    invalid2("stopWords must be a name");
-  }
-  if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
-    invalid2("stopWordList must be a list of words");
-  }
-  if (typeof name === "string" && Array.isArray(list)) {
-    invalid2("stopWords and stopWordList cannot both be present");
-  }
-  if (typeof name === "string") {
-    return name;
-  }
-  if (Array.isArray(list)) {
-    return new Set(list);
-  }
-  return void 0;
-}
-function restoredPartitionConfig(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!isPlainObject(raw)) {
-    invalid2("partitionConfig must be an object");
-  }
-  const { maxDocsPerPartition, maxPartitions, watermark } = raw;
-  if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
-    invalid2("partitionConfig maxDocsPerPartition must be a positive integer");
-  }
-  if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
-    invalid2("partitionConfig maxPartitions must be a positive integer");
-  }
-  if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
-    invalid2("partitionConfig watermark must be above 0 and at most 1");
-  }
-  return {
-    ...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
-    ...typeof maxPartitions === "number" ? { maxPartitions } : {},
-    ...typeof watermark === "number" ? { watermark } : {}
-  };
-}
-function restoredScoring(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
-    invalid2("defaultScoring must be 'local', 'dfs', or 'broadcast'");
-  }
-  return raw;
-}
-function restoredBoolean(raw, field) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (typeof raw !== "boolean") {
-    invalid2(`${field} must be a boolean`);
-  }
-  return raw;
-}
-function restoredRequired(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
-    invalid2("required must be a list of field paths");
-  }
-  return raw;
-}
-function restoredVectorPromotion(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!isPlainObject(raw) || raw.hnswConfig !== void 0 && !isPlainObject(raw.hnswConfig)) {
-    invalid2("vectorPromotion must be an object");
-  }
-  return raw;
-}
-function restoredEmbedding(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!isPlainObject(raw)) {
-    invalid2("embedding must be an object");
-  }
-  const { fields, adapter } = raw;
-  if (adapter !== void 0 && typeof adapter !== "string") {
-    invalid2("embedding adapter must be a name");
-  }
-  if (!isPlainObject(fields)) {
-    invalid2("embedding fields must be an object");
-  }
-  for (const value of Object.values(fields)) {
-    const isPath = typeof value === "string";
-    const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
-    if (!isPath && !isPathList) {
-      invalid2("embedding fields must map vector fields to source paths");
-    }
-  }
-  return {
-    fields,
-    ...adapter !== void 0 ? { adapter } : {}
-  };
-}
-function restoredConfigFields(envelope) {
-  if (envelope.tokenizer !== void 0 && typeof envelope.tokenizer !== "string") {
-    invalid2("tokenizer must be a name");
-  }
-  const stopWords = restoredStopWords(envelope.stopWords, envelope.stopWordList);
-  const bm25 = restoredBm25(envelope.bm25);
-  const surfaceForms = restoredBoolean(envelope.surfaceForms, "surfaceForms");
-  const partitions = restoredPartitionConfig(envelope.partitionConfig);
-  const defaultScoring = restoredScoring(envelope.defaultScoring);
-  const trackPositions = restoredBoolean(envelope.trackPositions, "trackPositions");
-  const strict = restoredBoolean(envelope.strict, "strict");
-  const required = restoredRequired(envelope.required);
-  const vectorPromotion = restoredVectorPromotion(envelope.vectorPromotion);
-  return {
-    ...typeof envelope.tokenizer === "string" ? { tokenizer: envelope.tokenizer } : {},
-    ...stopWords !== void 0 ? { stopWords } : {},
-    ...bm25 !== void 0 ? { bm25 } : {},
-    ...surfaceForms !== void 0 ? { surfaceForms } : {},
-    ...partitions !== void 0 ? { partitions } : {},
-    ...defaultScoring !== void 0 ? { defaultScoring } : {},
-    ...trackPositions !== void 0 ? { trackPositions } : {},
-    ...strict !== void 0 ? { strict } : {},
-    ...required !== void 0 ? { required } : {},
-    ...vectorPromotion !== void 0 ? { vectorPromotion } : {}
-  };
-}
-
 // src/engine/snapshot/index.ts
 async function recordRestoredDocuments(durability, indexName, restored) {
   if (restored.length === 0) return;
@@ -10552,6 +10465,7 @@ async function createSnapshot(manager, entry) {
     ...config.strict !== void 0 ? { strict: config.strict } : {},
     ...config.required !== void 0 ? { required: config.required } : {},
     ...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
+    ...config.patternValueLimit !== void 0 ? { patternValueLimit: config.patternValueLimit } : {},
     ...config.embedding !== void 0 ? {
       embedding: {
         fields: config.embedding.fields,
@@ -10604,8 +10518,8 @@ async function restoreFromSnapshot(indexName, data, deps) {
   const configFields = restoredConfigFields(envelope);
   const embedding = restoredEmbedding(envelope.embedding);
   const language = getLanguage(envelope.language);
-  const schema = envelope.schema;
-  validateSchema(schema);
+  const schema = validateSchema(envelope.schema);
+  validatePatternValueLimit(configFields.patternValueLimit);
   validateVectorPromotion(configFields.vectorPromotion);
   validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
   if (configFields.required !== void 0 && configFields.required.length > 0) {

package/dist/{chunk-THYRHHO2.mjs → chunk-YJDQ7NLA.mjs}

@@ -1,6 +1,6 @@
 import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
 import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-YNVIUGF5.mjs';
-import { compareCodePoints } from './chunk-YMZCMH5M.mjs';
+import { compareCodePoints } from './chunk-OI42GWLR.mjs';
 import { NarsilError, ErrorCodes } from './chunk-USKFU7J5.mjs';
 import { readSync, closeSync, openSync } from 'fs';
 

package/dist/{client-xVrOCYTW.d.ts → client-COek5GXK.d.ts}

@@ -1,6 +1,6 @@
-import { P as PartitionConfig, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-d5Afu03A.js';
-import { V as VectorMaintenanceResult, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, j as SuggestParams, k as SuggestResult } from './memory-CKF-9WLP.js';
-import { e as TaskRecord, M as MemoryStatsResponse, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-BEiNxDRs.js';
+import { P as PartitionConfig, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-Bgzr4g7b.js';
+import { V as VectorMaintenanceResult, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, j as SuggestParams, k as SuggestResult } from './memory-C7UMCMH3.js';
+import { e as TaskRecord, M as MemoryStatsResponse, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-BZt_zes6.js';
 
 /**
  * This is the `fetch` implementation the client sends every request through.