@delali/narsil 0.3.1-next.69 → 0.3.1-next.71

package/dist/{chunk-6IT3MH5M.mjs → chunk-IOR2K26O.mjs}

@@ -1,18 +1,18 @@
 import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
-import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, admitInsert, asBatchInsertError, embedDocumentFields, assertDocumentCarriesMappedVectors } from './chunk-YA52AC2V.mjs';
-import { generateId, getLanguage, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, createCompositePartition, deserializePayloadV2, createSharedFrozenSegment, createPartitionManager, buildSegmentPayload, resolvePartitionInsertOptions, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, resolveIndexAnalysis, deserializePayloadV1, decodeVectorIndexParts, nodeThreadId, resolveRequestThreadCount, createPartitionIndex, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, isCompositePartition, workerResourceLimits, deserializeMetadata, isNodeMainThread, spawnNodeWorker, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, serializeMetadata, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, decodeVectorFilePayload, documentsOfPayloadV2, countDead, decodeVectorGraphPayload, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createFrozenSegment } from './chunk-BEWOA4VV.mjs';
-import { detectRuntime } from './chunk-AV4IJKD2.mjs';
-import { validateWorkerConfig, validateIndexName, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-2BDG2IWP.mjs';
-import { BATCH_CHUNK_SIZE, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION } from './chunk-5JLSTECC.mjs';
-import { getNestedValue, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, kWayMerge, mergeFacets } from './chunk-4VU5TWCR.mjs';
-import { extractVectorFieldsFromSchema, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema, isTextFieldType, validateRequiredFields, validateDocument, validateDocumentStrict } from './chunk-6TGZG2TV.mjs';
-import { releaseLocksHeldBy } from './chunk-THYRHHO2.mjs';
+import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, admitInsert, asBatchInsertError, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-FVLQ63B7.mjs';
+import { generateId, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, createCompositePartition, deserializePayloadV2, createSharedFrozenSegment, createPartitionManager, buildSegmentPayload, resolvePartitionInsertOptions, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, resolveIndexAnalysis, deserializePayloadV1, decodeVectorIndexParts, nodeThreadId, resolveRequestThreadCount, createPartitionIndex, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, isCompositePartition, workerResourceLimits, deserializeMetadata, isNodeMainThread, spawnNodeWorker, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, serializeMetadata, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, decodeVectorFilePayload, documentsOfPayloadV2, countDead, decodeVectorGraphPayload, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createFrozenSegment } from './chunk-M76V3OGJ.mjs';
+import { getLanguage, detectRuntime } from './chunk-HWQLTLNG.mjs';
+import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-P4DSQ6M4.mjs';
+import { BATCH_CHUNK_SIZE, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION } from './chunk-GIVAGD3Q.mjs';
+import { getNestedValue, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, kWayMerge, mergeFacets } from './chunk-LMO5TJUS.mjs';
+import { extractVectorFieldsFromSchema, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema, isSingleStringFieldType, requireValidFieldTypes, validateRequiredFields, validateDocument, validateDocumentStrict } from './chunk-653BNMAF.mjs';
+import { releaseLocksHeldBy } from './chunk-YJDQ7NLA.mjs';
 import { requireSearchBackendSettings } from './chunk-YNVIUGF5.mjs';
 import { crc32, nativeCrc32, crc32Final, crc32Init, crc32Update } from './chunk-3L4U6EJQ.mjs';
 import { CHECKSUM_TIMEOUT_MS, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-IFBKLBGB.mjs';
-import { compareCodePoints } from './chunk-YMZCMH5M.mjs';
+import { compareCodePoints } from './chunk-OI42GWLR.mjs';
 import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
-import { NarsilError, ErrorCodes } from './chunk-USKFU7J5.mjs';
+import { ErrorCodes, NarsilError } from './chunk-USKFU7J5.mjs';
 import { encode, decode } from '@msgpack/msgpack';
 
 // src/engine/config-keys.ts
@@ -356,7 +356,7 @@ function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
           { field: source, targetField }
         );
       }
-      if (!isTextFieldType(sourceType)) {
+      if (!isSingleStringFieldType(sourceType)) {
         throw new NarsilError(
           ErrorCodes.EMBEDDING_CONFIG_INVALID,
           `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
@@ -1574,6 +1574,7 @@ async function decodeSnapshotBundle(data) {
   if (!raw.schema || typeof raw.schema !== "object") {
     throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle missing schema");
   }
+  requireValidFieldTypes(raw.schema);
   if (typeof raw.language !== "string") {
     throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle missing language");
   }
@@ -2533,6 +2534,7 @@ function validateManifestCommon(raw) {
   if (typeof raw.language !== "string") {
     throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest is missing language");
   }
+  requireValidFieldTypes(raw.schema);
   return { schema: raw.schema, language: raw.language };
 }
 function normalizeCheckpoint2(raw) {
@@ -2953,6 +2955,9 @@ function reconstructSchemaFromMetadata(metadata) {
   if (metadata.vectorPromotion !== void 0) {
     config.vectorPromotion = metadata.vectorPromotion;
   }
+  if (metadata.patternValueLimit !== void 0) {
+    config.patternValueLimit = metadata.patternValueLimit;
+  }
   return config;
 }
 function createEntryMessage(entry, sourceId) {
@@ -5667,6 +5672,9 @@ function buildMetadata(indexName, hooks, documentCount) {
   if (config.vectorPromotion !== void 0) {
     metadata.vectorPromotion = config.vectorPromotion;
   }
+  if (config.patternValueLimit !== void 0) {
+    metadata.patternValueLimit = config.patternValueLimit;
+  }
   if (config.indexUuid !== void 0) {
     metadata.indexUuid = config.indexUuid;
   }
@@ -5780,7 +5788,8 @@ function createDurabilityFromTier(tier, wiring) {
         ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
         ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
         ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
-        ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {}
+        ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
+        ...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {}
       };
     },
     createIndexFromMetadata: wiring.createIndexFromMetadata,
@@ -5795,6 +5804,11 @@ function getVectorFieldPaths(schema) {
 }
 
 // src/engine/index-from-metadata.ts
+var RECOVERY_CONFIG_ERROR_CODES = /* @__PURE__ */ new Set([
+  ErrorCodes.CONFIG_INVALID,
+  ErrorCodes.SCHEMA_INVALID_TYPE,
+  ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION
+]);
 function resolveEmbeddingAdapter(deps, metadata, schema) {
   const name = metadata.embedding?.adapter ?? null;
   if (!metadata.embedding) return { adapter: null, name };
@@ -5813,6 +5827,15 @@ function resolveEmbeddingAdapter(deps, metadata, schema) {
   }
   return { adapter: candidate, name };
 }
+function recoveryFailure(metadata, err) {
+  if (!(err instanceof NarsilError) || !RECOVERY_CONFIG_ERROR_CODES.has(err.code)) return err;
+  return new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
+    ...err.details,
+    indexName: metadata.indexName,
+    tokenizer: metadata.tokenizer,
+    stopWords: metadata.stopWords
+  });
+}
 async function createIndexFromMetadata(deps, metadata, loadData) {
   const existing = deps.indexRegistry.get(metadata.indexName);
   if (existing !== void 0) {
@@ -5823,19 +5846,18 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
   }
   const indexConfig = reconstructSchemaFromMetadata(metadata);
   const language = getLanguage(indexConfig.language ?? "english");
+  try {
+    indexConfig.schema = validateSchema(indexConfig.schema);
+    validatePatternValueLimit(indexConfig.patternValueLimit);
+  } catch (err) {
+    throw recoveryFailure(metadata, err);
+  }
   const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
   try {
     validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
     if (loadData) deps.executor.createIndex(metadata.indexName, indexConfig, language);
   } catch (err) {
-    if (err instanceof NarsilError && err.code === ErrorCodes.CONFIG_INVALID) {
-      throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
-        indexName: metadata.indexName,
-        tokenizer: metadata.tokenizer,
-        stopWords: metadata.stopWords
-      });
-    }
-    throw err;
+    throw recoveryFailure(metadata, err);
   }
   deps.indexRegistry.set(metadata.indexName, {
     config: indexConfig,
@@ -6442,21 +6464,10 @@ function createHeapPressureNotifier(deps) {
 }
 
 // src/engine/watermark.ts
-function createWatermarkNotifier(deps) {
+function createWatermarkNotifier(deps, scope = "index") {
   const latchedCapacity = /* @__PURE__ */ new Map();
-  function check(indexName) {
-    const partitionConfig = deps.getPartitionConfig(indexName);
-    const maxDocs = partitionConfig?.maxDocsPerPartition;
-    const watermark = partitionConfig?.watermark;
-    if (maxDocs === void 0 || watermark === void 0) {
-      latchedCapacity.delete(indexName);
-      return;
-    }
-    const manager = deps.getManager(indexName);
-    if (!manager) {
-      latchedCapacity.delete(indexName);
-      return;
-    }
+  const latchedPartitions = /* @__PURE__ */ new Map();
+  function checkIndex(indexName, manager, maxDocs, watermark) {
     const partitionCount = manager.partitionCount;
     const capacity = maxDocs * partitionCount;
     const documentCount = manager.countDocuments();
@@ -6469,19 +6480,63 @@ function createWatermarkNotifier(deps) {
     latchedCapacity.set(indexName, capacity);
     deps.emit({ indexName, documentCount, capacity, partitionCount });
   }
+  function checkPartitions(indexName, manager, maxDocs, watermark, insertedDocIds) {
+    const threshold = watermark * maxDocs;
+    const latched = latchedPartitions.get(indexName) ?? /* @__PURE__ */ new Set();
+    latchedPartitions.set(indexName, latched);
+    const checked = /* @__PURE__ */ new Set();
+    for (const docId of insertedDocIds) {
+      const partitionId = manager.partitionIdOf(docId) ?? manager.routePartition(docId);
+      if (checked.has(partitionId)) continue;
+      checked.add(partitionId);
+      const documentCount = manager.getPartition(partitionId).count();
+      if (documentCount < threshold) {
+        latched.delete(partitionId);
+        continue;
+      }
+      if (latched.has(partitionId)) continue;
+      latched.add(partitionId);
+      deps.emit({ indexName, documentCount, capacity: maxDocs, partitionCount: manager.partitionCount, partitionId });
+    }
+  }
+  function check(indexName, inserted) {
+    const partitionConfig = deps.getPartitionConfig(indexName);
+    const maxDocs = partitionConfig?.maxDocsPerPartition;
+    const watermark = partitionConfig?.watermark;
+    if (maxDocs === void 0 || watermark === void 0) {
+      forget(indexName);
+      return;
+    }
+    const manager = deps.getManager(indexName);
+    if (manager === void 0) {
+      forget(indexName);
+      return;
+    }
+    if (scope === "index") {
+      checkIndex(indexName, manager, maxDocs, watermark);
+      return;
+    }
+    if (inserted !== void 0) {
+      checkPartitions(indexName, manager, maxDocs, watermark, typeof inserted === "string" ? [inserted] : inserted);
+    }
+  }
   function forget(indexName) {
     latchedCapacity.delete(indexName);
+    latchedPartitions.delete(indexName);
   }
   return { check, forget };
 }
 
 // src/engine/notifiers.ts
 function wireWatermarkNotifier(wiring) {
-  return createWatermarkNotifier({
-    getManager: wiring.getManager,
-    getPartitionConfig: (indexName) => wiring.indexRegistry.get(indexName)?.config.partitions,
-    emit: (payload) => emitEngineEvent(wiring.eventHandlers, "partitionWatermark", payload)
-  });
+  return createWatermarkNotifier(
+    {
+      getManager: wiring.getManager,
+      getPartitionConfig: (indexName) => wiring.indexRegistry.get(indexName)?.config.partitions,
+      emit: (payload) => emitEngineEvent(wiring.eventHandlers, "partitionWatermark", payload)
+    },
+    wiring.capacityScope
+  );
 }
 function warnUnheardHeapPressure(payload) {
   console.warn(
@@ -8452,8 +8507,32 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
   };
 }
 
+// src/engine/mutations/write-visibility.ts
+async function awaitWriteVisibility(ctx, indexName) {
+  await ctx.awaitRebalanceReplay(indexName);
+  await ctx.orchestrator.awaitWrites(indexName);
+}
+
+// src/engine/write-events.ts
+var WRITE_EVENT = "write";
+function emitWriteEvent(source, indexName) {
+  const handlers = source.eventHandlers.get(WRITE_EVENT);
+  if (handlers === void 0 || handlers.size === 0) return;
+  const emit = () => {
+    emitEngineEvent(source.eventHandlers, WRITE_EVENT, { indexName });
+  };
+  awaitWriteVisibility(source.mutationCtx, indexName).then(emit, emit);
+}
+async function emitWriteEventAfter(source, indexName, write) {
+  try {
+    return await write;
+  } finally {
+    emitWriteEvent(source, indexName);
+  }
+}
+
 // src/engine/core.ts
-function createEngineCore(config, hooks) {
+function createEngineCore(config, hooks, capacityScope = "index") {
   validateWorkerConfig(config?.workers, config?.lifecycle);
   const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
   const vectorCopyPolicy = {
@@ -8591,7 +8670,11 @@ function createEngineCore(config, hooks) {
       }
       return names;
     },
-    reloadIndex: (indexName) => durability?.manager.reloadIndex?.(indexName) ?? Promise.resolve(),
+    reloadIndex: (indexName) => emitWriteEventAfter(
+      { eventHandlers, mutationCtx },
+      indexName,
+      durability?.manager.reloadIndex?.(indexName) ?? Promise.resolve()
+    ),
     onError(error) {
       if (emitEngineEvent(eventHandlers, "invalidationError", { error }) === 0) {
         console.warn("Invalidation error:", error);
@@ -8601,7 +8684,8 @@ function createEngineCore(config, hooks) {
   const notifierWiring = {
     eventHandlers,
     indexRegistry,
-    getManager: (indexName) => executor.getManager(indexName)
+    getManager: (indexName) => executor.getManager(indexName),
+    capacityScope
   };
   const watermarkNotifier = wireWatermarkNotifier(notifierWiring);
   const heapPressureNotifier = wireHeapPressureNotifier(notifierWiring);
@@ -8645,6 +8729,7 @@ function createEngineCore(config, hooks) {
     pendingRebalanceWrites: (indexName) => waqMap.get(indexName)?.size ?? 0,
     rebalanceTargetPartitionCount: (indexName) => rebalanceTargets.get(indexName),
     bufferedDocState: (indexName, docId) => waqMap.get(indexName)?.bufferedDocState(docId),
+    capacityScope,
     checkWatermark: watermarkNotifier.check,
     checkHeapPressure: heapPressureNotifier.check
   };
@@ -8778,13 +8863,16 @@ async function runLifecycleHook(core, hookName, context) {
 async function announceIndexCreated(core, name, config) {
   await runLifecycleHook(core, "onIndexCreate", { indexName: name, config });
 }
-async function createEngineIndex(core, config, name, indexConfig, indexUuid) {
+async function createEngineIndex(core, config, name, requestedConfig, creation = {}) {
+  const { indexUuid, adapterBinding = "required" } = creation;
   core.guardShutdown();
   validateIndexName(name);
   if (core.indexRegistry.has(name)) {
     throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
   }
-  validateSchema(indexConfig.schema);
+  const schema = validateSchema(requestedConfig.schema);
+  const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
+  validatePatternValueLimit(indexConfig.patternValueLimit);
   validateVectorPromotion(indexConfig.vectorPromotion);
   validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
   validateBM25Params(indexConfig.bm25);
@@ -8810,24 +8898,31 @@ async function createEngineIndex(core, config, name, indexConfig, indexUuid) {
   let resolvedEmbeddingAdapter = null;
   let embeddingAdapterName = null;
   if (indexConfig.embedding) {
-    let configuredAdapter = indexConfig.embedding.adapter;
+    const configuredAdapter = indexConfig.embedding.adapter;
+    const fields = indexConfig.embedding.fields;
     if (typeof configuredAdapter === "string") {
       embeddingAdapterName = configuredAdapter;
       const registered = core.embeddingAdapters.get(configuredAdapter);
-      if (!registered) {
+      if (registered !== void 0) {
+        resolvedEmbeddingAdapter = validateEmbeddingConfig(
+          { fields, adapter: registered },
+          indexConfig.schema,
+          config?.embedding
+        );
+      } else if (adapterBinding === "required") {
         throw new NarsilError(
           ErrorCodes.EMBEDDING_CONFIG_INVALID,
           `Embedding adapter "${configuredAdapter}" is not registered on this engine`,
           { adapter: configuredAdapter, available: [...core.embeddingAdapters.keys()] }
         );
       }
-      configuredAdapter = registered;
+    } else if (adapterBinding === "required" || configuredAdapter !== void 0 || config?.embedding !== void 0) {
+      resolvedEmbeddingAdapter = validateEmbeddingConfig(
+        { fields, adapter: configuredAdapter },
+        indexConfig.schema,
+        config?.embedding
+      );
     }
-    resolvedEmbeddingAdapter = validateEmbeddingConfig(
-      { fields: indexConfig.embedding.fields, adapter: configuredAdapter },
-      indexConfig.schema,
-      config?.embedding
-    );
   }
   if (indexConfig.required && indexConfig.required.length > 0) {
     validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
@@ -8944,12 +9039,6 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
   }
 }
 
-// src/engine/mutations/write-visibility.ts
-async function awaitWriteVisibility(ctx, indexName) {
-  await ctx.awaitRebalanceReplay(indexName);
-  await ctx.orchestrator.awaitWrites(indexName);
-}
-
 // src/engine/mutations/insert.ts
 async function insertDocument(ctx, indexName, document, docId, options) {
   ctx.guardShutdown();
@@ -9042,7 +9131,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
     await applyInsert();
   }
   if (buffered) {
-    ctx.checkWatermark(indexName);
+    ctx.checkWatermark(indexName, resolvedDocId);
     ctx.checkHeapPressure(indexName);
     if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
     return resolvedDocId;
@@ -9066,7 +9155,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
       vecIndex.scheduleBuild();
     }
   }
-  ctx.checkWatermark(indexName);
+  ctx.checkWatermark(indexName, resolvedDocId);
   ctx.checkHeapPressure(indexName);
   await ctx.orchestrator.scaleOutReadyIndexes();
   if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
@@ -9177,6 +9266,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
   const required = entry.config.required;
   const admitted = [];
   const admittedIds = /* @__PURE__ */ new Set();
+  const pending = createPendingAdmissions();
   for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
     if (ctx.abortController.signal.aborted) break;
     const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
@@ -9209,15 +9299,16 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
         if (extractedVectors.size > 0) {
           validateVectorDimensions(extractedVectors, vecIndexes);
         }
-        admitInsert(ctx, indexName, manager, docId, admitted.length);
+        admitInsert(ctx, indexName, manager, docId, pending);
         if (admittedIds.has(docId) || manager.has(docId)) {
           throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
         }
-        validateDocument(partitionDoc, entry.config.schema);
+        validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
         if (strict) {
           validateDocumentStrict(partitionDoc, entry.config.schema);
         }
         admittedIds.add(docId);
+        recordPendingAdmission(ctx, pending, manager, docId);
         admitted.push({ docId, document: documents[i], partitionDoc, extractedVectors });
       } catch (err) {
         failed.push({ docId, error: asBatchInsertError(err) });
@@ -9616,7 +9707,7 @@ async function insertBatchViaSegments(ctx, indexName, documents, options) {
   for (const fieldPath of outcome.touchedVectorFields) {
     vecIndexes.get(fieldPath)?.scheduleBuild();
   }
-  ctx.checkWatermark(indexName);
+  ctx.checkWatermark(indexName, outcome.succeeded);
   ctx.checkHeapPressure(indexName);
   await ctx.orchestrator.scaleOutReadyIndexes();
   if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
@@ -9749,7 +9840,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
       vecIndex.scheduleBuild();
     }
   }
-  ctx.checkWatermark(indexName);
+  ctx.checkWatermark(indexName, succeeded);
   ctx.checkHeapPressure(indexName);
   await ctx.orchestrator.scaleOutReadyIndexes();
   if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
@@ -9982,7 +10073,7 @@ function applyOfUpdate(ctx, indexName, prepared) {
       const bufferedState = ctx.bufferedDocState(indexName, docId);
       const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
       if (!exists) {
-        manager.assertCapacity(ctx.pendingRebalanceWrites(indexName), ctx.rebalanceTargetPartitionCount(indexName));
+        admitBufferedUpsert(ctx, indexName, manager, docId);
       }
     }
     if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
@@ -10341,162 +10432,6 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
   ctx.checkWatermark(indexName);
 }
 
-// src/engine/snapshot/restore-config.ts
-function invalid2(message) {
-  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
-}
-function isPlainObject(value) {
-  return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-function restoredBm25(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!isPlainObject(raw)) {
-    invalid2("bm25 must be an object");
-  }
-  const { k1, b } = raw;
-  if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
-    invalid2("bm25 k1 must be a finite number");
-  }
-  if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
-    invalid2("bm25 b must be a finite number");
-  }
-  return {
-    ...k1 !== void 0 ? { k1 } : {},
-    ...b !== void 0 ? { b } : {}
-  };
-}
-function restoredStopWords(name, list) {
-  if (name !== void 0 && typeof name !== "string") {
-    invalid2("stopWords must be a name");
-  }
-  if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
-    invalid2("stopWordList must be a list of words");
-  }
-  if (typeof name === "string" && Array.isArray(list)) {
-    invalid2("stopWords and stopWordList cannot both be present");
-  }
-  if (typeof name === "string") {
-    return name;
-  }
-  if (Array.isArray(list)) {
-    return new Set(list);
-  }
-  return void 0;
-}
-function restoredPartitionConfig(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!isPlainObject(raw)) {
-    invalid2("partitionConfig must be an object");
-  }
-  const { maxDocsPerPartition, maxPartitions, watermark } = raw;
-  if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
-    invalid2("partitionConfig maxDocsPerPartition must be a positive integer");
-  }
-  if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
-    invalid2("partitionConfig maxPartitions must be a positive integer");
-  }
-  if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
-    invalid2("partitionConfig watermark must be above 0 and at most 1");
-  }
-  return {
-    ...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
-    ...typeof maxPartitions === "number" ? { maxPartitions } : {},
-    ...typeof watermark === "number" ? { watermark } : {}
-  };
-}
-function restoredScoring(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
-    invalid2("defaultScoring must be 'local', 'dfs', or 'broadcast'");
-  }
-  return raw;
-}
-function restoredBoolean(raw, field) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (typeof raw !== "boolean") {
-    invalid2(`${field} must be a boolean`);
-  }
-  return raw;
-}
-function restoredRequired(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
-    invalid2("required must be a list of field paths");
-  }
-  return raw;
-}
-function restoredVectorPromotion(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!isPlainObject(raw) || raw.hnswConfig !== void 0 && !isPlainObject(raw.hnswConfig)) {
-    invalid2("vectorPromotion must be an object");
-  }
-  return raw;
-}
-function restoredEmbedding(raw) {
-  if (raw === void 0) {
-    return void 0;
-  }
-  if (!isPlainObject(raw)) {
-    invalid2("embedding must be an object");
-  }
-  const { fields, adapter } = raw;
-  if (adapter !== void 0 && typeof adapter !== "string") {
-    invalid2("embedding adapter must be a name");
-  }
-  if (!isPlainObject(fields)) {
-    invalid2("embedding fields must be an object");
-  }
-  for (const value of Object.values(fields)) {
-    const isPath = typeof value === "string";
-    const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
-    if (!isPath && !isPathList) {
-      invalid2("embedding fields must map vector fields to source paths");
-    }
-  }
-  return {
-    fields,
-    ...adapter !== void 0 ? { adapter } : {}
-  };
-}
-function restoredConfigFields(envelope) {
-  if (envelope.tokenizer !== void 0 && typeof envelope.tokenizer !== "string") {
-    invalid2("tokenizer must be a name");
-  }
-  const stopWords = restoredStopWords(envelope.stopWords, envelope.stopWordList);
-  const bm25 = restoredBm25(envelope.bm25);
-  const surfaceForms = restoredBoolean(envelope.surfaceForms, "surfaceForms");
-  const partitions = restoredPartitionConfig(envelope.partitionConfig);
-  const defaultScoring = restoredScoring(envelope.defaultScoring);
-  const trackPositions = restoredBoolean(envelope.trackPositions, "trackPositions");
-  const strict = restoredBoolean(envelope.strict, "strict");
-  const required = restoredRequired(envelope.required);
-  const vectorPromotion = restoredVectorPromotion(envelope.vectorPromotion);
-  return {
-    ...typeof envelope.tokenizer === "string" ? { tokenizer: envelope.tokenizer } : {},
-    ...stopWords !== void 0 ? { stopWords } : {},
-    ...bm25 !== void 0 ? { bm25 } : {},
-    ...surfaceForms !== void 0 ? { surfaceForms } : {},
-    ...partitions !== void 0 ? { partitions } : {},
-    ...defaultScoring !== void 0 ? { defaultScoring } : {},
-    ...trackPositions !== void 0 ? { trackPositions } : {},
-    ...strict !== void 0 ? { strict } : {},
-    ...required !== void 0 ? { required } : {},
-    ...vectorPromotion !== void 0 ? { vectorPromotion } : {}
-  };
-}
-
 // src/engine/snapshot/index.ts
 async function recordRestoredDocuments(durability, indexName, restored) {
   if (restored.length === 0) return;
@@ -10552,6 +10487,7 @@ async function createSnapshot(manager, entry) {
     ...config.strict !== void 0 ? { strict: config.strict } : {},
     ...config.required !== void 0 ? { required: config.required } : {},
     ...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
+    ...config.patternValueLimit !== void 0 ? { patternValueLimit: config.patternValueLimit } : {},
     ...config.embedding !== void 0 ? {
       embedding: {
         fields: config.embedding.fields,
@@ -10604,8 +10540,8 @@ async function restoreFromSnapshot(indexName, data, deps) {
   const configFields = restoredConfigFields(envelope);
   const embedding = restoredEmbedding(envelope.embedding);
   const language = getLanguage(envelope.language);
-  const schema = envelope.schema;
-  validateSchema(schema);
+  const schema = validateSchema(envelope.schema);
+  validatePatternValueLimit(configFields.patternValueLimit);
   validateVectorPromotion(configFields.vectorPromotion);
   validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
   if (configFields.required !== void 0 && configFields.required.length > 0) {
@@ -10971,15 +10907,18 @@ function createNarsilFromCore(core, config) {
       release();
     }
   }
+  function withOpenIndexWrite(indexName, action) {
+    return emitWriteEventAfter(core, indexName, withOpenIndex(indexName, action));
+  }
   const narsil = {
     createIndex(name, indexConfig) {
-      return createEngineIndex(core, config, name, indexConfig);
+      return emitWriteEventAfter(core, name, createEngineIndex(core, config, name, indexConfig));
     },
     registerEmbeddingAdapter(name, adapter) {
       registerEngineEmbeddingAdapter(core, name, adapter);
     },
     dropIndex(name) {
-      return dropEngineIndex(core, name);
+      return emitWriteEventAfter(core, name, dropEngineIndex(core, name));
     },
     open(indexName) {
       guardShutdown();
@@ -11025,22 +10964,22 @@ function createNarsilFromCore(core, config) {
       return executor.getManager(indexName)?.getPartitionStats() ?? [];
     },
     insert(indexName, document, docId, options) {
-      return withOpenIndex(indexName, () => insertDocument(mutationCtx, indexName, document, docId, options));
+      return withOpenIndexWrite(indexName, () => insertDocument(mutationCtx, indexName, document, docId, options));
     },
     insertBatch(indexName, documents, options) {
-      return withOpenIndex(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents, options));
+      return withOpenIndexWrite(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents, options));
     },
     remove(indexName, docId, options) {
-      return withOpenIndex(indexName, () => removeDocument(mutationCtx, indexName, docId, options));
+      return withOpenIndexWrite(indexName, () => removeDocument(mutationCtx, indexName, docId, options));
     },
     removeBatch(indexName, docIds, options) {
-      return withOpenIndex(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
+      return withOpenIndexWrite(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
     },
     update(indexName, docId, document, options) {
-      return withOpenIndex(indexName, () => updateDocument(mutationCtx, indexName, docId, document, options));
+      return withOpenIndexWrite(indexName, () => updateDocument(mutationCtx, indexName, docId, document, options));
     },
     updateBatch(indexName, updates, options) {
-      return withOpenIndex(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
+      return withOpenIndexWrite(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
     },
     waitForWrites(indexName) {
       guardShutdown();
@@ -11088,7 +11027,7 @@ function createNarsilFromCore(core, config) {
     async rebuildAnalysis(indexName) {
       guardShutdown();
       requireIndex(indexName);
-      await withOpenIndex(indexName, () => core.analysisRebuild.rebuild(indexName));
+      await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
     },
     async snapshot(indexName) {
       guardShutdown();
@@ -11096,11 +11035,11 @@ function createNarsilFromCore(core, config) {
     },
     async restore(indexName, data) {
       guardShutdown();
-      await restoreFromSnapshot(indexName, data, {
+      const restored = restoreFromSnapshot(indexName, data, {
         executor,
         indexRegistry,
         getVectorFieldPaths,
-        dropIndex: narsil.dropIndex.bind(narsil),
+        dropIndex: (name) => dropEngineIndex(core, name),
         announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
         requireManager,
         durability,
@@ -11111,6 +11050,7 @@ function createNarsilFromCore(core, config) {
         clearAnalysisStale: core.analysisRebuild.clearStale,
         indexState: core.indexState
       });
+      await emitWriteEventAfter(core, indexName, restored);
       core.heapPressureNotifier.check(indexName);
     },
     async checkpoint(indexName) {
@@ -11123,7 +11063,7 @@ function createNarsilFromCore(core, config) {
     async clear(indexName) {
       guardShutdown();
       requireIndex(indexName);
-      await withOpenIndex(indexName, async () => {
+      await withOpenIndexWrite(indexName, async () => {
         await executor.execute({ type: "clear", indexName, requestId: indexName });
         await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
         core.watermarkNotifier.forget(indexName);
@@ -11132,7 +11072,7 @@ function createNarsilFromCore(core, config) {
     async rebalance(indexName, targetPartitionCount) {
       guardShutdown();
       requireIndex(indexName);
-      return withOpenIndex(
+      return withOpenIndexWrite(
         indexName,
         () => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
       );
@@ -11198,14 +11138,14 @@ function createNarsilFromCore(core, config) {
     async compactVectors(indexName, fieldName) {
       guardShutdown();
       requireIndex(indexName);
-      await withOpenIndex(indexName, async () => {
+      await withOpenIndexWrite(indexName, async () => {
         compactVectors(requireManager(indexName), indexName, fieldName);
       });
     },
     async optimizeVectors(indexName, fieldName) {
       guardShutdown();
       requireIndex(indexName);
-      await withOpenIndex(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
+      await withOpenIndexWrite(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
     },
     vectorMaintenanceStatus(indexName) {
       guardShutdown();