@delali/narsil 0.3.1-next.70 → 0.3.1-next.72

package/dist/adapters/filesystem.mjs

@@ -1,4 +1,4 @@
-import { NarsilError, ErrorCodes } from '../chunk-USKFU7J5.mjs';
+import { NarsilError, ErrorCodes } from '../chunk-RV77FR2F.mjs';
 
 // src/persistence/filesystem.ts
 var cachedFs = null;

package/dist/adapters/indexeddb.mjs

@@ -1,4 +1,4 @@
-import { NarsilError, ErrorCodes } from '../chunk-USKFU7J5.mjs';
+import { NarsilError, ErrorCodes } from '../chunk-RV77FR2F.mjs';
 
 // src/persistence/constants.ts
 var DEFAULT_INDEXEDDB_NAME = "narsil";

package/dist/{capabilities-Dt-7B4Wz.d.ts → capabilities-D6gjjQyA.d.ts}

@@ -1,6 +1,6 @@
 import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
-import { C as ClusterNamespace } from './config-rBpbgTw8.js';
-import { h as TaskStore } from './tasks-BZt_zes6.js';
+import { C as ClusterNamespace } from './config-BLguB-om.js';
+import { h as TaskStore } from './tasks-BECd0b__.js';
 
 /**
  * Context handed to {@link OnRequestHook} for every inbound request, captured

package/dist/{chunk-P4DSQ6M4.mjs → chunk-2X2ZQOAB.mjs}

@@ -1,9 +1,9 @@
-import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-GIVAGD3Q.mjs';
-import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, requireValidFilter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex, fuzzyTermMatches } from './chunk-LMO5TJUS.mjs';
-import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isWordIndexedFieldType, isGeopointOrVectorType, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-653BNMAF.mjs';
+import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-XZ4PAREY.mjs';
+import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, requireValidFilter, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex, fuzzyTermMatches } from './chunk-QR4OKRVY.mjs';
+import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isWordIndexedFieldType, isGeopointOrVectorType, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-M6XF5HDE.mjs';
 import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
-import { compareCodePoints, compareComparableKeys } from './chunk-OI42GWLR.mjs';
-import { NarsilError, ErrorCodes } from './chunk-USKFU7J5.mjs';
+import { holdsUnpairedSurrogate, compareCodePoints, compareComparableKeys } from './chunk-LJYTW7GT.mjs';
+import { NarsilError, ErrorCodes } from './chunk-RV77FR2F.mjs';
 
 // src/search/cursor-binding.ts
 var ABSENT_TAG = 0;
@@ -225,6 +225,9 @@ function requireValidTypoTolerance(params) {
 }
 
 // src/engine/query/shared.ts
+function withPatternWork(context) {
+  return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap) };
+}
 function partitionsFor(manager, partitionIds) {
   return partitionsIn(manager, partitionIds);
 }
@@ -271,18 +274,19 @@ function broadcastStatsForWorker(params, context, scoring) {
   }
   return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
 }
-function searchOptionsFor(manager) {
+function searchOptionsFor(manager, patternWork) {
   return {
     bm25Params: manager.config.bm25,
     stopWords: manager.analysis.stopWords,
-    customTokenizer: manager.analysis.customTokenizer
+    customTokenizer: manager.analysis.customTokenizer,
+    patternWork
   };
 }
-function collectFilterDocIds(manager, params, schema, partitionIds) {
+function collectFilterDocIds(manager, params, schema, partitionIds, patternWork) {
   const filterDocIds = /* @__PURE__ */ new Set();
   if (!params.filters) return filterDocIds;
   for (const partition of partitionsFor(manager, partitionIds)) {
-    const partitionFiltered = partition.applyFilters(params.filters, schema);
+    const partitionFiltered = partition.applyFilters(params.filters, schema, patternWork);
     for (const docId of partitionFiltered) {
       filterDocIds.add(docId);
     }
@@ -317,11 +321,11 @@ function requireValidQueryOptions(params, config) {
   const strict = config.strict === true;
   requireValidTypoTolerance(params);
   requireSortableFields(params.sort, config.schema, strict);
-  requireValidQueryFilter(params.filters, config.schema, strict);
+  requireValidQueryFilter(params.filters, config.schema);
   requireUsableQueryFields(params, config.schema, strict);
 }
-function requireValidQueryFilter(filters, schema, strict = false) {
-  if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema), strict);
+function requireValidQueryFilter(filters, schema) {
+  if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema));
 }
 function requireVectorSearchable(params, context, needsVector) {
   if (!needsVector) return;
@@ -395,6 +399,13 @@ function validateDocId(docId) {
   if (docId.includes("\0")) {
     throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
   }
+  if (holdsUnpairedSurrogate(docId)) {
+    throw new NarsilError(
+      ErrorCodes.DOC_VALIDATION_FAILED,
+      "Document ID holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8",
+      { docId }
+    );
+  }
 }
 function validatePartitionConfig(partitions) {
   const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
@@ -493,8 +504,9 @@ function firstIdAfter(sorted, anchor) {
   }
   return low;
 }
-function collectFilterMatches(partitions, filters, schema) {
-  const perPartition = partitions.map((partition) => partition.filterMatches(filters, schema));
+function collectFilterMatches(partitions, filters, schema, patternWorkCap) {
+  const meter = createPatternWorkMeter(patternWorkCap);
+  const perPartition = partitions.map((partition) => partition.filterMatches(filters, schema, meter));
   let total = 0;
   for (const matches of perPartition) total += matches.count;
   return { matchesFor: (partitionIndex) => perPartition[partitionIndex] ?? null, total };
@@ -572,7 +584,7 @@ function executeListDocuments(params, context) {
   const limit = clampListLimit(params.limit);
   const signature = sortSignatureOf(params.sort);
   requireSortableFields(params.sort, schema, context.strict === true);
-  requireValidQueryFilter(params.filters, schema, context.strict === true);
+  requireValidQueryFilter(params.filters, schema);
   const binding = listBindingOf(params.filters);
   let cursor = null;
   if (params.cursor !== void 0) {
@@ -580,7 +592,7 @@ function executeListDocuments(params, context) {
     requireMatchingCursor(cursor, params.cursor, signature, false, binding);
   }
   const partitions = context.partitionIds === void 0 ? manager.getAllPartitions() : context.partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
-  const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema);
+  const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema, context.patternWorkCap);
   const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
   const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
   const projection = resolveProjection(params.document);
@@ -947,7 +959,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
   const flatSchema = flattenSchema(schema);
   const fieldTypes = fields.map((field) => flatSchema[field]);
   const modes = sortModesOf(normalized);
-  const options = searchOptionsFor(manager);
+  const options = searchOptionsFor(manager, context.patternWork);
   const partitions = partitionsFor(manager, context.partitionIds);
   const partitionLimit = offset + limit + 1;
   const merged = [];
@@ -1148,7 +1160,7 @@ async function executeVectorSearch(params, context, limit, offset, cursorDepth =
   let filterDocIds;
   let filterPartitions;
   if (params.filters) {
-    filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds);
+    filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds, context.patternWork);
     if (filterDocIds.size === 0) {
       return { scored: [], totalMatched: 0 };
     }
@@ -1176,11 +1188,19 @@ async function textLeg(textOnlyParams, context) {
       indexName,
       textOnlyParams,
       broadcastStatsForWorker(textOnlyParams, context, scoring),
-      context.partitionIds
+      context.partitionIds,
+      context.patternWork
     );
     if (viaWorker) return viaWorker;
   }
-  return fanOutQuery(manager, textOnlyParams, language, config.schema, scoring, searchOptionsFor(manager));
+  return fanOutQuery(
+    manager,
+    textOnlyParams,
+    language,
+    config.schema,
+    scoring,
+    searchOptionsFor(manager, context.patternWork)
+  );
 }
 async function vectorLeg(vectorConfig, queryVector, context, filterDocIds, k) {
   const { manager } = context;
@@ -1206,7 +1226,7 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
   const fusion = resolveHybridFusion(params.hybrid);
   let filterDocIds;
   if (params.filters) {
-    filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds);
+    filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds, context.patternWork);
     if (filterDocIds.size === 0) {
       return { scored: [], totalMatched: 0 };
     }
@@ -1250,7 +1270,8 @@ function requireCursorDepth(cursor, signature, binding) {
   requireMatchingCursor(decoded, cursor, signature, true, binding);
   return decoded.depth ?? 0;
 }
-async function executeQuery(params, context) {
+async function executeQuery(params, callerContext) {
+  const context = withPatternWork(callerContext);
   const { manager, language, config, workerSearch, indexName } = context;
   const startTime = now();
   const limit = clampLimit(params.limit);
@@ -1296,11 +1317,24 @@ async function executeQuery(params, context) {
       fanOutResult = await executeHybridSearch(params, context, limit, offset, depth);
     } else {
       const scoring = scoringConfigFor(params, context);
-      const workerResult = workerSearch ? await workerSearch(indexName, params, broadcastStatsForWorker(params, context, scoring), context.partitionIds) : null;
+      const workerResult = workerSearch ? await workerSearch(
+        indexName,
+        params,
+        broadcastStatsForWorker(params, context, scoring),
+        context.partitionIds,
+        context.patternWork
+      ) : null;
       if (workerResult) {
         fanOutResult = workerResult;
       } else {
-        fanOutResult = await fanOutQuery(manager, params, language, config.schema, scoring, searchOptionsFor(manager));
+        fanOutResult = await fanOutQuery(
+          manager,
+          params,
+          language,
+          config.schema,
+          scoring,
+          searchOptionsFor(manager, context.patternWork)
+        );
       }
     }
     const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
@@ -1420,7 +1454,8 @@ async function executeQuery(params, context) {
     coverage: coverageFor(manager, context.partitionIds)
   };
 }
-async function executePreflight(params, context) {
+async function executePreflight(params, callerContext) {
+  const context = withPatternWork(callerContext);
   const { manager, language, config, workerSearch, indexName } = context;
   const startTime = now();
   requireKnownMode(params);
@@ -1446,12 +1481,18 @@ async function executePreflight(params, context) {
     countExact = result.matchedExact !== false;
   } else if (countsWithoutScores(params)) {
     totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
-      searchOptions: searchOptionsFor(manager),
+      searchOptions: searchOptionsFor(manager, context.patternWork),
       partitionIds: context.partitionIds
     });
   } else {
     const scoring = scoringConfigFor(params, context);
-    const workerResult = workerSearch ? await workerSearch(indexName, params, broadcastStatsForWorker(params, context, scoring), context.partitionIds) : null;
+    const workerResult = workerSearch ? await workerSearch(
+      indexName,
+      params,
+      broadcastStatsForWorker(params, context, scoring),
+      context.partitionIds,
+      context.patternWork
+    ) : null;
     if (workerResult) {
       totalMatched = workerResult.totalMatched;
     } else {
@@ -1461,7 +1502,7 @@ async function executePreflight(params, context) {
         language,
         config.schema,
         scoring,
-        searchOptionsFor(manager)
+        searchOptionsFor(manager, context.patternWork)
       );
       totalMatched = fanOutResult.totalMatched;
     }