@delali/narsil 0.3.1-next.77 → 0.3.1-next.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,12 +1,12 @@
1
- import { requirePatternSearchFor, patternIndexOf, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, requirePatternSearch, PATTERN_HOST, everyValueFacetConfig, mergeFacets, undeclaredFilterField, requireValidOperands, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, coveringRanges } from './chunk-D4EF7J3I.mjs';
1
+ import { requirePatternSearchFor, patternIndexOf, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-P2BEMMOD.mjs';
2
2
  import { getLanguage, detectRuntime } from './chunk-4MQ5F4IL.mjs';
3
- import { MAX_DOC_ID_LENGTH, GEO_UNIT_METRES } from './chunk-V3B24XR7.mjs';
3
+ import { MAX_DOC_ID_LENGTH, GEO_UNIT_METRES } from './chunk-IOZPBNSG.mjs';
4
4
  import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-AMGDG4ZJ.mjs';
5
5
  import { noteFallback } from './chunk-2E35UM3G.mjs';
6
- import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, PATTERN_RUN_CODE_POINTS, bitsetFromSet, isPatternTolerance, bitsetIsEmpty, bitsetAnd, bitsetNot } from './chunk-4344I4MR.mjs';
7
- import { encodePartialSpellings, countSpelling, expandTermPrefix, suggestDisplayTerms, spellingCountsOf, codePointOrder, encodeStringBlob, partialWordSearchModule, partitionDocumentFrequency } from './chunk-PJU6EF22.mjs';
8
- import { tokenizeOptions, getNestedValue, visitTokens, textFieldsChanged, getFlatSchema, tokenize, getOrCreateFieldNameIndex, tokenizeIterator, getFieldValueByInternalId } from './chunk-NO3ABFLE.mjs';
9
- import { partialFieldsOf, schemaFieldsOf, isWordIndexedBase, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, parsedTypeOf, isListBase, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-356UOTK2.mjs';
6
+ import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, PATTERN_RUN_CODE_POINTS, bitsetAndInto, bitsetFromSet, isPatternTolerance, bitsetIsEmpty, bitsetAnd, bitsetNot } from './chunk-4344I4MR.mjs';
7
+ import { encodePartialSpellings, countSpelling, expandTermPrefix, suggestDisplayTerms, sameFields, spellingCountsOf, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, partialWordSearchModule, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-RTR44V2A.mjs';
8
+ import { tokenizeOptions, getNestedValue, visitTokens, textFieldsChanged, getFlatSchema, tokenize, getOrCreateFieldNameIndex, getFieldValueByInternalId } from './chunk-WFVOBZS4.mjs';
9
+ import { partialFieldsOf, schemaFieldsOf, isWordIndexedBase, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, parsedTypeOf, isListBase, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-PXDQFYD5.mjs';
10
10
  import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
11
11
  import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
12
12
  import { compareCodePoints, toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-IBR6634K.mjs';
@@ -263,6 +263,9 @@ function createDocumentStore() {
263
263
  },
264
264
  toInternal(externalId) {
265
265
  return forwardMap.get(externalId);
266
+ },
267
+ isLive(internalId) {
268
+ return reverseMap[internalId] !== void 0;
266
269
  }
267
270
  };
268
271
  return {
@@ -1667,27 +1670,27 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
1667
1670
  const surfaceCounts = /* @__PURE__ */ new Map();
1668
1671
  if (state.trackPositions) {
1669
1672
  const tokenFreqs = /* @__PURE__ */ new Map();
1670
- for (const t of tokenizeIterator(text, language, opts)) {
1671
- const existing = tokenFreqs.get(t.token);
1673
+ visitTokens(text, language, opts, (token, position, surface) => {
1674
+ const existing = tokenFreqs.get(token);
1672
1675
  if (existing) {
1673
1676
  existing.count++;
1674
- existing.positions.push(t.position);
1677
+ existing.positions.push(position);
1675
1678
  } else {
1676
- tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
1679
+ tokenFreqs.set(token, { count: 1, positions: [position] });
1677
1680
  }
1678
- fieldTokenList.push(t.token);
1679
- countOccurrence(surfaceCounts, spellings, t.surface, t.token);
1680
- }
1681
+ fieldTokenList.push(token);
1682
+ countOccurrence(surfaceCounts, spellings, surface, token);
1683
+ });
1681
1684
  for (const [token, freq] of tokenFreqs) {
1682
1685
  state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex2, freq.positions);
1683
1686
  }
1684
1687
  } else {
1685
1688
  const tokenCounts = /* @__PURE__ */ new Map();
1686
- for (const t of tokenizeIterator(text, language, opts)) {
1687
- tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
1688
- fieldTokenList.push(t.token);
1689
- countOccurrence(surfaceCounts, spellings, t.surface, t.token);
1690
- }
1689
+ visitTokens(text, language, opts, (token, _position, surface) => {
1690
+ tokenCounts.set(token, (tokenCounts.get(token) ?? 0) + 1);
1691
+ fieldTokenList.push(token);
1692
+ countOccurrence(surfaceCounts, spellings, surface, token);
1693
+ });
1691
1694
  for (const [token, count] of tokenCounts) {
1692
1695
  state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
1693
1696
  }
@@ -1704,20 +1707,22 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
1704
1707
  if (state.trackPositions) {
1705
1708
  const tokenFreqs = /* @__PURE__ */ new Map();
1706
1709
  let positionOffset = 0;
1707
- for (const item of arr) {
1708
- let itemTokenCount = 0;
1709
- for (const t of tokenizeIterator(item, language, opts)) {
1710
- const existing = tokenFreqs.get(t.token);
1711
- if (existing) {
1712
- existing.count++;
1713
- existing.positions.push(positionOffset + t.position);
1714
- } else {
1715
- tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
1716
- }
1717
- fieldTokenList.push(t.token);
1718
- countOccurrence(surfaceCounts, spellings, t.surface, t.token);
1719
- itemTokenCount++;
1710
+ let itemTokenCount = 0;
1711
+ const addToken = (token, position, surface) => {
1712
+ const existing = tokenFreqs.get(token);
1713
+ if (existing) {
1714
+ existing.count++;
1715
+ existing.positions.push(positionOffset + position);
1716
+ } else {
1717
+ tokenFreqs.set(token, { count: 1, positions: [positionOffset + position] });
1720
1718
  }
1719
+ fieldTokenList.push(token);
1720
+ countOccurrence(surfaceCounts, spellings, surface, token);
1721
+ itemTokenCount++;
1722
+ };
1723
+ for (const item of arr) {
1724
+ itemTokenCount = 0;
1725
+ visitTokens(item, language, opts, addToken);
1721
1726
  positionOffset += itemTokenCount;
1722
1727
  }
1723
1728
  fieldLengths[fieldPath] = fieldTokenList.length;
@@ -1726,13 +1731,12 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
1726
1731
  }
1727
1732
  } else {
1728
1733
  const tokenCounts = /* @__PURE__ */ new Map();
1729
- for (const item of arr) {
1730
- for (const t of tokenizeIterator(item, language, opts)) {
1731
- tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
1732
- fieldTokenList.push(t.token);
1733
- countOccurrence(surfaceCounts, spellings, t.surface, t.token);
1734
- }
1735
- }
1734
+ const addToken = (token, _position, surface) => {
1735
+ tokenCounts.set(token, (tokenCounts.get(token) ?? 0) + 1);
1736
+ fieldTokenList.push(token);
1737
+ countOccurrence(surfaceCounts, spellings, surface, token);
1738
+ };
1739
+ for (const item of arr) visitTokens(item, language, opts, addToken);
1736
1740
  fieldLengths[fieldPath] = fieldTokenList.length;
1737
1741
  for (const [token, count] of tokenCounts) {
1738
1742
  state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
@@ -2179,46 +2183,7 @@ function narrowByFieldTests(fieldPath, f, parsed, context, narrow) {
2179
2183
  narrowByElements(fieldPath, f, parsed, context, bits, narrow);
2180
2184
  }
2181
2185
 
2182
- // src/filters/keys.ts
2183
- var FILTER_EXPRESSION_KEYS = ["fields", "and", "or", "not"];
2184
- var FIELD_FILTER_OPERATORS = [
2185
- "eq",
2186
- "ne",
2187
- "gt",
2188
- "lt",
2189
- "gte",
2190
- "lte",
2191
- "between",
2192
- "in",
2193
- "nin",
2194
- "startsWith",
2195
- "endsWith",
2196
- "contains",
2197
- "wildcard",
2198
- "regex",
2199
- "caseFold",
2200
- "tolerance",
2201
- "containsAll",
2202
- "matchesAny",
2203
- "size",
2204
- "exists",
2205
- "notExists",
2206
- "isEmpty",
2207
- "isNotEmpty",
2208
- "radius",
2209
- "polygon"
2210
- ];
2211
-
2212
2186
  // src/filters/evaluator.ts
2213
- var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
2214
- var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
2215
- function requireKnownKeys(value, known, describe) {
2216
- for (const key of Object.keys(value)) {
2217
- if (!known.has(key)) {
2218
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, describe(key), { key });
2219
- }
2220
- }
2221
- }
2222
2187
  var Narrowing = class {
2223
2188
  bits = null;
2224
2189
  narrow = (matched) => {
@@ -2230,24 +2195,11 @@ var Narrowing = class {
2230
2195
  }
2231
2196
  };
2232
2197
  function narrowByField(fieldPath, filter, context, narrow) {
2233
- requireKnownKeys(
2234
- filter,
2235
- OPERATOR_KEYS,
2236
- (key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
2237
- );
2238
- const type = context.fieldTypes[fieldPath];
2239
- const parsed = parsedTypeOf(type);
2240
- if (type === void 0 || parsed === void 0) throw undeclaredFilterField(fieldPath);
2241
- const f = filter;
2242
- requireValidOperands(fieldPath, f, type);
2243
- narrowByFieldTests(fieldPath, f, parsed, context, narrow.narrow);
2198
+ const parsed = parsedTypeOf(context.fieldTypes[fieldPath]);
2199
+ if (parsed === void 0) throw undeclaredFilterField(fieldPath);
2200
+ narrowByFieldTests(fieldPath, filter, parsed, context, narrow.narrow);
2244
2201
  }
2245
- function evaluateFilters(expression, context) {
2246
- requireKnownKeys(
2247
- expression,
2248
- EXPRESSION_KEYS,
2249
- (key) => `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`
2250
- );
2202
+ function evaluateExpression(expression, context) {
2251
2203
  const narrowing = new Narrowing();
2252
2204
  if (expression.fields) {
2253
2205
  for (const [fieldPath, filter] of Object.entries(expression.fields)) {
@@ -2255,32 +2207,24 @@ function evaluateFilters(expression, context) {
2255
2207
  }
2256
2208
  }
2257
2209
  if (expression.and?.length) {
2258
- for (const nested of expression.and) narrowing.narrow(evaluateFilters(nested, context));
2210
+ for (const nested of expression.and) narrowing.narrow(evaluateExpression(nested, context));
2259
2211
  }
2260
2212
  if (expression.or?.length) {
2261
2213
  const union = createBitSet(context.capacity);
2262
- for (const nested of expression.or) bitsetOrInto(union, evaluateFilters(nested, context));
2214
+ for (const nested of expression.or) bitsetOrInto(union, evaluateExpression(nested, context));
2263
2215
  narrowing.narrow(union);
2264
2216
  }
2265
2217
  if (expression.not) {
2266
- const excluded = evaluateFilters(expression.not, context);
2218
+ const excluded = evaluateExpression(expression.not, context);
2267
2219
  const kept = context.allDocIdsBitset.slice();
2268
2220
  bitsetAndNotInto(kept, excluded);
2269
2221
  narrowing.narrow(kept);
2270
2222
  }
2271
2223
  return narrowing.value(() => context.allDocIdsBitset);
2272
2224
  }
2273
-
2274
- // src/core/partition/constants.ts
2275
- var TARGET_BLOCK_ENTRIES = 128;
2276
- var PRUNING_REJECTION_SLACK = 1e-12;
2277
- var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
2278
- var SEGMENT_TOKEN_TABLE_START = 1 << 12;
2279
- var SEGMENT_POSTING_LOG_START = 1 << 14;
2280
- var SEGMENT_POSITION_POOL_START = 1 << 16;
2281
- var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
2282
- var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
2283
- var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
2225
+ function evaluateFilters(expression, context) {
2226
+ return evaluateExpression(validatedFilterOf(expression, context.fieldTypes), context);
2227
+ }
2284
2228
 
2285
2229
  // src/core/partition/sort-columns/order.ts
2286
2230
  var MISSING_RANK = -1;
@@ -2859,7 +2803,7 @@ function ensureColumns(state) {
2859
2803
  function textRangeBitsetOf(state, fieldPath, fieldType, bounds, capacity) {
2860
2804
  const column = ensureColumns(state).column(fieldPath, fieldType, "min");
2861
2805
  const resolver = state.docStore.resolver();
2862
- return selectTextRange(column, bounds, capacity, (internalId) => resolver.toExternal(internalId) !== void 0);
2806
+ return selectTextRange(column, bounds, capacity, (internalId) => resolver.isLive(internalId));
2863
2807
  }
2864
2808
  function sortedPageOf(state, request) {
2865
2809
  ensureColumns(state);
@@ -2996,7 +2940,7 @@ function partitionFilterMatches(state, filters, schema, meter) {
2996
2940
  const base = wordIndex << 5;
2997
2941
  while (word !== 0) {
2998
2942
  const trailingZeros = Math.clz32(word & -word) ^ 31;
2999
- if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
2943
+ if (resolver.isLive(base + trailingZeros)) count++;
3000
2944
  word &= word - 1;
3001
2945
  }
3002
2946
  }
@@ -3265,7 +3209,7 @@ function searchFulltextMatches(state, params) {
3265
3209
  const base = wordIndex << 5;
3266
3210
  while (word !== 0) {
3267
3211
  const trailingZeros = Math.clz32(word & -word) ^ 31;
3268
- if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
3212
+ if (resolver.isLive(base + trailingZeros)) count++;
3269
3213
  word &= word - 1;
3270
3214
  }
3271
3215
  }
@@ -3522,7 +3466,9 @@ var EMPTY_FROZEN_SLOT = {
3522
3466
  readerFor: () => null,
3523
3467
  spellings: () => null,
3524
3468
  bytes: () => 0,
3525
- afterRemoval: () => void 0
3469
+ afterRemoval: () => void 0,
3470
+ tableOfEveryDocument: () => null,
3471
+ countKeptSpellings: () => void 0
3526
3472
  };
3527
3473
  function createGrowingPartialWords(view) {
3528
3474
  return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
@@ -3779,7 +3725,7 @@ function kthBestScore(buffer, selection, resolver) {
3779
3725
  const internalId = touched[index];
3780
3726
  const score = scores[internalId];
3781
3727
  if (size === wanted && score <= selection[0]) continue;
3782
- if (resolver.toExternal(internalId) === void 0) continue;
3728
+ if (!resolver.isLive(internalId)) continue;
3783
3729
  let at = 0;
3784
3730
  if (size < wanted) {
3785
3731
  at = size++;
@@ -5305,7 +5251,7 @@ function compositeFilterMatches(subs, layout, filters, schema, meter) {
5305
5251
  const base = wordIndex << 5;
5306
5252
  while (word !== 0) {
5307
5253
  const trailingZeros = Math.clz32(word & -word) ^ 31;
5308
- if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
5254
+ if (resolver.isLive(base + trailingZeros)) count++;
5309
5255
  word &= word - 1;
5310
5256
  }
5311
5257
  }
@@ -5611,6 +5557,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5611
5557
  }
5612
5558
  return void 0;
5613
5559
  }
5560
+ function frozenStored(docId) {
5561
+ for (const segment of frozen) {
5562
+ const stored = segment.docStore.get(docId);
5563
+ if (stored !== void 0) return stored;
5564
+ }
5565
+ return void 0;
5566
+ }
5614
5567
  return {
5615
5568
  get partitionId() {
5616
5569
  return partitionId;
@@ -5783,18 +5736,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5783
5736
  expandPartialWords(subs(), analysis, []);
5784
5737
  },
5785
5738
  get(docId, projection) {
5786
- const owner = frozenOwner(docId);
5787
- if (owner !== void 0) {
5788
- const stored = owner.docStore.get(docId);
5789
- return stored === void 0 ? void 0 : cloneProjected(stored.fields, projection);
5790
- }
5739
+ const stored = frozenStored(docId);
5740
+ if (stored !== void 0) return cloneProjected(stored.fields, projection);
5791
5741
  return live.get(docId, projection);
5792
5742
  },
5793
5743
  getRef(docId) {
5794
- const owner = frozenOwner(docId);
5795
- if (owner !== void 0) {
5796
- return owner.docStore.get(docId)?.fields;
5797
- }
5744
+ const stored = frozenStored(docId);
5745
+ if (stored !== void 0) return stored.fields;
5798
5746
  return live.getRef(docId);
5799
5747
  },
5800
5748
  has(docId) {
@@ -8578,6 +8526,9 @@ function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
8578
8526
  },
8579
8527
  toInternal(externalId) {
8580
8528
  return liveOrdinal(externalId);
8529
+ },
8530
+ isLive(internalId) {
8531
+ return internalId >= 0 && internalId < capacity && !tombstones.has(internalId);
8581
8532
  }
8582
8533
  };
8583
8534
  return {
@@ -8738,7 +8689,6 @@ function wrapDocumentArray(documents) {
8738
8689
  }
8739
8690
 
8740
8691
  // src/core/partition/frozen/external-ids.ts
8741
- var encoder = new TextEncoder();
8742
8692
  var decoder = new TextDecoder();
8743
8693
  function encodeExternalIdTableData(docIds) {
8744
8694
  const { blob, offsets } = encodeStringBlob(docIds);
@@ -8757,11 +8707,13 @@ function wrapExternalIdTable(data) {
8757
8707
  const { blob, offsets, sortedOrdinals } = data;
8758
8708
  const count = sortedOrdinals.length;
8759
8709
  const decoded = new Array(count);
8710
+ const decodedOrdinals = /* @__PURE__ */ new Map();
8760
8711
  function idAt(ordinal) {
8761
8712
  let id = decoded[ordinal];
8762
8713
  if (id === void 0) {
8763
8714
  id = decoder.decode(blob.subarray(offsets[ordinal], offsets[ordinal + 1]));
8764
8715
  decoded[ordinal] = id;
8716
+ decodedOrdinals.set(id, ordinal);
8765
8717
  }
8766
8718
  return id;
8767
8719
  }
@@ -8770,7 +8722,9 @@ function wrapExternalIdTable(data) {
8770
8722
  bytes: blob.byteLength + offsets.byteLength + sortedOrdinals.byteLength,
8771
8723
  idAt,
8772
8724
  ordinalOf(docId) {
8773
- const query = encoder.encode(docId);
8725
+ const known = decodedOrdinals.get(docId);
8726
+ if (known !== void 0) return known;
8727
+ const query = encodeLookupKey(docId);
8774
8728
  let lo = 0;
8775
8729
  let hi = count;
8776
8730
  while (lo < hi) {
@@ -9214,7 +9168,7 @@ function createLazySurfaceReader(load) {
9214
9168
  }
9215
9169
 
9216
9170
  // src/core/partition/frozen/token-table.ts
9217
- var encoder2 = new TextEncoder();
9171
+ var encoder = new TextEncoder();
9218
9172
  var decoder3 = new TextDecoder();
9219
9173
  function compareBytes2(blob, start, end, query) {
9220
9174
  const length = end - start;
@@ -9278,7 +9232,7 @@ function wrapFrozenTokenTable(data) {
9278
9232
  size,
9279
9233
  bytes: blob.byteLength + offsets.byteLength + payloadSlots.byteLength + documentFrequencies.byteLength,
9280
9234
  find(token) {
9281
- const query = encoder2.encode(token);
9235
+ const query = encodeLookupKey(token);
9282
9236
  const at = lowerBound3(query);
9283
9237
  if (at >= size) return -1;
9284
9238
  return compareBytes2(blob, offsets[at], offsets[at + 1], query) === 0 ? at : -1;
@@ -9301,7 +9255,7 @@ function wrapFrozenTokenTable(data) {
9301
9255
  return documentFrequencies[sortedIndex];
9302
9256
  },
9303
9257
  prefixRange(prefix) {
9304
- const bytes = encoder2.encode(prefix);
9258
+ const bytes = encoder.encode(prefix);
9305
9259
  const start = lowerBound3(bytes);
9306
9260
  return { start, end: prefixEnd(start, bytes) };
9307
9261
  },
@@ -9319,8 +9273,8 @@ function wrapFrozenTokenTable(data) {
9319
9273
  fromCodePoint = unit;
9320
9274
  toCodePoint = nextCodePoint(unit);
9321
9275
  }
9322
- const from = encoder2.encode(String.fromCodePoint(fromCodePoint));
9323
- const to = encoder2.encode(String.fromCodePoint(toCodePoint));
9276
+ const from = encoder.encode(String.fromCodePoint(fromCodePoint));
9277
+ const to = encoder.encode(String.fromCodePoint(toCodePoint));
9324
9278
  return { start: lowerBound3(from), end: lowerBound3(to) };
9325
9279
  }
9326
9280
  };
@@ -9341,7 +9295,7 @@ function createFrozenTombstones() {
9341
9295
  return revision;
9342
9296
  },
9343
9297
  has(internalId) {
9344
- return removed.has(internalId);
9298
+ return removed.size > 0 && removed.has(internalId);
9345
9299
  },
9346
9300
  add(internalId) {
9347
9301
  if (removed.has(internalId)) return false;
@@ -9383,7 +9337,7 @@ function sharedSurfaceTable(payload) {
9383
9337
  return { blob: sharedUint8(table.blob), offsets: sharedUint32(table.offsets), counts: sharedUint32(table.counts) };
9384
9338
  }
9385
9339
  function sharePartialWordTable(table) {
9386
- if (typeof SharedArrayBuffer !== "function") return table;
9340
+ if (typeof SharedArrayBuffer !== "function" || table.runEntries.buffer instanceof SharedArrayBuffer) return table;
9387
9341
  return {
9388
9342
  fields: [...table.fields],
9389
9343
  spellings: sharedUint8(table.spellings),
@@ -9404,17 +9358,19 @@ function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
9404
9358
  const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
9405
9359
  return table === null ? null : sharePartialWordTable(table);
9406
9360
  }
9361
+ function partialWordsMatch(table, analysis) {
9362
+ if (table === null) return analysis === null || analysis.fields.length === 0;
9363
+ return analysis === null || sameFields(table.fields, analysis.fields);
9364
+ }
9407
9365
  function shareSegmentArrays(arrays, analysis, segmentId) {
9408
- return {
9409
- ...arrays,
9410
- segmentId: segmentId ?? generateId(),
9411
- partialWords: sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis)
9412
- };
9366
+ const partialWords = partialWordsMatch(arrays.partialWords, analysis) ? arrays.partialWords : sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis);
9367
+ return { ...arrays, segmentId: segmentId ?? generateId(), partialWords };
9413
9368
  }
9414
- function segmentArraysOf(payload, documentData) {
9369
+ function segmentArraysOf(payload, documentData, reusedPartialWords = null) {
9415
9370
  const tokenTable = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
9416
9371
  const idTable = encodeExternalIdTableData(payload.docIds);
9417
9372
  const surfaceTable = payload.surfaceForms === null ? null : encodeSurfaceTable(payload.surfaceForms);
9373
+ const partialWords = reusedPartialWords ?? (payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, wrapFrozenTokenTable(tokenTable)));
9418
9374
  return {
9419
9375
  documentCount: payload.documentCount,
9420
9376
  fieldNames: payload.fieldNames,
@@ -9436,14 +9392,15 @@ function segmentArraysOf(payload, documentData) {
9436
9392
  enums: payload.enums,
9437
9393
  geo: payload.geo,
9438
9394
  pattern: payload.pattern,
9439
- surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable
9395
+ surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable,
9396
+ partialWords
9440
9397
  };
9441
9398
  }
9442
9399
  function freezeSegmentShared(payload, documents, segmentId, analysis = null) {
9443
9400
  if (typeof SharedArrayBuffer !== "function") return null;
9444
9401
  return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId, analysis);
9445
9402
  }
9446
- function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null) {
9403
+ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null, reusedPartialWords = null) {
9447
9404
  if (typeof SharedArrayBuffer !== "function") return null;
9448
9405
  const tokenData = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
9449
9406
  const idData = encodeExternalIdTableData(payload.docIds);
@@ -9511,7 +9468,7 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis =
9511
9468
  postings: sharedUint32(entry.postings)
9512
9469
  })),
9513
9470
  surfaceTable: sharedSurfaceTable(payload),
9514
- partialWords: sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis)
9471
+ partialWords: reusedPartialWords === null ? sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis) : sharePartialWordTable(reusedPartialWords)
9515
9472
  };
9516
9473
  }
9517
9474
 
@@ -9547,11 +9504,6 @@ function buildGeoReaders(entries) {
9547
9504
  }
9548
9505
  return readers;
9549
9506
  }
9550
- function* survivingDocuments(documentSource, tombstones) {
9551
- for (let ordinal = 0; ordinal < documentSource.count; ordinal++) {
9552
- if (!tombstones.has(ordinal)) yield documentSource.readAt(ordinal);
9553
- }
9554
- }
9555
9507
  function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
9556
9508
  const tombstones = createFrozenTombstones();
9557
9509
  const postingViews = createFrozenPostingViews(source, tombstones);
@@ -9561,8 +9513,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
9561
9513
  }
9562
9514
  const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
9563
9515
  const partialWords = createFrozenPartialWords(partialWordTable, source.tokenTable, {
9516
+ count: documentSource.count,
9564
9517
  documentAt: (ordinal) => documentSource.readAt(ordinal),
9565
- survivors: () => survivingDocuments(documentSource, tombstones)
9518
+ isTombstoned: (ordinal) => tombstones.has(ordinal)
9566
9519
  });
9567
9520
  const numericIndexes = /* @__PURE__ */ new Map();
9568
9521
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
@@ -10014,9 +9967,9 @@ function uniqueSurvivors(range) {
10014
9967
  }
10015
9968
  return seen.size;
10016
9969
  }
10017
- function countRange(range, counts, keepsEveryDocument) {
9970
+ function countRange(range, counts, keepsEveryDocument2) {
10018
9971
  const { arrays } = range.input.segment;
10019
- if (keepsEveryDocument) {
9972
+ if (keepsEveryDocument2) {
10020
9973
  counts.postings += range.end - range.start;
10021
9974
  if (arrays.positionOffsets !== null) {
10022
9975
  counts.positions += arrays.positionOffsets[range.end] - arrays.positionOffsets[range.start];
@@ -10041,15 +9994,15 @@ function mergePostings2(inputs) {
10041
9994
  for (const input of inputs) {
10042
9995
  if (input.segment.arrays.positionOffsets !== null) hasPositions = true;
10043
9996
  }
10044
- const keepsEveryDocument = /* @__PURE__ */ new Map();
9997
+ const keepsEveryDocument2 = /* @__PURE__ */ new Map();
10045
9998
  for (const input of inputs)
10046
- keepsEveryDocument.set(
9999
+ keepsEveryDocument2.set(
10047
10000
  input,
10048
10001
  input.remap.every((target) => target >= 0)
10049
10002
  );
10050
10003
  walkTokens(inputs, (_token, ranges) => {
10051
10004
  const counts = { postings: 0, positions: 0 };
10052
- for (const range of ranges) countRange(range, counts, keepsEveryDocument.get(range.input) === true);
10005
+ for (const range of ranges) countRange(range, counts, keepsEveryDocument2.get(range.input) === true);
10053
10006
  if (counts.postings === 0) return;
10054
10007
  tokenCount += 1;
10055
10008
  totalPostings += counts.postings;
@@ -10073,7 +10026,7 @@ function mergePostings2(inputs) {
10073
10026
  const remap = range.input.remap;
10074
10027
  const fieldIndexRemap = range.input.fieldIndexRemap;
10075
10028
  docs += uniqueSurvivors(range);
10076
- if (keepsEveryDocument.get(range.input) === true) {
10029
+ if (keepsEveryDocument2.get(range.input) === true) {
10077
10030
  const count = range.end - range.start;
10078
10031
  for (let k = 0; k < count; k++) {
10079
10032
  postingDocIds[postingCursor + k] = remap[arrays.postingDocIds[range.start + k]];
@@ -10250,12 +10203,33 @@ function mergeDocuments2(inputs, documentCount) {
10250
10203
  }
10251
10204
  return { blob, offsets };
10252
10205
  }
10206
+ function keepsEveryDocument(remap) {
10207
+ for (let ordinal = 0; ordinal < remap.length; ordinal++) {
10208
+ if (remap[ordinal] < 0) return false;
10209
+ }
10210
+ return true;
10211
+ }
10212
+ function mergePartialWords(inputs, mergedTokenCount, analysis) {
10213
+ if (analysis === null || analysis.fields.length === 0) return { spellings: null, reused: null };
10214
+ if (inputs.length === 1) {
10215
+ const [only] = inputs;
10216
+ const table = only.segment.partialWords.tableOfEveryDocument(analysis);
10217
+ const keepsTokens = mergedTokenCount === only.segment.arrays.tokenTable.size;
10218
+ if (table !== null && keepsTokens && keepsEveryDocument(only.remap)) return { spellings: null, reused: table };
10219
+ }
10220
+ const counts = /* @__PURE__ */ new Map();
10221
+ for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
10222
+ return { spellings: encodePartialSpellings(analysis.fields, counts), reused: null };
10223
+ }
10253
10224
  function mergeFrozenSegments(segments, analysis) {
10254
10225
  if (typeof SharedArrayBuffer !== "function") return null;
10255
- const merged = mergeSegmentContents(segments.map((segment) => ({ segment })));
10256
- return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis);
10226
+ const merged = mergeSegmentContents(
10227
+ segments.map((segment) => ({ segment })),
10228
+ analysis
10229
+ );
10230
+ return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis, merged.reusedPartialWords);
10257
10231
  }
10258
- function mergeSegmentContents(selections) {
10232
+ function mergeSegmentContents(selections, analysis = null) {
10259
10233
  const docIds = [];
10260
10234
  const fieldNames = [];
10261
10235
  const inputs = selections.map((selection) => ({
@@ -10266,6 +10240,7 @@ function mergeSegmentContents(selections) {
10266
10240
  const documentCount = docIds.length;
10267
10241
  const postings = mergePostings2(inputs);
10268
10242
  const lengths = mergeFieldLengths(inputs, documentCount);
10243
+ const partialWords = mergePartialWords(inputs, postings.tokens.length, analysis);
10269
10244
  const payload = {
10270
10245
  documentCount,
10271
10246
  docIds,
@@ -10282,10 +10257,10 @@ function mergeSegmentContents(selections) {
10282
10257
  totalFieldLengths: lengths.totalFieldLengths,
10283
10258
  docFrequencies: postings.docFrequencies,
10284
10259
  surfaceForms: mergeSurfaceForms2(inputs),
10285
- partialSpellings: null,
10260
+ partialSpellings: partialWords.spellings,
10286
10261
  ...mergeFieldIndexes2(inputs)
10287
10262
  };
10288
- return { payload, documents: mergeDocuments2(inputs, documentCount) };
10263
+ return { payload, documents: mergeDocuments2(inputs, documentCount), reusedPartialWords: partialWords.reused };
10289
10264
  }
10290
10265
 
10291
10266
  // src/core/partition/segment-postings.ts
@@ -11138,4 +11113,4 @@ function createDirectExecutor(options) {
11138
11113
  };
11139
11114
  }
11140
11115
 
11141
- export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerPartialWordSearch, registerStopWords, registerTokenizer, requirePartialWordSearchFor, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
11116
+ export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerPartialWordSearch, registerStopWords, registerTokenizer, requirePartialWordSearchFor, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
@@ -1,6 +1,7 @@
1
1
  // src/filters/constants.ts
2
2
  var MAX_FILTER_DEPTH = 30;
3
3
  var MAX_FILTER_TESTS = 1024;
4
+ var FILTER_OPERAND_COPY_DEPTH = 3;
4
5
  var MAX_FILTER_ARRAY_SIZE = 65536;
5
6
  var GEO_UNIT_METRES = { km: 1e3, mi: 1609.344, m: 1 };
6
7
 
@@ -51,4 +52,4 @@ var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
51
52
  var SUGGEST_OVERSAMPLE_PADDING = 10;
52
53
  var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
53
54
 
54
- export { ABSOLUTE_WINDOWS_PATH_PATTERN, BATCH_CHUNK_SIZE, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, GEO_UNIT_METRES, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_FILTER_ARRAY_SIZE, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LINEAR_ALPHA, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };
55
+ export { ABSOLUTE_WINDOWS_PATH_PATTERN, BATCH_CHUNK_SIZE, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, FILTER_OPERAND_COPY_DEPTH, FOREIGN_STATS_TTL_MS, GEO_UNIT_METRES, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_FILTER_ARRAY_SIZE, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LINEAR_ALPHA, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };