@delali/narsil 0.3.1-next.129 → 0.3.1-next.131

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/chunk-26A5JFT2.mjs +89 -0
  4. package/dist/browser/{chunk-5D6NJD6C.mjs → chunk-2Y3CO4EQ.mjs} +342 -201
  5. package/dist/browser/{chunk-WLD6DDIX.mjs → chunk-AIIJDWMG.mjs} +1 -1
  6. package/dist/browser/{chunk-YAGXZU4W.mjs → chunk-D7WQF6DV.mjs} +2 -2
  7. package/dist/browser/{chunk-WZBTVTSU.mjs → chunk-M5UAAXGU.mjs} +2 -2
  8. package/dist/browser/{chunk-XTFJZHQP.mjs → chunk-NSCSR6MA.mjs} +15 -5
  9. package/dist/browser/{chunk-3NVZD52E.mjs → chunk-UDGB2MWT.mjs} +5 -28
  10. package/dist/browser/{chunk-II5POT55.mjs → chunk-VBBLTD7O.mjs} +4 -3
  11. package/dist/browser/chunk-VEGHFP2P.mjs +30 -0
  12. package/dist/browser/{durability-wiring-HCCC6BJB.mjs → durability-wiring-FNZSRH4N.mjs} +8 -6
  13. package/dist/browser/{module-S2VDU2JE.mjs → module-4HIMOQM5.mjs} +6 -5
  14. package/dist/browser/phrase-module-WESYVU7K.mjs +327 -0
  15. package/dist/browser/{restore-queue-BZ7WWEC4.mjs → restore-queue-GEZ54CA5.mjs} +1 -1
  16. package/dist/browser/{runtime-PL4TTFJE.mjs → runtime-3X7ZJ5T3.mjs} +2 -1
  17. package/dist/browser/{snapshot-RTUCHI2L.mjs → snapshot-V7VF6BN6.mjs} +25 -10
  18. package/dist/browser/{write-IMU7G5NU.mjs → write-YIC2MKRY.mjs} +7 -5
  19. package/dist/{capabilities-BTjv-tK8.d.ts → capabilities-B6WxrWxt.d.ts} +3 -10
  20. package/dist/{chunk-3XHFVYIQ.mjs → chunk-2C3WXBQS.mjs} +1 -1
  21. package/dist/{chunk-37RTD6KD.mjs → chunk-2JWNJSNE.mjs} +8 -8
  22. package/dist/{chunk-K76F5YYX.mjs → chunk-2MKX2SCO.mjs} +219 -58
  23. package/dist/{chunk-AN3CUMPZ.mjs → chunk-2SEL6SBS.mjs} +4 -95
  24. package/dist/{chunk-OPO5SE4J.mjs → chunk-2WBY4Y4M.mjs} +1 -1
  25. package/dist/{chunk-PTIJAZWL.mjs → chunk-4GDLFDPZ.mjs} +1 -1
  26. package/dist/chunk-4UMZ7KQL.mjs +17 -0
  27. package/dist/{chunk-TY64A3ZW.mjs → chunk-5LX6MOIB.mjs} +2 -2
  28. package/dist/{chunk-3LJVM6XM.mjs → chunk-5W7AEVTH.mjs} +2 -2
  29. package/dist/{chunk-PCRWCOZC.mjs → chunk-6M6SDH3S.mjs} +43 -23
  30. package/dist/{chunk-MGD7XETB.mjs → chunk-6SSVIW7Y.mjs} +19 -3
  31. package/dist/{chunk-5LC5VMDQ.mjs → chunk-7OIQ2SIO.mjs} +4 -4
  32. package/dist/{chunk-4XEVROSP.mjs → chunk-7XY6VD7B.mjs} +1 -1
  33. package/dist/{chunk-WI5Q3L43.mjs → chunk-CZ7ISLXA.mjs} +2 -0
  34. package/dist/{chunk-RC27KF7F.mjs → chunk-D5TFRB5G.mjs} +3 -15
  35. package/dist/{chunk-F4UKMTKM.mjs → chunk-EUCX5K74.mjs} +4 -4
  36. package/dist/{chunk-CTF6WFTX.mjs → chunk-F6AAXI7T.mjs} +3 -3
  37. package/dist/{chunk-DX2FI4LU.mjs → chunk-FQBFZAWG.mjs} +3 -1
  38. package/dist/{chunk-XTER67YM.mjs → chunk-HCGJKDZV.mjs} +1 -1
  39. package/dist/{chunk-WQYB4TNR.mjs → chunk-JJ6EUMXB.mjs} +2 -2
  40. package/dist/{chunk-2HQQ7Y5D.mjs → chunk-JT3YZGCE.mjs} +2 -2
  41. package/dist/chunk-JUKYYXSP.mjs +93 -0
  42. package/dist/{chunk-WUIDPBSY.mjs → chunk-K77FJXKS.mjs} +17 -7
  43. package/dist/{chunk-CSSCLRGY.mjs → chunk-NMXRXFGZ.mjs} +2 -2
  44. package/dist/{chunk-44TB6AIZ.mjs → chunk-NNO7CO3A.mjs} +1 -1
  45. package/dist/{chunk-EZIGAOUJ.mjs → chunk-OGLV4I3L.mjs} +6 -5
  46. package/dist/{chunk-FRSWYMXH.mjs → chunk-PATMDIMN.mjs} +1 -1
  47. package/dist/{chunk-AFOZR4ZQ.mjs → chunk-PU6SKH7O.mjs} +2 -2
  48. package/dist/{chunk-TB7YMDD5.mjs → chunk-RCMNL7RI.mjs} +650 -100
  49. package/dist/{chunk-Y4NOHICF.mjs → chunk-RNSC4NFI.mjs} +1 -1
  50. package/dist/{chunk-RYH3PASD.mjs → chunk-RWESFSJI.mjs} +21 -11
  51. package/dist/{chunk-QQS4QZQA.mjs → chunk-SOHMMNCH.mjs} +4 -4
  52. package/dist/{chunk-K73PZTBR.mjs → chunk-SZGRYEIC.mjs} +4 -4
  53. package/dist/{chunk-43Q7HAI2.mjs → chunk-T4OFG4IQ.mjs} +4 -4
  54. package/dist/{chunk-2YOAGAGF.mjs → chunk-TGHNXN7W.mjs} +59 -30
  55. package/dist/{chunk-J4LRE5Q7.mjs → chunk-WVRDRLIC.mjs} +6 -6
  56. package/dist/{chunk-XRXBCOA4.mjs → chunk-X5CNUKD5.mjs} +2 -2
  57. package/dist/{chunk-Y4US5EGH.mjs → chunk-Y2U5VVWX.mjs} +1 -1
  58. package/dist/{chunk-5VSTAJET.mjs → chunk-Y4BVTQ53.mjs} +2 -2
  59. package/dist/{chunk-M6MBSVIJ.mjs → chunk-ZR4VM4A6.mjs} +1 -1
  60. package/dist/{client-BOcveV7A.d.ts → client-gNAfaHcE.d.ts} +3 -3
  61. package/dist/client.d.ts +7 -8
  62. package/dist/client.mjs +2 -0
  63. package/dist/{config-CPmIY3aH.d.ts → config-CmYJO34X.d.ts} +3 -3
  64. package/dist/{events-pM0lna7M.d.ts → config-D1cDv3sE.d.ts} +431 -3
  65. package/dist/direct-executor-HQ4QVDX7.mjs +28 -0
  66. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  67. package/dist/distribution/coordinator/etcd.mjs +1 -1
  68. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  69. package/dist/distribution/transport/grpc.mjs +1 -1
  70. package/dist/distribution.d.ts +7 -8
  71. package/dist/distribution.mjs +85 -45
  72. package/dist/{durability-wiring-6R26AJDI.mjs → durability-wiring-S7OX2MAI.mjs} +27 -25
  73. package/dist/embeddings/openai.mjs +2 -2
  74. package/dist/{engine-JjRAXpT3.d.ts → engine-m_Ddr1bz.d.ts} +2 -2
  75. package/dist/{index-DkQ2DlgR.d.ts → index-HYnqS1SE.d.ts} +2 -2
  76. package/dist/index.browser.mjs +5 -3
  77. package/dist/index.d.ts +6 -7
  78. package/dist/index.mjs +20 -18
  79. package/dist/{long-query-thread-6VLA6YJI.mjs → long-query-thread-BSL25PD6.mjs} +17 -15
  80. package/dist/{module-S4WGMFNW.mjs → module-2E4XYSGT.mjs} +11 -11
  81. package/dist/partial-words.mjs +7 -6
  82. package/dist/pattern.mjs +4 -3
  83. package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
  84. package/dist/persistence/durability/checkpoint-worker.mjs +23 -21
  85. package/dist/phrase-module-JERF2IDG.mjs +329 -0
  86. package/dist/react.d.ts +11 -8
  87. package/dist/react.mjs +2 -0
  88. package/dist/{rerank-V4J4MIWR.mjs → rerank-DMMT4FZA.mjs} +6 -6
  89. package/dist/rerankers/http.mjs +2 -2
  90. package/dist/{restore-queue-PUGWCPCX.mjs → restore-queue-JAMNRC6T.mjs} +1 -1
  91. package/dist/{runtime-TBFBQQQO.mjs → runtime-7MLQ7J64.mjs} +7 -7
  92. package/dist/{schema-DGAP_8HJ.d.ts → schema-BTXbHCqA.d.ts} +8 -5
  93. package/dist/server.d.ts +198 -11
  94. package/dist/server.mjs +30 -19
  95. package/dist/{snapshot-PUVSMV7E.mjs → snapshot-DOQLGGEX.mjs} +44 -29
  96. package/dist/synonyms.mjs +4 -3
  97. package/dist/tasks-Co8HfRD7.d.ts +704 -0
  98. package/dist/{thread-TA7QQBWP.mjs → thread-SKO7CN3R.mjs} +16 -14
  99. package/dist/vector/search-worker.d.ts +1 -1
  100. package/dist/vector/search-worker.mjs +4 -4
  101. package/dist/workers/entry.mjs +7 -7
  102. package/dist/{write-TGKYT3C4.mjs → write-ZKR7SNRT.mjs} +22 -20
  103. package/openapi.json +5215 -0
  104. package/package.json +16 -8
  105. package/dist/config-C3YXk5Rn.d.ts +0 -412
  106. package/dist/direct-executor-ULA6V5HC.mjs +0 -26
  107. package/dist/tasks-txGvNz6h.d.ts +0 -302
@@ -1,6 +1,8 @@
1
- import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-WLD6DDIX.mjs';
1
+ import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
2
+ import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-AIIJDWMG.mjs';
2
3
  import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
3
- import { ErrorCodes, NarsilError, createNarsilError, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, MIN_DOCUMENTS_FOR_SEGMENTS, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-3NVZD52E.mjs';
4
+ import { ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-VEGHFP2P.mjs';
5
+ import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-UDGB2MWT.mjs';
4
6
  import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
5
7
 
6
8
  // src/analysis/registry.ts
@@ -55,10 +57,26 @@ function getStopWords(name) {
55
57
  }
56
58
  return stopWords2;
57
59
  }
60
+ function holdsPositionsOutOfOrder(entries) {
61
+ for (let index = 1; index < entries.length; index++) {
62
+ if (entries[index].position < entries[index - 1].position) return true;
63
+ }
64
+ return false;
65
+ }
66
+ function inPositionOrder(tokenizer) {
67
+ return {
68
+ tokenize(text) {
69
+ const entries = tokenizer.tokenize(text);
70
+ if (!holdsPositionsOutOfOrder(entries)) return entries;
71
+ return [...entries].sort((first, second) => first.position - second.position);
72
+ }
73
+ };
74
+ }
58
75
  function resolveIndexAnalysis(config) {
76
+ const tokenizer = typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer;
59
77
  return {
60
78
  stopWords: typeof config.stopWords === "string" ? getStopWords(config.stopWords) : config.stopWords,
61
- customTokenizer: typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer
79
+ customTokenizer: tokenizer === void 0 ? void 0 : inPositionOrder(tokenizer)
62
80
  };
63
81
  }
64
82
 
@@ -2232,6 +2250,17 @@ function requireValidPatternWorkCap(cap) {
2232
2250
  );
2233
2251
  }
2234
2252
 
2253
+ // src/core/constants.ts
2254
+ var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
2255
+ var MAX_TERM_FREQUENCY = 65535;
2256
+ var ARRAY_POSITION_GAP = 100;
2257
+ var DEFAULT_BM25_K1 = 1.2;
2258
+ var DEFAULT_BM25_B = 0.75;
2259
+ var MAX_BM25_K1 = 1e3;
2260
+ var MAX_BM25_B = 1;
2261
+ var PENDING_REMOVAL_MERGE_FLOOR = 64;
2262
+ var PENDING_REMOVAL_MERGE_DIVISOR = 256;
2263
+
2235
2264
  // src/core/id-generator.ts
2236
2265
  function generateId() {
2237
2266
  const bytes = new Uint8Array(16);
@@ -3243,16 +3272,6 @@ function createDocumentStore() {
3243
3272
  };
3244
3273
  }
3245
3274
 
3246
- // src/core/constants.ts
3247
- var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
3248
- var MAX_TERM_FREQUENCY = 65535;
3249
- var DEFAULT_BM25_K1 = 1.2;
3250
- var DEFAULT_BM25_B = 0.75;
3251
- var MAX_BM25_K1 = 1e3;
3252
- var MAX_BM25_B = 1;
3253
- var PENDING_REMOVAL_MERGE_FLOOR = 64;
3254
- var PENDING_REMOVAL_MERGE_DIVISOR = 256;
3255
-
3256
3275
  // src/core/fuzzy.ts
3257
3276
  var FIRST_SURROGATE = 55296;
3258
3277
  var LAST_SURROGATE = 57343;
@@ -3750,6 +3769,13 @@ var LivePostingView = class {
3750
3769
  const starts = this.store.pool.positionStarts;
3751
3770
  return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
3752
3771
  }
3772
+ get positionStore() {
3773
+ return this.holdsPositions ? this.store.positions.values : null;
3774
+ }
3775
+ positionStartAt(row) {
3776
+ const starts = this.store.pool.positionStarts;
3777
+ return starts === null ? 0 : starts[this.syncedStart + row] + 1;
3778
+ }
3753
3779
  copyPositionsAt(row, target, offset) {
3754
3780
  if (!this.holdsPositions) return 0;
3755
3781
  const starts = this.store.pool.positionStarts;
@@ -4472,48 +4498,6 @@ function createSurfaceRegistry() {
4472
4498
  };
4473
4499
  }
4474
4500
 
4475
- // src/core/bitset.ts
4476
- function createBitSet(capacity) {
4477
- return new Uint32Array(capacity + 31 >>> 5 || 1);
4478
- }
4479
- function bitsetSet(bs, index) {
4480
- const word = index >>> 5;
4481
- if (word < bs.length) {
4482
- bs[word] |= 1 << (index & 31);
4483
- }
4484
- }
4485
- function bitsetHas(bs, index) {
4486
- const word = index >>> 5;
4487
- if (word >= bs.length) return false;
4488
- return (bs[word] & 1 << (index & 31)) !== 0;
4489
- }
4490
- function bitsetAndInto(target, other) {
4491
- const shared2 = Math.min(target.length, other.length);
4492
- for (let i = 0; i < shared2; i++) target[i] &= other[i];
4493
- target.fill(0, shared2);
4494
- }
4495
- function bitsetOrInto(target, other) {
4496
- const shared2 = Math.min(target.length, other.length);
4497
- for (let i = 0; i < shared2; i++) target[i] |= other[i];
4498
- }
4499
- function bitsetAndNotInto(target, removed) {
4500
- const shared2 = Math.min(target.length, removed.length);
4501
- for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
4502
- }
4503
- function bitsetIsEmpty(bs) {
4504
- for (let i = 0; i < bs.length; i++) {
4505
- if (bs[i] !== 0) return false;
4506
- }
4507
- return true;
4508
- }
4509
- function bitsetFromSet(set, capacity) {
4510
- const bs = createBitSet(capacity);
4511
- for (const id of set) {
4512
- bs[id >>> 5] |= 1 << (id & 31);
4513
- }
4514
- return bs;
4515
- }
4516
-
4517
4501
  // src/core/partition/facets.ts
4518
4502
  var UNDECLARED_FIELD_TYPE = "";
4519
4503
  function keptFacetValues(limit) {
@@ -5784,7 +5768,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
5784
5768
  if (state.trackPositions) {
5785
5769
  const tokenFreqs = /* @__PURE__ */ new Map();
5786
5770
  let positionOffset = 0;
5787
- let itemTokenCount = 0;
5771
+ let lastItemPosition = -1;
5788
5772
  const addToken = (token, position, surface) => {
5789
5773
  const existing = tokenFreqs.get(token);
5790
5774
  if (existing) {
@@ -5795,12 +5779,12 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
5795
5779
  }
5796
5780
  fieldTokenList.push(token);
5797
5781
  countOccurrence(surfaceCounts, spellings, surface, token);
5798
- itemTokenCount++;
5782
+ lastItemPosition = position;
5799
5783
  };
5800
5784
  for (const item of arr) {
5801
- itemTokenCount = 0;
5785
+ lastItemPosition = -1;
5802
5786
  visitTokens(item, language, opts, addToken);
5803
- positionOffset += itemTokenCount;
5787
+ if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
5804
5788
  }
5805
5789
  fieldLengths[fieldPath] = fieldTokenList.length;
5806
5790
  for (const [token, freq] of tokenFreqs) {
@@ -6797,18 +6781,6 @@ function evaluateFilters(expression, context) {
6797
6781
  return evaluateExpression(validatedFilterOf(expression, context.fieldTypes), context);
6798
6782
  }
6799
6783
 
6800
- // src/core/partition/constants.ts
6801
- var TARGET_BLOCK_ENTRIES = 128;
6802
- var PRUNING_REJECTION_SLACK = 1e-12;
6803
- var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
6804
- var SEGMENT_TOKEN_TABLE_START = 1 << 12;
6805
- var SEGMENT_POSTING_LOG_START = 1 << 14;
6806
- var SEGMENT_POSITION_POOL_START = 1 << 16;
6807
- var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
6808
- var LOOKUP_KEY_SCRATCH_BYTES = 4096;
6809
- var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
6810
- var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
6811
-
6812
6784
  // src/core/partition/sort-columns/order.ts
6813
6785
  var MISSING_RANK = -1;
6814
6786
  var BYTES_PER_RANK = 4;
@@ -7549,6 +7521,57 @@ function partitionFilterMatches(state, filters, schema, meter) {
7549
7521
  };
7550
7522
  }
7551
7523
 
7524
+ // src/search/quotes.ts
7525
+ var QUOTE_CHARACTER = /["“”„]/;
7526
+ function holdsQuoteCharacter(term) {
7527
+ return QUOTE_CHARACTER.test(term);
7528
+ }
7529
+
7530
+ // src/search/phrase-code.ts
7531
+ var loaded = null;
7532
+ var loading = null;
7533
+ function requireLoadedPhraseCode() {
7534
+ if (loaded === null) {
7535
+ throw new NarsilError(ErrorCodes.SEARCH_PHRASE_NOT_READY, "The phrase code has not loaded. Try the query again");
7536
+ }
7537
+ return loaded;
7538
+ }
7539
+ function loadPhraseCode() {
7540
+ if (loading === null) {
7541
+ loading = import('./phrase-module-WESYVU7K.mjs').then(
7542
+ (code) => {
7543
+ loaded = code;
7544
+ return code;
7545
+ },
7546
+ (err) => {
7547
+ loading = null;
7548
+ throw new NarsilError(
7549
+ ErrorCodes.SEARCH_PHRASE_NOT_READY,
7550
+ "The phrase code failed to load. Remove the quotes, or try again once it can load",
7551
+ { cause: err instanceof Error ? err.message : String(err) }
7552
+ );
7553
+ }
7554
+ );
7555
+ }
7556
+ return loading;
7557
+ }
7558
+ function loadPhraseCodeFor(term) {
7559
+ if (loaded !== null || term === void 0 || !holdsQuoteCharacter(term)) return null;
7560
+ return loadPhraseCode();
7561
+ }
7562
+ function analyseQueryPhrases(term, language, analysis) {
7563
+ if (!holdsQuoteCharacter(term)) return null;
7564
+ return requireLoadedPhraseCode().analyseQueryPhrases(
7565
+ term.split(QUOTE_CHARACTER),
7566
+ (text) => tokenize(text, language, {
7567
+ stem: true,
7568
+ removeStopWords: true,
7569
+ customTokenizer: analysis.customTokenizer,
7570
+ stopWordOverride: analysis.stopWords
7571
+ }).tokens
7572
+ );
7573
+ }
7574
+
7552
7575
  // src/core/scorer.ts
7553
7576
  function computeIDF(docFrequency, totalDocs) {
7554
7577
  if (totalDocs === 0) return 0;
@@ -7861,11 +7884,20 @@ function searchedFieldMarks(fieldNames, fields) {
7861
7884
  function searchFulltextMatches(state, params) {
7862
7885
  const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
7863
7886
  const fieldNames = state.fieldNameTable.names;
7864
- const matched = createBitSet(state.docStore.internalIdCapacity());
7865
7887
  const searched = searchedFieldMarks(fieldNames, fields);
7866
7888
  const partial = partialFieldMarks(fieldNames, params.partialFields);
7867
7889
  const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
7868
- for (const qt of queryTokens) {
7890
+ const capacity = state.docStore.internalIdCapacity();
7891
+ const matched = params.phrases === void 0 ? createBitSet(capacity) : requireLoadedPhraseCode().matchPhrases(
7892
+ state.invertedIdx,
7893
+ params.phrases,
7894
+ searched,
7895
+ filterBitset,
7896
+ capacity,
7897
+ params.phraseWork,
7898
+ false
7899
+ ).matched;
7900
+ for (const qt of params.phrases === void 0 ? queryTokens : []) {
7869
7901
  const group = termGroupOf(termGroups, qt.token);
7870
7902
  for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
7871
7903
  markPostingList(matched, match.postingList, searched, filterBitset);
@@ -8751,10 +8783,10 @@ function searchFulltext(state, params) {
8751
8783
  globalStats,
8752
8784
  maxResults,
8753
8785
  termMatch,
8754
- filterBitset
8786
+ phrases
8755
8787
  } = params;
8756
8788
  const collectComponents = params.collectComponents !== false;
8757
- if (queryTokens.length === 0) {
8789
+ if (queryTokens.length === 0 && phrases === void 0) {
8758
8790
  return { scored: [], totalMatched: 0 };
8759
8791
  }
8760
8792
  const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
@@ -8772,9 +8804,48 @@ function searchFulltext(state, params) {
8772
8804
  const onePair = boundPairs.length === 2;
8773
8805
  const onlyK1 = boundPairs[0];
8774
8806
  const onlyB = boundPairs[1];
8807
+ const phraseMatches = phrases === void 0 ? null : requireLoadedPhraseCode().matchPhrases(
8808
+ state.invertedIdx,
8809
+ phrases,
8810
+ searchable,
8811
+ params.filterBitset,
8812
+ state.docStore.internalIdCapacity(),
8813
+ params.phraseWork,
8814
+ true
8815
+ );
8816
+ if (phraseMatches !== null && bitsetIsEmpty(phraseMatches.matched)) {
8817
+ return { scored: [], totalMatched: 0 };
8818
+ }
8819
+ const filterBitset = phraseMatches?.matched ?? params.filterBitset;
8775
8820
  function documentFrequency(token, list) {
8776
8821
  return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
8777
8822
  }
8823
+ function scorePhraseRows(rows, matched) {
8824
+ let idf = 0;
8825
+ for (let entry = 0; entry < rows.phrase.tokens.length; entry++) {
8826
+ idf += computeIDF(documentFrequency(rows.phrase.tokens[entry], rows.lists[entry]), totalDocs);
8827
+ }
8828
+ for (let row = 0; row < rows.docIds.length; row++) {
8829
+ const internalId = rows.docIds.values[row];
8830
+ if (!bitsetHas(matched, internalId)) continue;
8831
+ const fieldIndex = rows.fieldIndices.values[row];
8832
+ const frequency = rows.frequencies.values[row];
8833
+ const avgLen = averageLengths[fieldIndex];
8834
+ const fieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
8835
+ const score = scoresAreZero ? 0 : computeBM25WithIDF(
8836
+ frequency,
8837
+ idf,
8838
+ fieldLength,
8839
+ avgLen,
8840
+ onePair ? onlyK1 : k1s[fieldIndex],
8841
+ onePair ? onlyB : bs[fieldIndex]
8842
+ );
8843
+ addScore(scoreBuffer, internalId, score * boosts[fieldIndex]);
8844
+ if (components !== null) {
8845
+ recordComponents(components, internalId, fieldNames[fieldIndex], rows.phrase.key, frequency, fieldLength, idf);
8846
+ }
8847
+ }
8848
+ }
8778
8849
  const groupContext = {
8779
8850
  index: state.invertedIdx,
8780
8851
  fields: fieldScoring,
@@ -8857,7 +8928,7 @@ function searchFulltext(state, params) {
8857
8928
  mergeGroupContribution(internalId, contribution);
8858
8929
  }
8859
8930
  }
8860
- const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
8931
+ const prunableList = phrases === void 0 ? prunableSingleTermList(params, state.invertedIdx, fieldNames) : null;
8861
8932
  if (prunableList !== null && maxResults !== void 0) {
8862
8933
  return singleTermTopK({
8863
8934
  list: prunableList,
@@ -8868,7 +8939,7 @@ function searchFulltext(state, params) {
8868
8939
  resolver
8869
8940
  });
8870
8941
  }
8871
- const prunableLists = prunableMultiTermLists(params, state.invertedIdx);
8942
+ const prunableLists = phrases === void 0 ? prunableMultiTermLists(params, state.invertedIdx) : null;
8872
8943
  if (prunableLists !== null && maxResults !== void 0) {
8873
8944
  const pruned = multiTermTopK({
8874
8945
  terms: prunableLists,
@@ -8909,6 +8980,9 @@ function searchFulltext(state, params) {
8909
8980
  if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
8910
8981
  }
8911
8982
  }
8983
+ if (phraseMatches !== null) {
8984
+ for (const rows of phraseMatches.rows) scorePhraseRows(rows, phraseMatches.matched);
8985
+ }
8912
8986
  const totalMatched = scoreBuffer.touchedCount;
8913
8987
  const k = maxResults === void 0 ? totalMatched : Math.max(0, Math.min(maxResults, totalMatched));
8914
8988
  const scored = topKFromBuffer(scoreBuffer, k, resolver, components);
@@ -10566,6 +10640,9 @@ function heapPop(heap) {
10566
10640
  function collectCandidateTerms(subs, params) {
10567
10641
  const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
10568
10642
  const terms = /* @__PURE__ */ new Set();
10643
+ for (const phrase of params.phrases ?? []) {
10644
+ for (const token of phrase.tokens) terms.add(token);
10645
+ }
10569
10646
  for (const qt of queryTokens) {
10570
10647
  const group = termGroupOf(termGroups, qt.token);
10571
10648
  if (group !== void 0) {
@@ -10667,7 +10744,7 @@ function searchSingleSub(subs, layout, params, index) {
10667
10744
  return { ...result, matchedOrdinalBitset };
10668
10745
  }
10669
10746
  function compositeSearchFulltext(subs, layout, params) {
10670
- if (params.queryTokens.length === 0) {
10747
+ if (params.queryTokens.length === 0 && params.phrases === void 0) {
10671
10748
  return { scored: [], totalMatched: 0 };
10672
10749
  }
10673
10750
  const single = onlyPopulatedSub(subs);
@@ -11556,7 +11633,7 @@ var PREPARED_TYPO_TEST_BYTES = Math.ceil(MAX_FILTER_TESTS / TYPO_CLAIM_SLOTS_PER
11556
11633
  function capExceeded(cap) {
11557
11634
  return new NarsilError(
11558
11635
  ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
11559
- `The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
11636
+ `The engine stopped this search before it checked every candidate, because the work on its pattern tests and phrases passed the cap of ${cap} units. Narrow the tests or the phrases, or raise the patternWorkCap engine setting`,
11560
11637
  { cap }
11561
11638
  );
11562
11639
  }
@@ -11787,15 +11864,19 @@ function addPartitionTokens(tokens, addedMembers, partition, term, analysed, exp
11787
11864
  }
11788
11865
  }
11789
11866
  }
11790
- function addReachedTokens(tokens, partitions, term, expansion, language, analysis) {
11867
+ function addReachedTokens(tokens, partitions, term, requested, language, analysis) {
11868
+ if (requested.options.exact === true) return [];
11869
+ const quoted = analyseQueryPhrases(term, language, analysis);
11870
+ const words = quoted?.words ?? term;
11871
+ if (quoted !== null && words.trim().length === 0) return [];
11872
+ const expansion = quoted?.endsWithPhrase === true ? { schema: requested.schema, options: { ...requested.options, prefix: false } } : requested;
11791
11873
  const { options } = expansion;
11792
- if (options.exact === true) return [];
11793
11874
  if (!((options.tolerance ?? 0) > 0 || options.prefix === true || options.partialWords === true)) return [];
11794
- const analysed = queryTokensOf(term, language, analysis);
11875
+ const analysed = queryTokensOf(words, language, analysis);
11795
11876
  if (analysed === null) return [];
11796
11877
  const addedMembers = /* @__PURE__ */ new Set();
11797
11878
  for (const partition of partitions) {
11798
- addPartitionTokens(tokens, addedMembers, partition, term, analysed, expansion, language, analysis);
11879
+ addPartitionTokens(tokens, addedMembers, partition, words, analysed, expansion, language, analysis);
11799
11880
  }
11800
11881
  for (const token of addedMembers) tokens.add(token);
11801
11882
  return [...addedMembers];
@@ -11818,17 +11899,24 @@ function partitionsIn(manager, partitionIds) {
11818
11899
 
11819
11900
  // src/partitioning/distributed-scoring.ts
11820
11901
  function analysedQueryTokens(term, language, analysis) {
11821
- const { tokens } = tokenize(term, language, {
11822
- stem: true,
11823
- removeStopWords: true,
11824
- customTokenizer: analysis.customTokenizer,
11825
- stopWordOverride: analysis.stopWords
11826
- });
11902
+ const quoted = analyseQueryPhrases(term, language, analysis);
11903
+ const words = quoted?.words ?? term;
11827
11904
  const unique = /* @__PURE__ */ new Set();
11828
- for (const { token } of tokens) {
11829
- unique.add(token);
11905
+ if (quoted === null || words.trim().length > 0) {
11906
+ const { tokens } = tokenize(words, language, {
11907
+ stem: true,
11908
+ removeStopWords: true,
11909
+ customTokenizer: analysis.customTokenizer,
11910
+ stopWordOverride: analysis.stopWords
11911
+ });
11912
+ for (const { token } of tokens) {
11913
+ unique.add(token);
11914
+ }
11830
11915
  }
11831
11916
  for (const synonym of analysis.synonyms?.expand(unique)?.tokens ?? []) unique.add(synonym);
11917
+ for (const phrase of quoted?.phrases ?? []) {
11918
+ for (const token of phrase.tokens) unique.add(token);
11919
+ }
11832
11920
  return unique;
11833
11921
  }
11834
11922
  function copyNumericRecord(source) {
@@ -12227,6 +12315,14 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
12227
12315
  }
12228
12316
 
12229
12317
  // src/search/fulltext.ts
12318
+ function analysedWords(term, language, options) {
12319
+ return tokenize(term, language, {
12320
+ stem: true,
12321
+ removeStopWords: true,
12322
+ customTokenizer: options?.customTokenizer,
12323
+ stopWordOverride: options?.stopWords
12324
+ }).tokens;
12325
+ }
12230
12326
  function prepareFulltextQuery(partition, params, language, schema, options) {
12231
12327
  if (!params.term || params.term.trim().length === 0) return null;
12232
12328
  if (params.fields && params.fields.length === 0) return null;
@@ -12234,14 +12330,16 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
12234
12330
  if (params.fields) {
12235
12331
  validateSearchFields(params.fields, flatSchema);
12236
12332
  }
12237
- const queryTokenResult = tokenize(params.term, language, {
12238
- stem: true,
12239
- removeStopWords: true,
12240
- customTokenizer: options?.customTokenizer,
12241
- stopWordOverride: options?.stopWords
12333
+ const quoted = analyseQueryPhrases(params.term, language, {
12334
+ stopWords: options?.stopWords,
12335
+ customTokenizer: options?.customTokenizer
12242
12336
  });
12243
- if (queryTokenResult.tokens.length === 0) return null;
12244
- const queryTokens = deduplicateTokens(queryTokenResult.tokens);
12337
+ const wordParams = quoted === null ? params : requireLoadedPhraseCode().withWordsOnly(params, quoted);
12338
+ const phrases = quoted !== null && quoted.phrases.length > 0 ? quoted.phrases : void 0;
12339
+ const words = wordParams.term ?? "";
12340
+ const analysedTokens = quoted !== null && words.trim().length === 0 ? [] : analysedWords(words, language, options);
12341
+ if (analysedTokens.length === 0 && phrases === void 0) return null;
12342
+ const queryTokens = deduplicateTokens(analysedTokens);
12245
12343
  let filterBitset;
12246
12344
  if (params.filters) {
12247
12345
  filterBitset = partition.applyFiltersBitset(
@@ -12251,16 +12349,19 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
12251
12349
  );
12252
12350
  if (bitsetIsEmpty(filterBitset)) return null;
12253
12351
  }
12254
- const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
12352
+ if (analysedTokens.length === 0) return { queryTokens, phrases, filterBitset };
12353
+ const lastToken = analysedTokens[analysedTokens.length - 1];
12255
12354
  const synonyms = params.exact === true ? void 0 : options?.synonyms?.expand(queryTokens.map((qt) => qt.token));
12256
- const groups = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, options, synonyms);
12257
- return { queryTokens, ...groups, filterBitset };
12355
+ const groups = resolveTermGroups(partition, wordParams, language, schema, queryTokens, lastToken, options, synonyms);
12356
+ return { queryTokens, phrases, ...groups, filterBitset };
12258
12357
  }
12259
12358
  function fulltextMatches(partition, params, language, schema, options) {
12260
12359
  const prepared = prepareFulltextQuery(partition, params, language, schema, options);
12261
12360
  if (prepared === null) return null;
12262
12361
  return partition.searchFulltextMatches({
12263
12362
  queryTokens: prepared.queryTokens,
12363
+ phrases: prepared.phrases,
12364
+ phraseWork: options?.patternWork,
12264
12365
  termGroups: prepared.termGroups,
12265
12366
  partialFields: prepared.partialFields,
12266
12367
  fields: params.fields,
@@ -12275,13 +12376,15 @@ function fulltextSearch(partition, params, language, schema, options) {
12275
12376
  if (prepared === null) {
12276
12377
  return { scored: [], totalMatched: 0 };
12277
12378
  }
12278
- const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
12379
+ const { queryTokens, phrases, termGroups, partialFields, filterBitset } = prepared;
12279
12380
  const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
12280
12381
  const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
12281
12382
  const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
12282
12383
  const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
12283
12384
  const rawResult = partition.searchFulltext({
12284
12385
  queryTokens,
12386
+ phrases,
12387
+ phraseWork: options?.patternWork,
12285
12388
  termGroups,
12286
12389
  partialFields,
12287
12390
  fields: params.fields,
@@ -12299,7 +12402,7 @@ function fulltextSearch(partition, params, language, schema, options) {
12299
12402
  });
12300
12403
  let scored = rawResult.scored;
12301
12404
  const termMatch = params.termMatch ?? "any";
12302
- if (termMatch !== "any") {
12405
+ if (termMatch !== "any" && queryTokens.length > 0) {
12303
12406
  scored = filterByTermCoverage(
12304
12407
  scored,
12305
12408
  queryTokens,
@@ -12308,7 +12411,8 @@ function fulltextSearch(partition, params, language, schema, options) {
12308
12411
  params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
12309
12412
  params.exact ?? false,
12310
12413
  termGroups,
12311
- partialFields ?? []
12414
+ partialFields ?? [],
12415
+ phrases
12312
12416
  );
12313
12417
  }
12314
12418
  if (params.minScore !== void 0 && params.minScore > 0) {
@@ -12350,8 +12454,9 @@ function deduplicateTokens(tokens) {
12350
12454
  }
12351
12455
  return result;
12352
12456
  }
12353
- function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
12457
+ function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields, phrases) {
12354
12458
  const requiredCount = policy === "all" ? queryTokens.length : policy;
12459
+ const phraseKeys = phrases === void 0 ? null : new Set(phrases.map((phrase) => phrase.key));
12355
12460
  if (requiredCount <= 0) return scored;
12356
12461
  if (requiredCount > queryTokens.length) return [];
12357
12462
  const groups = /* @__PURE__ */ new Map();
@@ -12367,12 +12472,22 @@ function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLeng
12367
12472
  });
12368
12473
  }
12369
12474
  return scored.filter((doc) => {
12370
- const matched = countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields);
12475
+ const matched = countDocTermMatches(
12476
+ doc,
12477
+ queryTokens,
12478
+ tolerance,
12479
+ prefixLength,
12480
+ exact,
12481
+ groups,
12482
+ partialFields,
12483
+ phraseKeys
12484
+ );
12371
12485
  return matched >= requiredCount;
12372
12486
  });
12373
12487
  }
12374
- function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
12375
- const indexTokens = Object.keys(doc.idf);
12488
+ function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields, phraseKeys) {
12489
+ const componentKeys = Object.keys(doc.idf);
12490
+ const indexTokens = phraseKeys === null ? componentKeys : componentKeys.filter((key) => !phraseKeys.has(key));
12376
12491
  if (indexTokens.length === 0) return 0;
12377
12492
  let count = 0;
12378
12493
  for (const qt of queryTokens) {
@@ -12527,6 +12642,8 @@ function fanOutMatchCount(manager, params, language, schema, options) {
12527
12642
 
12528
12643
  // src/workers/query-actions.ts
12529
12644
  async function runQueryAction(entry, action) {
12645
+ const loadingPhraseCode = loadPhraseCodeFor(action.params.term);
12646
+ if (loadingPhraseCode !== null) await loadingPhraseCode;
12530
12647
  const patternWork = joinPatternWorkMeter(action.patternWork, action.params.filters);
12531
12648
  const bm25Params = action.type === "query" ? action.bm25 ?? void 0 : entry.config.bm25;
12532
12649
  const searchOptions = { ...entry.searchOptions, patternWork, bm25Params };
@@ -13647,6 +13764,10 @@ function createFrozenPostingViews(payload, tombstones, counts) {
13647
13764
  if (positionOffsets === null) return 0;
13648
13765
  return positionOffsets[start + row + 1] - positionOffsets[start + row];
13649
13766
  },
13767
+ positionStore: positionValues,
13768
+ positionStartAt(row) {
13769
+ return positionOffsets === null ? 0 : positionOffsets[start + row];
13770
+ },
13650
13771
  docIdSet: {
13651
13772
  get size() {
13652
13773
  return counts.documentFrequency(sortedIndex);
@@ -13765,7 +13886,7 @@ function addForms(target, forms) {
13765
13886
  function createFrozenSurfaceSlot(load, documents) {
13766
13887
  let registry = null;
13767
13888
  let uncounted = null;
13768
- function loaded2() {
13889
+ function loaded3() {
13769
13890
  if (registry === null) {
13770
13891
  const built = createSurfaceRegistry();
13771
13892
  const forms = load();
@@ -13779,26 +13900,26 @@ function createFrozenSurfaceSlot(load, documents) {
13779
13900
  }
13780
13901
  return {
13781
13902
  reader: {
13782
- candidatesForPrefix: (prefix) => loaded2().candidatesForPrefix(prefix),
13783
- stemChangedTotalFor: (token) => loaded2().stemChangedTotalFor(token),
13784
- size: () => loaded2().size(),
13785
- serialize: () => loaded2().serialize()
13903
+ candidatesForPrefix: (prefix) => loaded3().candidatesForPrefix(prefix),
13904
+ stemChangedTotalFor: (token) => loaded3().stemChangedTotalFor(token),
13905
+ size: () => loaded3().size(),
13906
+ serialize: () => loaded3().serialize()
13786
13907
  },
13787
13908
  holdsAny() {
13788
- return loaded2().size() > 0;
13909
+ return loaded3().size() > 0;
13789
13910
  },
13790
13911
  holds(spelling) {
13791
- return loaded2().holds(spelling);
13912
+ return loaded3().holds(spelling);
13792
13913
  },
13793
13914
  subtractSpellings(ordinal, spellings) {
13794
13915
  uncounted ??= new Uint8Array(documents.count);
13795
13916
  uncounted[ordinal] = 1;
13796
13917
  if (spellings.length === 0) return;
13797
- const target = loaded2();
13918
+ const target = loaded3();
13798
13919
  for (const spelling of spellings) target.subtract(spelling, 1);
13799
13920
  },
13800
13921
  addKeptForms(target, mergedOrdinals, analysis) {
13801
- const source = loaded2();
13922
+ const source = loaded3();
13802
13923
  if (analysis === null) {
13803
13924
  addForms(target, source.serialize());
13804
13925
  return;
@@ -14853,36 +14974,6 @@ function mergeSegmentContents(selections, analysis = null, textAnalysis = null)
14853
14974
  return { payload, documents: mergeDocuments2(inputs, documentCount), reusedPartialWords: partialWords.reused };
14854
14975
  }
14855
14976
 
14856
- // src/core/partition/growable-uint32.ts
14857
- var GrowableUint32 = class {
14858
- values;
14859
- length = 0;
14860
- constructor(capacity) {
14861
- this.values = new Uint32Array(capacity);
14862
- }
14863
- growTo(needed, kept) {
14864
- let capacity = this.values.length * 2;
14865
- while (capacity < needed) capacity *= 2;
14866
- const next = new Uint32Array(capacity);
14867
- next.set(this.values.subarray(0, kept));
14868
- this.values = next;
14869
- }
14870
- reserve(extra) {
14871
- const needed = this.length + extra;
14872
- if (needed > this.values.length) this.growTo(needed, this.length);
14873
- }
14874
- push(value) {
14875
- if (this.length === this.values.length) this.growTo(this.length + 1, this.length);
14876
- this.values[this.length++] = value;
14877
- }
14878
- ensureIndex(index) {
14879
- if (index >= this.values.length) this.growTo(index + 1, this.values.length);
14880
- }
14881
- clear() {
14882
- this.length = 0;
14883
- }
14884
- };
14885
-
14886
14977
  // src/core/partition/segment-postings.ts
14887
14978
  var TextPostings = class {
14888
14979
  constructor(trackPositions) {
@@ -15152,8 +15243,10 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
15152
15243
  const spellings = /* @__PURE__ */ new Map();
15153
15244
  let fieldSpellings = null;
15154
15245
  let positionOffset = 0;
15246
+ let lastItemPosition = -1;
15155
15247
  const addToken = (token, position, surface) => {
15156
15248
  postings.addToken(token, positionOffset + position);
15249
+ lastItemPosition = position;
15157
15250
  surfaces.count(surface, token);
15158
15251
  if (fieldSpellings !== null) countPartialSpelling(fieldSpellings, token, surface ?? token, 1);
15159
15252
  };
@@ -15172,8 +15265,9 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
15172
15265
  postings.beginField();
15173
15266
  positionOffset = 0;
15174
15267
  for (const text of fieldType === "string" ? [value] : value) {
15268
+ lastItemPosition = -1;
15175
15269
  visitTokens(text, language, tokenizing, addToken);
15176
- positionOffset = postings.fieldLength;
15270
+ if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
15177
15271
  }
15178
15272
  postings.endField(ordinal, fieldPath);
15179
15273
  lengths.record(ordinal, fieldPath, postings.fieldLength);
@@ -15291,30 +15385,30 @@ function runSegmentAction(entry, action) {
15291
15385
 
15292
15386
  // src/workers/vector-code.ts
15293
15387
  var VECTOR_TYPE_NAME = "vector";
15294
- var loaded = null;
15295
- var loading = null;
15388
+ var loaded2 = null;
15389
+ var loading2 = null;
15296
15390
  function loadedVectorCode() {
15297
- return loaded;
15391
+ return loaded2;
15298
15392
  }
15299
15393
  function requireLoadedVectorCode() {
15300
- if (loaded === null) throw new Error("The engine reached vector code before it loaded that code");
15301
- return loaded;
15394
+ if (loaded2 === null) throw new Error("The engine reached vector code before it loaded that code");
15395
+ return loaded2;
15302
15396
  }
15303
15397
  function loadVectorCode() {
15304
- if (loaded !== null) return Promise.resolve(loaded);
15305
- if (loading === null) {
15306
- loading = import('./module-S2VDU2JE.mjs').then(
15398
+ if (loaded2 !== null) return Promise.resolve(loaded2);
15399
+ if (loading2 === null) {
15400
+ loading2 = import('./module-4HIMOQM5.mjs').then(
15307
15401
  (code) => {
15308
- loaded = code;
15402
+ loaded2 = code;
15309
15403
  return code;
15310
15404
  },
15311
15405
  (err) => {
15312
- loading = null;
15406
+ loading2 = null;
15313
15407
  throw err;
15314
15408
  }
15315
15409
  );
15316
15410
  }
15317
- return loading;
15411
+ return loading2;
15318
15412
  }
15319
15413
  function mayDeclareVectorField(schema, depth) {
15320
15414
  if (!isPlainObject(schema) || depth > MAX_NESTING_DEPTH + 1) return false;
@@ -15326,7 +15420,7 @@ function mayDeclareVectorField(schema, depth) {
15326
15420
  return false;
15327
15421
  }
15328
15422
  function loadVectorCodeFor(schema) {
15329
- if (loaded !== null || !mayDeclareVectorField(schema, 1)) return null;
15423
+ if (loaded2 !== null || !mayDeclareVectorField(schema, 1)) return null;
15330
15424
  return loadVectorCode();
15331
15425
  }
15332
15426
 
@@ -15522,8 +15616,8 @@ function createDirectExecutor(options) {
15522
15616
  );
15523
15617
  }
15524
15618
  case "createIndex": {
15525
- const loading2 = loadVectorCodeFor(action.config.schema);
15526
- if (loading2 !== null) await loading2;
15619
+ const loading3 = loadVectorCodeFor(action.config.schema);
15620
+ if (loading3 !== null) await loading3;
15527
15621
  const language = getLanguage(action.config.language ?? "english");
15528
15622
  const manager = addIndex(
15529
15623
  action.indexName,
@@ -15892,6 +15986,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
15892
15986
  }
15893
15987
  }
15894
15988
  return {
15989
+ rebuildMode,
15895
15990
  markStale(index) {
15896
15991
  const existing = stale.get(index.indexName);
15897
15992
  if (existing !== void 0 && existing.run !== null) {
@@ -16020,6 +16115,17 @@ function resolveDurabilityTier(config) {
16020
16115
  return { kind: "snapshot", config: {}, adapter: config.persistence };
16021
16116
  }
16022
16117
 
16118
+ // src/engine/array-positions.ts
16119
+ function holdsStringArrayField(schema) {
16120
+ return schemaFieldsOf(schema).some((field) => field.base === "string[]");
16121
+ }
16122
+ function arrayPositionsAreStale(config, storedGap) {
16123
+ return storedGap !== ARRAY_POSITION_GAP && config.trackPositions !== false && holdsStringArrayField(config.schema);
16124
+ }
16125
+ function storedArrayPositionGap(value) {
16126
+ return typeof value === "number" ? value : null;
16127
+ }
16128
+
16023
16129
  // src/engine/recovery-schema.ts
16024
16130
  function unflattenSchema(flat) {
16025
16131
  const root = {};
@@ -16342,11 +16448,12 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
16342
16448
  documentCount: metadata.documentCount ?? 0,
16343
16449
  partitionCount: metadata.partitionCount,
16344
16450
  analysisRevision: metadata.analysisRevision ?? null,
16451
+ arrayPositionGap: metadata.arrayPositionGap ?? null,
16345
16452
  synonyms
16346
16453
  });
16347
16454
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
16348
16455
  else deps.indexState().registerClosed(metadata.indexName);
16349
- if (metadata.analysisRevision !== language.revision) {
16456
+ if (metadata.analysisRevision !== language.revision || arrayPositionsAreStale(indexConfig, metadata.arrayPositionGap ?? null)) {
16350
16457
  deps.analysisRebuild().markStale({
16351
16458
  indexName: metadata.indexName,
16352
16459
  language: language.name,
@@ -17662,10 +17769,10 @@ async function replicateToWorkers(state, action) {
17662
17769
  return;
17663
17770
  }
17664
17771
  const detached = detachCallerDocuments(state, action);
17665
- const loading2 = state.copyLoadBuffers.get(action.indexName);
17666
- if (loading2 !== void 0) {
17772
+ const loading3 = state.copyLoadBuffers.get(action.indexName);
17773
+ if (loading3 !== void 0) {
17667
17774
  queueForCopies(state, detached);
17668
- if (loading2.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop();
17775
+ if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop();
17669
17776
  return;
17670
17777
  }
17671
17778
  const queue = state.replicationQueues.get(action.indexName);
@@ -17676,9 +17783,9 @@ async function replicateToWorkers(state, action) {
17676
17783
  }
17677
17784
  function queueForCopies(state, action) {
17678
17785
  if (!("indexName" in action)) return;
17679
- const loading2 = state.copyLoadBuffers.get(action.indexName);
17680
- if (loading2 !== void 0) {
17681
- loading2.push(action);
17786
+ const loading3 = state.copyLoadBuffers.get(action.indexName);
17787
+ if (loading3 !== void 0) {
17788
+ loading3.push(action);
17682
17789
  return;
17683
17790
  }
17684
17791
  if (state.workerPool === null || !state.scaledOutIndexes.has(action.indexName)) return;
@@ -19345,10 +19452,10 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
19345
19452
  stopRequestThreads: () => stopRequestThreads(state),
19346
19453
  desyncIndex: (indexName) => {
19347
19454
  state.desyncedIndexes.add(indexName);
19348
- const loading2 = state.copyTransitions.get(indexName);
19455
+ const loading3 = state.copyTransitions.get(indexName);
19349
19456
  const wasScaledOut = state.scaledOutIndexes.delete(indexName);
19350
19457
  if (wasScaledOut) withdrawCopies(indexName);
19351
- return wasScaledOut || loading2 !== void 0 && loading2.kind !== "drop";
19458
+ return wasScaledOut || loading3 !== void 0 && loading3.kind !== "drop";
19352
19459
  },
19353
19460
  async releaseIndexCopies(indexName) {
19354
19461
  const hold = state.copyReleaseHolds.get(indexName);
@@ -19397,7 +19504,7 @@ async function emitWriteEventAfter(source, indexName, write) {
19397
19504
  // src/engine/core.ts
19398
19505
  async function createEngineCore(config, hooks, capacityScope = "index") {
19399
19506
  validateWorkerConfig(config?.workers, config?.lifecycle);
19400
- const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-PL4TTFJE.mjs')).resolveRerankSettings(config);
19507
+ const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-3X7ZJ5T3.mjs')).resolveRerankSettings(config);
19401
19508
  const eventHandlers = eventHandlersOf(config?.events);
19402
19509
  const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
19403
19510
  const vectorCopyPolicy = {
@@ -19408,7 +19515,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19408
19515
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
19409
19516
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
19410
19517
  }
19411
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-HCCC6BJB.mjs');
19518
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-FNZSRH4N.mjs');
19412
19519
  const filesystemDurability = durabilityTier?.kind === "wal";
19413
19520
  const executor = createDirectExecutor({
19414
19521
  vectorWorkerCopies: vectorCopyPolicy,
@@ -19574,7 +19681,9 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19574
19681
  resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
19575
19682
  recordCurrentAnalysis: (indexName) => {
19576
19683
  const entry = indexRegistry.get(indexName);
19577
- if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
19684
+ if (entry === void 0 || analysisRebuild.isStale(indexName)) return;
19685
+ entry.analysisRevision = entry.language.revision;
19686
+ entry.arrayPositionGap = ARRAY_POSITION_GAP;
19578
19687
  },
19579
19688
  durabilityManager: durability?.manager ?? null
19580
19689
  });
@@ -19705,7 +19814,7 @@ async function shutdownEngine(core) {
19705
19814
  }
19706
19815
  }
19707
19816
  if (core.rerankAdapters.size > 0) {
19708
- await (await import('./runtime-PL4TTFJE.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19817
+ await (await import('./runtime-3X7ZJ5T3.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19709
19818
  }
19710
19819
  await orchestrator.shutdown();
19711
19820
  await executor.shutdown();
@@ -20134,6 +20243,7 @@ function indexConfigurationOf(entry) {
20134
20243
  entry.language,
20135
20244
  entry.embeddingAdapterName,
20136
20245
  entry.analysisRevision,
20246
+ entry.arrayPositionGap,
20137
20247
  synonyms
20138
20248
  );
20139
20249
  }
@@ -20143,7 +20253,8 @@ function requireConfigurationWithinSizeLimit(config, language) {
20143
20253
  config,
20144
20254
  language,
20145
20255
  typeof adapter === "string" ? adapter : null,
20146
- language.revision
20256
+ language.revision,
20257
+ ARRAY_POSITION_GAP
20147
20258
  );
20148
20259
  if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
20149
20260
  const bytes = encode(payload).byteLength;
@@ -20154,12 +20265,13 @@ function requireConfigurationWithinSizeLimit(config, language) {
20154
20265
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
20155
20266
  );
20156
20267
  }
20157
- function configurationPayloadOf(config, language, adapterName, analysisRevision, synonyms) {
20268
+ function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
20158
20269
  const bm25 = config.bm25;
20159
20270
  return {
20160
20271
  schema: config.schema,
20161
20272
  language: language.name,
20162
20273
  ...analysisRevision !== null ? { analysisRevision } : {},
20274
+ ...arrayPositionGap !== null ? { arrayPositionGap } : {},
20163
20275
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
20164
20276
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
20165
20277
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
@@ -20336,6 +20448,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
20336
20448
  documentCount: 0,
20337
20449
  partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
20338
20450
  analysisRevision: language.revision,
20451
+ arrayPositionGap: ARRAY_POSITION_GAP,
20339
20452
  synonyms: synonyms?.encoded ?? null
20340
20453
  });
20341
20454
  await core.indexState.registerOpen(name);
@@ -23037,14 +23150,14 @@ function portabilityOperations(core, config, engine) {
23037
23150
  return {
23038
23151
  snapshot(indexName) {
23039
23152
  return withOpenIndex2(indexName, async () => {
23040
- const { createSnapshot } = await import('./snapshot-RTUCHI2L.mjs');
23153
+ const { createSnapshot } = await import('./snapshot-V7VF6BN6.mjs');
23041
23154
  return createSnapshot(snapshotDepsOf(indexName));
23042
23155
  });
23043
23156
  },
23044
23157
  async restore(indexName, data) {
23045
23158
  core.guardShutdown();
23046
23159
  validateIndexName(indexName);
23047
- const restored = import('./restore-queue-BZ7WWEC4.mjs').then(
23160
+ const restored = import('./restore-queue-GEZ54CA5.mjs').then(
23048
23161
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
23049
23162
  );
23050
23163
  await emitWriteEventAfter(core, indexName, restored);
@@ -23053,8 +23166,8 @@ function portabilityOperations(core, config, engine) {
23053
23166
  exportDocuments(indexName) {
23054
23167
  return withOpenIndex2(indexName, async () => {
23055
23168
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
23056
- import('./snapshot-RTUCHI2L.mjs'),
23057
- import('./write-IMU7G5NU.mjs')
23169
+ import('./snapshot-V7VF6BN6.mjs'),
23170
+ import('./write-YIC2MKRY.mjs')
23058
23171
  ]);
23059
23172
  const header = indexConfigurationOf(core.requireIndex(indexName));
23060
23173
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -23461,6 +23574,13 @@ function validQueryFilter(filters, schema) {
23461
23574
  if (filters === void 0 || filters === null) return filters;
23462
23575
  return validatedFilterOf(filters, flattenSchema(schema));
23463
23576
  }
23577
+ function requireUsablePhrases(params, context) {
23578
+ if (params.term === void 0) return;
23579
+ const quoted = analyseQueryPhrases(params.term, context.language, context.manager.analysis);
23580
+ if (quoted === null || quoted.keptPhraseCount === 0) return;
23581
+ requireLoadedPhraseCode().requirePhrasesWithinLimits(quoted, context.config.trackPositions !== false);
23582
+ context.requirePhraseReady?.();
23583
+ }
23464
23584
  function requireVectorSearchable(params, context, needsVector) {
23465
23585
  if (!needsVector) return;
23466
23586
  const field = params.vector?.field;
@@ -23866,8 +23986,8 @@ function highlightField(text, queryTokens, language, options) {
23866
23986
  }
23867
23987
 
23868
23988
  // src/engine/query/highlight.ts
23869
- function writtenWordsOf(params, language, analysis) {
23870
- return tokenize(params.term ?? "", language, {
23989
+ function writtenWordsOf(term, language, analysis) {
23990
+ return tokenize(term, language, {
23871
23991
  stem: false,
23872
23992
  removeStopWords: true,
23873
23993
  customTokenizer: analysis.customTokenizer,
@@ -23885,18 +24005,21 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
23885
24005
  highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
23886
24006
  };
23887
24007
  }
23888
- function applyHighlights(hits, params, language, analysis, schema, readStoredDocument, synonymLookup) {
23889
- if (!params.highlight) return;
23890
- const queryTokenResult = tokenize(params.term ?? "", language, {
24008
+ function applyHighlights(hits, request, language, analysis, schema, readStoredDocument, synonymLookup) {
24009
+ if (!request.highlight) return;
24010
+ const quoted = analyseQueryPhrases(request.term ?? "", language, analysis);
24011
+ const params = quoted === null ? request : requireLoadedPhraseCode().withWordsOnly(request, quoted);
24012
+ const words = params.term ?? "";
24013
+ const highlightConfig = request.highlight;
24014
+ const queryTokens = quoted !== null && words.trim().length === 0 ? [] : tokenize(words, language, {
23891
24015
  stem: true,
23892
24016
  removeStopWords: true,
23893
24017
  customTokenizer: analysis.customTokenizer,
23894
24018
  stopWordOverride: analysis.stopWords
23895
- });
23896
- const queryTokens = queryTokenResult.tokens;
24019
+ }).tokens;
23897
24020
  const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
23898
24021
  const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
23899
- const written = usesPrefix || usesPartialWords ? writtenWordsOf(params, language, analysis) : [];
24022
+ const written = usesPrefix || usesPartialWords ? writtenWordsOf(words, language, analysis) : [];
23900
24023
  let prefixToken;
23901
24024
  let completedToken;
23902
24025
  if (usesPrefix) {
@@ -23910,12 +24033,14 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
23910
24033
  }
23911
24034
  }
23912
24035
  const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
23913
- const synonyms = params.exact === true ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
24036
+ const wordSynonyms = params.exact === true || queryTokens.length === 0 ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
24037
+ const phraseTokens = quoted?.phrases.flatMap((phrase) => phrase.tokens) ?? [];
24038
+ const synonyms = phraseTokens.length === 0 ? wordSynonyms : [...wordSynonyms ?? [], ...phraseTokens];
23914
24039
  for (const hit of hits) {
23915
24040
  const highlights = {};
23916
24041
  const projected = hit.document;
23917
24042
  let stored;
23918
- for (const field of params.highlight.fields) {
24043
+ for (const field of highlightConfig.fields) {
23919
24044
  let fieldValue = projected[field];
23920
24045
  if (typeof fieldValue !== "string" && readStoredDocument !== void 0) {
23921
24046
  stored ??= readStoredDocument(hit.id);
@@ -23923,9 +24048,9 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
23923
24048
  }
23924
24049
  if (typeof fieldValue === "string") {
23925
24050
  highlights[field] = highlightField(fieldValue, queryTokens, language, {
23926
- preTag: params.highlight.preTag,
23927
- postTag: params.highlight.postTag,
23928
- maxSnippetLength: params.highlight.maxSnippetLength,
24051
+ preTag: highlightConfig.preTag,
24052
+ postTag: highlightConfig.postTag,
24053
+ maxSnippetLength: highlightConfig.maxSnippetLength,
23929
24054
  prefixToken,
23930
24055
  completedToken,
23931
24056
  tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
@@ -24347,13 +24472,17 @@ function requireCursorDepth(cursor, signature, binding) {
24347
24472
  }
24348
24473
  function hybridSearchOf(params, context, limit, offset, cursorDepth) {
24349
24474
  if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
24350
- return import('./runtime-PL4TTFJE.mjs').then(
24475
+ return import('./runtime-3X7ZJ5T3.mjs').then(
24351
24476
  (runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
24352
24477
  );
24353
24478
  }
24354
24479
  function executeQuery(request, callerContext) {
24355
- if (request.rerank === void 0) return runQuery(request, callerContext);
24356
- return import('./runtime-PL4TTFJE.mjs').then(
24480
+ if (request.rerank === void 0) {
24481
+ const loadingPhraseCode = loadPhraseCodeFor(request.term);
24482
+ if (loadingPhraseCode === null) return runQuery(request, callerContext);
24483
+ return loadingPhraseCode.then(() => runQuery(request, callerContext));
24484
+ }
24485
+ return import('./runtime-3X7ZJ5T3.mjs').then(
24357
24486
  (runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
24358
24487
  );
24359
24488
  }
@@ -24380,6 +24509,7 @@ async function runQuery(request, callerContext) {
24380
24509
  }
24381
24510
  const params = validQueryParams(request, config);
24382
24511
  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
24512
+ if (!isVectorOnly) requireUsablePhrases(params, context);
24383
24513
  const projection = resolveProjection(params.document);
24384
24514
  let paginated;
24385
24515
  let nextCursor;
@@ -24556,6 +24686,9 @@ async function executePreflight(request, callerContext) {
24556
24686
  const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
24557
24687
  const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
24558
24688
  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
24689
+ const loadingPhraseCode = loadPhraseCodeFor(params.term);
24690
+ if (loadingPhraseCode !== null) await loadingPhraseCode;
24691
+ if (!isVectorOnly) requireUsablePhrases(params, context);
24559
24692
  const requestedVectorField = params.vector?.field;
24560
24693
  const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
24561
24694
  let totalMatched;
@@ -24716,6 +24849,12 @@ async function readUnderOneAnalysis(core, indexName, read) {
24716
24849
  }
24717
24850
  return { result, analysisStale };
24718
24851
  }
24852
+ function phraseReadinessOf(core, indexName, config) {
24853
+ return () => {
24854
+ if (!core.analysisRebuild.isStale(indexName) || !holdsStringArrayField(config.schema)) return;
24855
+ requireLoadedPhraseCode().refusePhraseUntilRebuilt(indexName, core.analysisRebuild.rebuildMode);
24856
+ };
24857
+ }
24719
24858
  function scopedParams(params, options) {
24720
24859
  {
24721
24860
  return params;
@@ -24736,7 +24875,7 @@ async function runEngineQuery(core, indexName, params, options) {
24736
24875
  const manager = core.requireManager(indexName);
24737
24876
  const scoped = scopedParams(params, options);
24738
24877
  await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
24739
- const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-PL4TTFJE.mjs');
24878
+ const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-3X7ZJ5T3.mjs');
24740
24879
  const resolvedParams = await resolveVectorText(
24741
24880
  rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
24742
24881
  entry.embeddingAdapter,
@@ -24756,7 +24895,8 @@ async function runEngineQuery(core, indexName, params, options) {
24756
24895
  indexBm25: options?.indexBm25,
24757
24896
  cursorBinding: lazyQueryBinding(params),
24758
24897
  patternWorkCap: core.patternWorkCap,
24759
- rerank: rerank?.rerankRunnerFor(core)
24898
+ rerank: rerank?.rerankRunnerFor(core),
24899
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24760
24900
  };
24761
24901
  const { result, analysisStale } = await readUnderOneAnalysis(
24762
24902
  core,
@@ -24805,7 +24945,8 @@ async function runEnginePreflight(core, indexName, params, options) {
24805
24945
  partitionIds: options?.partitionIds,
24806
24946
  indexBm25: options?.indexBm25,
24807
24947
  cursorBinding: lazyQueryBinding(params),
24808
- patternWorkCap: core.patternWorkCap
24948
+ patternWorkCap: core.patternWorkCap,
24949
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24809
24950
  };
24810
24951
  const { result, analysisStale } = await readUnderOneAnalysis(
24811
24952
  core,
@@ -25171,4 +25312,4 @@ async function createNarsil(config) {
25171
25312
  // src/index.ts
25172
25313
  var VERSION = "0.1.0";
25173
25314
 
25174
- export { BYTES_TAG, DATE_TAG, DEFAULT_BM25_B, DEFAULT_BM25_K1, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
25315
+ export { ARRAY_POSITION_GAP, BYTES_TAG, DATE_TAG, DEFAULT_BM25_B, DEFAULT_BM25_K1, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, storedArrayPositionGap, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };