@delali/narsil 0.3.1-next.128 → 0.3.1-next.130

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/chunk-26A5JFT2.mjs +89 -0
  4. package/dist/browser/{chunk-4Y2EGA3U.mjs → chunk-2Y3CO4EQ.mjs} +878 -310
  5. package/dist/browser/{chunk-2CBQ2PCC.mjs → chunk-AIIJDWMG.mjs} +1 -1
  6. package/dist/browser/{chunk-P7C2S5JX.mjs → chunk-D7WQF6DV.mjs} +2 -2
  7. package/dist/browser/{chunk-V5DCQIOT.mjs → chunk-M5UAAXGU.mjs} +2 -2
  8. package/dist/browser/{chunk-B65QVTST.mjs → chunk-NSCSR6MA.mjs} +24 -9
  9. package/dist/browser/{chunk-LJS6PREA.mjs → chunk-UDGB2MWT.mjs} +7 -28
  10. package/dist/browser/{chunk-6POI2KNI.mjs → chunk-VBBLTD7O.mjs} +4 -3
  11. package/dist/browser/chunk-VEGHFP2P.mjs +30 -0
  12. package/dist/browser/{durability-wiring-PUC4QSCR.mjs → durability-wiring-FNZSRH4N.mjs} +8 -6
  13. package/dist/browser/{module-ACWDC7YI.mjs → module-4HIMOQM5.mjs} +6 -5
  14. package/dist/browser/phrase-module-WESYVU7K.mjs +327 -0
  15. package/dist/browser/{restore-queue-MLUEUEAN.mjs → restore-queue-GEZ54CA5.mjs} +1 -1
  16. package/dist/browser/{runtime-BX6XRFYR.mjs → runtime-3X7ZJ5T3.mjs} +2 -1
  17. package/dist/browser/{snapshot-U5Y346XG.mjs → snapshot-V7VF6BN6.mjs} +25 -10
  18. package/dist/browser/{write-GZHR6AVG.mjs → write-YIC2MKRY.mjs} +7 -5
  19. package/dist/{capabilities-9YwwCcqP.d.ts → capabilities-Bl8G9taj.d.ts} +10 -3
  20. package/dist/{chunk-VK45J47R.mjs → chunk-2C3WXBQS.mjs} +1 -1
  21. package/dist/{chunk-QSCVJJW5.mjs → chunk-2JWNJSNE.mjs} +8 -8
  22. package/dist/{chunk-IBGMFR5Y.mjs → chunk-2MKX2SCO.mjs} +481 -121
  23. package/dist/{chunk-P34VZEXB.mjs → chunk-2SEL6SBS.mjs} +4 -95
  24. package/dist/{chunk-7ZZQNOIY.mjs → chunk-2WBY4Y4M.mjs} +1 -1
  25. package/dist/{chunk-HFJXEFFV.mjs → chunk-4GDLFDPZ.mjs} +3 -3
  26. package/dist/chunk-4UMZ7KQL.mjs +17 -0
  27. package/dist/{chunk-ZR63OGT2.mjs → chunk-5LX6MOIB.mjs} +2 -2
  28. package/dist/{chunk-UKXMSO5H.mjs → chunk-6M6SDH3S.mjs} +189 -45
  29. package/dist/{chunk-IR4KQ4WB.mjs → chunk-6SSVIW7Y.mjs} +49 -31
  30. package/dist/{chunk-4IHMMDRF.mjs → chunk-7OIQ2SIO.mjs} +4 -4
  31. package/dist/{chunk-6ODIY6ZB.mjs → chunk-7XY6VD7B.mjs} +1 -1
  32. package/dist/{chunk-OBWUWZRA.mjs → chunk-CZ7ISLXA.mjs} +3 -0
  33. package/dist/{chunk-OFYKBKKV.mjs → chunk-D5TFRB5G.mjs} +3 -15
  34. package/dist/{chunk-KCQG2PJT.mjs → chunk-EUCX5K74.mjs} +4 -4
  35. package/dist/{chunk-HWQDZB2U.mjs → chunk-F6AAXI7T.mjs} +3 -3
  36. package/dist/{chunk-E74YLAIG.mjs → chunk-FQBFZAWG.mjs} +4 -1
  37. package/dist/{chunk-U6NGTWI3.mjs → chunk-HCGJKDZV.mjs} +1 -1
  38. package/dist/{chunk-SJEIIR56.mjs → chunk-JJ6EUMXB.mjs} +2 -2
  39. package/dist/{chunk-N57XXGPG.mjs → chunk-JT3YZGCE.mjs} +23 -4
  40. package/dist/chunk-JUKYYXSP.mjs +93 -0
  41. package/dist/{chunk-JIXO442E.mjs → chunk-K77FJXKS.mjs} +26 -10
  42. package/dist/{chunk-4PQNDHSA.mjs → chunk-NMXRXFGZ.mjs} +2 -2
  43. package/dist/{chunk-6PN3U6XW.mjs → chunk-NNO7CO3A.mjs} +1 -1
  44. package/dist/{chunk-542TD4BL.mjs → chunk-PATMDIMN.mjs} +1 -1
  45. package/dist/{chunk-DG62H7OD.mjs → chunk-PU6SKH7O.mjs} +2 -2
  46. package/dist/chunk-Q2AP2FSN.mjs +15 -0
  47. package/dist/{chunk-22CK2LQO.mjs → chunk-QQTXT65X.mjs} +6 -5
  48. package/dist/{chunk-LXWDQLTW.mjs → chunk-RNSC4NFI.mjs} +1 -1
  49. package/dist/{chunk-JEXJDCQH.mjs → chunk-RWESFSJI.mjs} +28 -13
  50. package/dist/{chunk-SEG4CL6W.mjs → chunk-SOHMMNCH.mjs} +4 -4
  51. package/dist/{chunk-OGFSUXYY.mjs → chunk-SZGRYEIC.mjs} +4 -4
  52. package/dist/{chunk-LOBYK2FG.mjs → chunk-T4OFG4IQ.mjs} +20 -9
  53. package/dist/{chunk-ENBN5VQ6.mjs → chunk-TGHNXN7W.mjs} +150 -52
  54. package/dist/{chunk-Z4DZ4KK2.mjs → chunk-UJWGQIXU.mjs} +1 -0
  55. package/dist/{chunk-ANFNYCQG.mjs → chunk-WVRDRLIC.mjs} +6 -6
  56. package/dist/{chunk-62IPM62F.mjs → chunk-X5CNUKD5.mjs} +2 -2
  57. package/dist/{chunk-IE43SRL3.mjs → chunk-XABMCBHA.mjs} +12 -6
  58. package/dist/{chunk-OCB7TMUQ.mjs → chunk-Y2U5VVWX.mjs} +1 -1
  59. package/dist/{chunk-MFPJVITN.mjs → chunk-Y4BVTQ53.mjs} +2 -2
  60. package/dist/{chunk-N6J6EJW3.mjs → chunk-YBWSV5DV.mjs} +2 -2
  61. package/dist/{chunk-FUUP45IS.mjs → chunk-ZR4VM4A6.mjs} +1 -1
  62. package/dist/{client-CsuhZ13c.d.ts → client-Cx-TiqPE.d.ts} +19 -3
  63. package/dist/client.d.ts +8 -8
  64. package/dist/client.mjs +10 -2
  65. package/dist/{config-CtTfxbpV.d.ts → config-CzB9YvF_.d.ts} +2 -2
  66. package/dist/{config-DLVnlByZ.d.ts → config-hOWGBeC6.d.ts} +7 -5
  67. package/dist/direct-executor-HQ4QVDX7.mjs +28 -0
  68. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  69. package/dist/distribution/coordinator/etcd.mjs +18 -1
  70. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.mjs +33 -1
  72. package/dist/distribution/transport/grpc.mjs +1 -1
  73. package/dist/distribution.d.ts +71 -43
  74. package/dist/distribution.mjs +411 -127
  75. package/dist/{durability-wiring-UGGUNATA.mjs → durability-wiring-S7OX2MAI.mjs} +28 -26
  76. package/dist/embeddings/openai.mjs +2 -2
  77. package/dist/{engine-Bq5wz6c1.d.ts → engine-B_W9sdZK.d.ts} +38 -8
  78. package/dist/{events-x2aVu0_o.d.ts → events-DErop1LM.d.ts} +23 -2
  79. package/dist/{index-Cl5qQauj.d.ts → index-CKI1fDEl.d.ts} +37 -3
  80. package/dist/index.browser.mjs +5 -3
  81. package/dist/index.d.ts +7 -7
  82. package/dist/index.mjs +21 -19
  83. package/dist/{long-query-thread-5BIPZVS6.mjs → long-query-thread-WVSMIBYP.mjs} +17 -15
  84. package/dist/{module-7OHZ2OPI.mjs → module-2E4XYSGT.mjs} +11 -11
  85. package/dist/partial-words.mjs +7 -6
  86. package/dist/pattern.mjs +4 -3
  87. package/dist/persistence/durability/checkpoint-worker.d.ts +3 -1
  88. package/dist/persistence/durability/checkpoint-worker.mjs +24 -22
  89. package/dist/phrase-module-JERF2IDG.mjs +329 -0
  90. package/dist/react.d.ts +7 -7
  91. package/dist/react.mjs +3 -0
  92. package/dist/{rerank-NYX5S7OG.mjs → rerank-DMMT4FZA.mjs} +6 -6
  93. package/dist/rerankers/http.mjs +2 -2
  94. package/dist/{restore-queue-NVMWJMHI.mjs → restore-queue-JAMNRC6T.mjs} +1 -1
  95. package/dist/{runtime-XNYW6AWX.mjs → runtime-7MLQ7J64.mjs} +7 -7
  96. package/dist/{schema-4W_7tk1f.d.ts → schema-CS-clLun.d.ts} +54 -28
  97. package/dist/server.d.ts +10 -10
  98. package/dist/server.mjs +27 -15
  99. package/dist/{snapshot-P7IQGNNQ.mjs → snapshot-DOQLGGEX.mjs} +45 -30
  100. package/dist/synonyms.mjs +4 -3
  101. package/dist/{tasks-DLQIfXqy.d.ts → tasks-BfF-VRhm.d.ts} +4 -6
  102. package/dist/{thread-QPETQEPK.mjs → thread-DNK5XWXG.mjs} +16 -14
  103. package/dist/vector/search-worker.d.ts +1 -1
  104. package/dist/vector/search-worker.mjs +4 -4
  105. package/dist/workers/entry.mjs +8 -8
  106. package/dist/{write-WZVM7ZL6.mjs → write-ZKR7SNRT.mjs} +23 -21
  107. package/package.json +7 -7
  108. package/dist/direct-executor-6KCVVHYT.mjs +0 -26
@@ -1,6 +1,8 @@
1
- import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-2CBQ2PCC.mjs';
1
+ import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
2
+ import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-AIIJDWMG.mjs';
2
3
  import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
3
- import { ErrorCodes, NarsilError, createNarsilError, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, MIN_DOCUMENTS_FOR_SEGMENTS, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-LJS6PREA.mjs';
4
+ import { ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-VEGHFP2P.mjs';
5
+ import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-UDGB2MWT.mjs';
4
6
  import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
5
7
 
6
8
  // src/analysis/registry.ts
@@ -55,10 +57,26 @@ function getStopWords(name) {
55
57
  }
56
58
  return stopWords2;
57
59
  }
60
+ function holdsPositionsOutOfOrder(entries) {
61
+ for (let index = 1; index < entries.length; index++) {
62
+ if (entries[index].position < entries[index - 1].position) return true;
63
+ }
64
+ return false;
65
+ }
66
+ function inPositionOrder(tokenizer) {
67
+ return {
68
+ tokenize(text) {
69
+ const entries = tokenizer.tokenize(text);
70
+ if (!holdsPositionsOutOfOrder(entries)) return entries;
71
+ return [...entries].sort((first, second) => first.position - second.position);
72
+ }
73
+ };
74
+ }
58
75
  function resolveIndexAnalysis(config) {
76
+ const tokenizer = typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer;
59
77
  return {
60
78
  stopWords: typeof config.stopWords === "string" ? getStopWords(config.stopWords) : config.stopWords,
61
- customTokenizer: typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer
79
+ customTokenizer: tokenizer === void 0 ? void 0 : inPositionOrder(tokenizer)
62
80
  };
63
81
  }
64
82
 
@@ -2232,6 +2250,17 @@ function requireValidPatternWorkCap(cap) {
2232
2250
  );
2233
2251
  }
2234
2252
 
2253
+ // src/core/constants.ts
2254
+ var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
2255
+ var MAX_TERM_FREQUENCY = 65535;
2256
+ var ARRAY_POSITION_GAP = 100;
2257
+ var DEFAULT_BM25_K1 = 1.2;
2258
+ var DEFAULT_BM25_B = 0.75;
2259
+ var MAX_BM25_K1 = 1e3;
2260
+ var MAX_BM25_B = 1;
2261
+ var PENDING_REMOVAL_MERGE_FLOOR = 64;
2262
+ var PENDING_REMOVAL_MERGE_DIVISOR = 256;
2263
+
2235
2264
  // src/core/id-generator.ts
2236
2265
  function generateId() {
2237
2266
  const bytes = new Uint8Array(16);
@@ -3243,12 +3272,6 @@ function createDocumentStore() {
3243
3272
  };
3244
3273
  }
3245
3274
 
3246
- // src/core/constants.ts
3247
- var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
3248
- var MAX_TERM_FREQUENCY = 65535;
3249
- var PENDING_REMOVAL_MERGE_FLOOR = 64;
3250
- var PENDING_REMOVAL_MERGE_DIVISOR = 256;
3251
-
3252
3275
  // src/core/fuzzy.ts
3253
3276
  var FIRST_SURROGATE = 55296;
3254
3277
  var LAST_SURROGATE = 57343;
@@ -3746,6 +3769,13 @@ var LivePostingView = class {
3746
3769
  const starts = this.store.pool.positionStarts;
3747
3770
  return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
3748
3771
  }
3772
+ get positionStore() {
3773
+ return this.holdsPositions ? this.store.positions.values : null;
3774
+ }
3775
+ positionStartAt(row) {
3776
+ const starts = this.store.pool.positionStarts;
3777
+ return starts === null ? 0 : starts[this.syncedStart + row] + 1;
3778
+ }
3749
3779
  copyPositionsAt(row, target, offset) {
3750
3780
  if (!this.holdsPositions) return 0;
3751
3781
  const starts = this.store.pool.positionStarts;
@@ -4468,48 +4498,6 @@ function createSurfaceRegistry() {
4468
4498
  };
4469
4499
  }
4470
4500
 
4471
- // src/core/bitset.ts
4472
- function createBitSet(capacity) {
4473
- return new Uint32Array(capacity + 31 >>> 5 || 1);
4474
- }
4475
- function bitsetSet(bs, index) {
4476
- const word = index >>> 5;
4477
- if (word < bs.length) {
4478
- bs[word] |= 1 << (index & 31);
4479
- }
4480
- }
4481
- function bitsetHas(bs, index) {
4482
- const word = index >>> 5;
4483
- if (word >= bs.length) return false;
4484
- return (bs[word] & 1 << (index & 31)) !== 0;
4485
- }
4486
- function bitsetAndInto(target, other) {
4487
- const shared2 = Math.min(target.length, other.length);
4488
- for (let i = 0; i < shared2; i++) target[i] &= other[i];
4489
- target.fill(0, shared2);
4490
- }
4491
- function bitsetOrInto(target, other) {
4492
- const shared2 = Math.min(target.length, other.length);
4493
- for (let i = 0; i < shared2; i++) target[i] |= other[i];
4494
- }
4495
- function bitsetAndNotInto(target, removed) {
4496
- const shared2 = Math.min(target.length, removed.length);
4497
- for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
4498
- }
4499
- function bitsetIsEmpty(bs) {
4500
- for (let i = 0; i < bs.length; i++) {
4501
- if (bs[i] !== 0) return false;
4502
- }
4503
- return true;
4504
- }
4505
- function bitsetFromSet(set, capacity) {
4506
- const bs = createBitSet(capacity);
4507
- for (const id of set) {
4508
- bs[id >>> 5] |= 1 << (id & 31);
4509
- }
4510
- return bs;
4511
- }
4512
-
4513
4501
  // src/core/partition/facets.ts
4514
4502
  var UNDECLARED_FIELD_TYPE = "";
4515
4503
  function keptFacetValues(limit) {
@@ -5780,7 +5768,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
5780
5768
  if (state.trackPositions) {
5781
5769
  const tokenFreqs = /* @__PURE__ */ new Map();
5782
5770
  let positionOffset = 0;
5783
- let itemTokenCount = 0;
5771
+ let lastItemPosition = -1;
5784
5772
  const addToken = (token, position, surface) => {
5785
5773
  const existing = tokenFreqs.get(token);
5786
5774
  if (existing) {
@@ -5791,12 +5779,12 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
5791
5779
  }
5792
5780
  fieldTokenList.push(token);
5793
5781
  countOccurrence(surfaceCounts, spellings, surface, token);
5794
- itemTokenCount++;
5782
+ lastItemPosition = position;
5795
5783
  };
5796
5784
  for (const item of arr) {
5797
- itemTokenCount = 0;
5785
+ lastItemPosition = -1;
5798
5786
  visitTokens(item, language, opts, addToken);
5799
- positionOffset += itemTokenCount;
5787
+ if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
5800
5788
  }
5801
5789
  fieldLengths[fieldPath] = fieldTokenList.length;
5802
5790
  for (const [token, freq] of tokenFreqs) {
@@ -6793,18 +6781,6 @@ function evaluateFilters(expression, context) {
6793
6781
  return evaluateExpression(validatedFilterOf(expression, context.fieldTypes), context);
6794
6782
  }
6795
6783
 
6796
- // src/core/partition/constants.ts
6797
- var TARGET_BLOCK_ENTRIES = 128;
6798
- var PRUNING_REJECTION_SLACK = 1e-12;
6799
- var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
6800
- var SEGMENT_TOKEN_TABLE_START = 1 << 12;
6801
- var SEGMENT_POSTING_LOG_START = 1 << 14;
6802
- var SEGMENT_POSITION_POOL_START = 1 << 16;
6803
- var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
6804
- var LOOKUP_KEY_SCRATCH_BYTES = 4096;
6805
- var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
6806
- var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
6807
-
6808
6784
  // src/core/partition/sort-columns/order.ts
6809
6785
  var MISSING_RANK = -1;
6810
6786
  var BYTES_PER_RANK = 4;
@@ -7545,20 +7521,115 @@ function partitionFilterMatches(state, filters, schema, meter) {
7545
7521
  };
7546
7522
  }
7547
7523
 
7524
+ // src/search/quotes.ts
7525
+ var QUOTE_CHARACTER = /["“”„]/;
7526
+ function holdsQuoteCharacter(term) {
7527
+ return QUOTE_CHARACTER.test(term);
7528
+ }
7529
+
7530
+ // src/search/phrase-code.ts
7531
+ var loaded = null;
7532
+ var loading = null;
7533
+ function requireLoadedPhraseCode() {
7534
+ if (loaded === null) {
7535
+ throw new NarsilError(ErrorCodes.SEARCH_PHRASE_NOT_READY, "The phrase code has not loaded. Try the query again");
7536
+ }
7537
+ return loaded;
7538
+ }
7539
+ function loadPhraseCode() {
7540
+ if (loading === null) {
7541
+ loading = import('./phrase-module-WESYVU7K.mjs').then(
7542
+ (code) => {
7543
+ loaded = code;
7544
+ return code;
7545
+ },
7546
+ (err) => {
7547
+ loading = null;
7548
+ throw new NarsilError(
7549
+ ErrorCodes.SEARCH_PHRASE_NOT_READY,
7550
+ "The phrase code failed to load. Remove the quotes, or try again once it can load",
7551
+ { cause: err instanceof Error ? err.message : String(err) }
7552
+ );
7553
+ }
7554
+ );
7555
+ }
7556
+ return loading;
7557
+ }
7558
+ function loadPhraseCodeFor(term) {
7559
+ if (loaded !== null || term === void 0 || !holdsQuoteCharacter(term)) return null;
7560
+ return loadPhraseCode();
7561
+ }
7562
+ function analyseQueryPhrases(term, language, analysis) {
7563
+ if (!holdsQuoteCharacter(term)) return null;
7564
+ return requireLoadedPhraseCode().analyseQueryPhrases(
7565
+ term.split(QUOTE_CHARACTER),
7566
+ (text) => tokenize(text, language, {
7567
+ stem: true,
7568
+ removeStopWords: true,
7569
+ customTokenizer: analysis.customTokenizer,
7570
+ stopWordOverride: analysis.stopWords
7571
+ }).tokens
7572
+ );
7573
+ }
7574
+
7548
7575
  // src/core/scorer.ts
7549
- var DEFAULT_K1 = 1.2;
7550
- var DEFAULT_B = 0.75;
7551
7576
  function computeIDF(docFrequency, totalDocs) {
7552
7577
  if (totalDocs === 0) return 0;
7553
7578
  return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
7554
7579
  }
7580
+ function pairKeepsBoundSound(k1, b) {
7581
+ return k1 >= 0 && k1 <= MAX_BM25_K1 && b >= 0 && b <= MAX_BM25_B;
7582
+ }
7555
7583
  function bm25PruningSound(params) {
7556
- const k1 = params?.k1 ?? DEFAULT_K1;
7557
- const b = params?.b ?? DEFAULT_B;
7558
- return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
7584
+ const k1 = params?.k1 ?? DEFAULT_BM25_K1;
7585
+ const b = params?.b ?? DEFAULT_BM25_B;
7586
+ if (!pairKeepsBoundSound(k1, b)) return false;
7587
+ const fields = params?.fields;
7588
+ if (fields === void 0) return true;
7589
+ for (const name of Object.keys(fields)) {
7590
+ if (!pairKeepsBoundSound(fields[name].k1 ?? k1, fields[name].b ?? b)) return false;
7591
+ }
7592
+ return true;
7593
+ }
7594
+ function bm25SetsValue(params) {
7595
+ if (typeof params !== "object" || params === null) return params !== void 0;
7596
+ if (params.k1 !== void 0 || params.b !== void 0) return true;
7597
+ const fields = params.fields;
7598
+ if (fields === void 0) return false;
7599
+ if (typeof fields !== "object" || fields === null) return true;
7600
+ for (const name of Object.keys(fields)) {
7601
+ const pair = fields[name];
7602
+ if (typeof pair !== "object" || pair === null) return true;
7603
+ if (pair.k1 !== void 0 || pair.b !== void 0) return true;
7604
+ }
7605
+ return false;
7559
7606
  }
7560
- function resolveBM25Params(params) {
7561
- return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
7607
+ function ownField(fields, name) {
7608
+ return fields !== void 0 && Object.hasOwn(fields, name) ? fields[name] : void 0;
7609
+ }
7610
+ function mergeBM25Levels(search, index) {
7611
+ if (search === void 0 || !bm25SetsValue(search)) return index;
7612
+ const k1 = search.k1 ?? index?.k1;
7613
+ const b = search.b ?? index?.b;
7614
+ const merged = {
7615
+ ...k1 !== void 0 ? { k1 } : {},
7616
+ ...b !== void 0 ? { b } : {}
7617
+ };
7618
+ const names = [...Object.keys(search.fields ?? {}), ...Object.keys(index?.fields ?? {})];
7619
+ if (names.length === 0) return merged;
7620
+ const fields = /* @__PURE__ */ Object.create(null);
7621
+ for (const name of names) {
7622
+ const fromSearch = ownField(search.fields, name);
7623
+ const fromIndex = ownField(index?.fields, name);
7624
+ const fieldK1 = fromSearch?.k1 ?? search.k1 ?? fromIndex?.k1;
7625
+ const fieldB = fromSearch?.b ?? search.b ?? fromIndex?.b;
7626
+ fields[name] = {
7627
+ ...fieldK1 !== void 0 ? { k1: fieldK1 } : {},
7628
+ ...fieldB !== void 0 ? { b: fieldB } : {}
7629
+ };
7630
+ }
7631
+ merged.fields = fields;
7632
+ return merged;
7562
7633
  }
7563
7634
  function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
7564
7635
  if (avgFieldLength === 0) return 0;
@@ -7567,29 +7638,49 @@ function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1,
7567
7638
  if (denominator === 0) return 0;
7568
7639
  return idf * (numerator / denominator);
7569
7640
  }
7570
- function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
7571
- if (totalDocs === 0) return 0;
7572
- const k1 = params?.k1 ?? DEFAULT_K1;
7573
- const b = params?.b ?? DEFAULT_B;
7574
- return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
7575
- }
7576
7641
 
7577
7642
  // src/core/partition/field-scoring.ts
7578
- function loadFieldScoring(state, fields, boost, averageFieldLengths) {
7643
+ var NO_FIELD_VALUES = new Float64Array(0);
7644
+ function differentPairsOf(searchable, k1s, bs) {
7645
+ const pairs = [];
7646
+ for (let fieldIndex = 0; fieldIndex < searchable.length; fieldIndex++) {
7647
+ if (searchable[fieldIndex] !== 1) continue;
7648
+ let seen = false;
7649
+ for (let at = 0; at < pairs.length && !seen; at += 2) {
7650
+ seen = pairs[at] === k1s[fieldIndex] && pairs[at + 1] === bs[fieldIndex];
7651
+ }
7652
+ if (!seen) pairs.push(k1s[fieldIndex], bs[fieldIndex]);
7653
+ }
7654
+ return pairs;
7655
+ }
7656
+ function loadFieldScoring(state, fields, boost, averageFieldLengths, bm25) {
7579
7657
  const fieldNames = state.fieldNameTable.names;
7658
+ const fieldCount = fieldNames.length;
7659
+ const k1 = bm25?.k1 ?? DEFAULT_BM25_K1;
7660
+ const b = bm25?.b ?? DEFAULT_BM25_B;
7661
+ const perField = bm25?.fields;
7580
7662
  const scoring = {
7581
- searchable: new Uint8Array(fieldNames.length),
7582
- boosts: new Float64Array(fieldNames.length),
7583
- averageLengths: new Float64Array(fieldNames.length),
7584
- lengthColumns: new Array(fieldNames.length).fill(null)
7663
+ searchable: new Uint8Array(fieldCount),
7664
+ boosts: new Float64Array(fieldCount),
7665
+ averageLengths: new Float64Array(fieldCount),
7666
+ lengthColumns: new Array(fieldCount).fill(null),
7667
+ k1s: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
7668
+ bs: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
7669
+ boundPairs: [k1, b]
7585
7670
  };
7586
- for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
7671
+ for (let fieldIndex = 0; fieldIndex < fieldCount; fieldIndex++) {
7587
7672
  const fieldName = fieldNames[fieldIndex];
7588
7673
  scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
7589
7674
  scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
7590
7675
  scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
7591
7676
  scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
7677
+ if (perField !== void 0) {
7678
+ const fieldPair = Object.hasOwn(perField, fieldName) ? perField[fieldName] : void 0;
7679
+ scoring.k1s[fieldIndex] = fieldPair?.k1 ?? k1;
7680
+ scoring.bs[fieldIndex] = fieldPair?.b ?? b;
7681
+ }
7592
7682
  }
7683
+ if (perField !== void 0) scoring.boundPairs = differentPairsOf(scoring.searchable, scoring.k1s, scoring.bs);
7593
7684
  return scoring;
7594
7685
  }
7595
7686
  function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
@@ -7605,6 +7696,15 @@ function bestSearchable(searchable, values) {
7605
7696
  }
7606
7697
  return best;
7607
7698
  }
7699
+ function blockEntryBound(fields, maxTermFrequency, idf, minFieldLength, maxAverageLength) {
7700
+ const pairs = fields.boundPairs;
7701
+ let best = 0;
7702
+ for (let at = 0; at < pairs.length; at += 2) {
7703
+ const bound = computeBM25WithIDF(maxTermFrequency, idf, minFieldLength, maxAverageLength, pairs[at], pairs[at + 1]);
7704
+ if (bound > best) best = bound;
7705
+ }
7706
+ return best;
7707
+ }
7608
7708
 
7609
7709
  // src/core/partition/posting-columns.ts
7610
7710
  function postingColumns(list) {
@@ -7698,8 +7798,9 @@ function groupPostingCount(matches) {
7698
7798
  return total;
7699
7799
  }
7700
7800
  function computeGroupContributions(context, matches, collect) {
7701
- const { fields, fieldNames, filterBitset, partialMarks, k1, b } = context;
7801
+ const { fields, fieldNames, filterBitset, partialMarks } = context;
7702
7802
  const scoresAreZero = context.totalDocs === 0;
7803
+ const onePair = fields.boundPairs.length === 2;
7703
7804
  const best = /* @__PURE__ */ new Map();
7704
7805
  for (const match of matches) {
7705
7806
  const perTerm = /* @__PURE__ */ new Map();
@@ -7719,7 +7820,14 @@ function computeGroupContributions(context, matches, collect) {
7719
7820
  const fieldBoost = fields.boosts[fieldIndex];
7720
7821
  const avgLen = fields.averageLengths[fieldIndex];
7721
7822
  const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
7722
- const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * factor;
7823
+ const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
7824
+ termFrequency,
7825
+ match.idf,
7826
+ actualFieldLength,
7827
+ avgLen,
7828
+ onePair ? fields.boundPairs[0] : fields.k1s[fieldIndex],
7829
+ onePair ? fields.boundPairs[1] : fields.bs[fieldIndex]
7830
+ )) * fieldBoost * factor;
7723
7831
  const existing = perTerm.get(internalId);
7724
7832
  if (existing) {
7725
7833
  existing.score += termScore;
@@ -7776,11 +7884,20 @@ function searchedFieldMarks(fieldNames, fields) {
7776
7884
  function searchFulltextMatches(state, params) {
7777
7885
  const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
7778
7886
  const fieldNames = state.fieldNameTable.names;
7779
- const matched = createBitSet(state.docStore.internalIdCapacity());
7780
7887
  const searched = searchedFieldMarks(fieldNames, fields);
7781
7888
  const partial = partialFieldMarks(fieldNames, params.partialFields);
7782
7889
  const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
7783
- for (const qt of queryTokens) {
7890
+ const capacity = state.docStore.internalIdCapacity();
7891
+ const matched = params.phrases === void 0 ? createBitSet(capacity) : requireLoadedPhraseCode().matchPhrases(
7892
+ state.invertedIdx,
7893
+ params.phrases,
7894
+ searched,
7895
+ filterBitset,
7896
+ capacity,
7897
+ params.phraseWork,
7898
+ false
7899
+ ).matched;
7900
+ for (const qt of params.phrases === void 0 ? queryTokens : []) {
7784
7901
  const group = termGroupOf(termGroups, qt.token);
7785
7902
  for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
7786
7903
  markPostingList(matched, match.postingList, searched, filterBitset);
@@ -8283,7 +8400,7 @@ function boostsKeepScoresNonNegative(fields) {
8283
8400
  }
8284
8401
  return true;
8285
8402
  }
8286
- function termUpperBound(scan2, list, maxBoost, maxAverageLength) {
8403
+ function samePairUpperBound(scan2, list, maxBoost, maxAverageLength) {
8287
8404
  if (scan2.scoresAreZero) return 0;
8288
8405
  const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
8289
8406
  let best = 0;
@@ -8301,7 +8418,24 @@ function termUpperBound(scan2, list, maxBoost, maxAverageLength) {
8301
8418
  }
8302
8419
  return best;
8303
8420
  }
8304
- function entryScore(scan2, entry, internalId) {
8421
+ function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
8422
+ if (scan2.scoresAreZero) return 0;
8423
+ const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
8424
+ let best = 0;
8425
+ for (let block = 0; block < bounds.blockCount; block++) {
8426
+ const bestEntry = blockEntryBound(
8427
+ scan2.fields,
8428
+ bounds.maxTermFrequency[block],
8429
+ scan2.idf,
8430
+ bounds.minFieldLength[block],
8431
+ maxAverageLength
8432
+ );
8433
+ const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
8434
+ if (blockBound > best) best = blockBound;
8435
+ }
8436
+ return best;
8437
+ }
8438
+ function samePairEntryScore(scan2, entry, internalId) {
8305
8439
  const { fields, columns } = scan2;
8306
8440
  const fieldIndex = columns.fieldNameIndices[entry];
8307
8441
  const averageLength = fields.averageLengths[fieldIndex];
@@ -8309,17 +8443,55 @@ function entryScore(scan2, entry, internalId) {
8309
8443
  const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan2.idf, fieldLength, averageLength, scan2.k1, scan2.b);
8310
8444
  return bm25 * fields.boosts[fieldIndex];
8311
8445
  }
8312
- function scoreEveryDocument(scan2, buffer) {
8446
+ function fieldPairEntryScore(scan2, entry, internalId) {
8447
+ const { fields, columns } = scan2;
8448
+ const fieldIndex = columns.fieldNameIndices[entry];
8449
+ const averageLength = fields.averageLengths[fieldIndex];
8450
+ const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
8451
+ const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(
8452
+ columns.termFrequencies[entry],
8453
+ scan2.idf,
8454
+ fieldLength,
8455
+ averageLength,
8456
+ fields.k1s[fieldIndex],
8457
+ fields.bs[fieldIndex]
8458
+ );
8459
+ return bm25 * fields.boosts[fieldIndex];
8460
+ }
8461
+ function samePairScoreEveryDocument(scan2, buffer) {
8462
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8463
+ const searchable = scan2.fields.searchable;
8464
+ for (let entry = 0; entry < count; entry++) {
8465
+ const internalId = docIds[entry];
8466
+ if (hasDeleted && deletedDocs.has(internalId)) continue;
8467
+ if (searchable[fieldNameIndices[entry]] !== 1) continue;
8468
+ addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
8469
+ }
8470
+ }
8471
+ function fieldPairScoreEveryDocument(scan2, buffer) {
8472
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8473
+ const searchable = scan2.fields.searchable;
8474
+ for (let entry = 0; entry < count; entry++) {
8475
+ const internalId = docIds[entry];
8476
+ if (hasDeleted && deletedDocs.has(internalId)) continue;
8477
+ if (searchable[fieldNameIndices[entry]] !== 1) continue;
8478
+ addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
8479
+ }
8480
+ }
8481
+ function samePairScoreKnownDocuments(scan2, buffer) {
8313
8482
  const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8314
8483
  const searchable = scan2.fields.searchable;
8484
+ let counted = 0;
8315
8485
  for (let entry = 0; entry < count; entry++) {
8316
8486
  const internalId = docIds[entry];
8317
8487
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8318
8488
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8319
- addScore(buffer, internalId, entryScore(scan2, entry, internalId));
8489
+ if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
8490
+ else if (markCounted(buffer, internalId)) counted++;
8320
8491
  }
8492
+ return counted;
8321
8493
  }
8322
- function scoreKnownDocumentsAndCountTheRest(scan2, buffer) {
8494
+ function fieldPairScoreKnownDocuments(scan2, buffer) {
8323
8495
  const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8324
8496
  const searchable = scan2.fields.searchable;
8325
8497
  let counted = 0;
@@ -8327,31 +8499,58 @@ function scoreKnownDocumentsAndCountTheRest(scan2, buffer) {
8327
8499
  const internalId = docIds[entry];
8328
8500
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8329
8501
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8330
- if (hasScore(buffer, internalId)) addScore(buffer, internalId, entryScore(scan2, entry, internalId));
8502
+ if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
8331
8503
  else if (markCounted(buffer, internalId)) counted++;
8332
8504
  }
8333
8505
  return counted;
8334
8506
  }
8335
- function queryOrderScore(scans, internalId) {
8507
+ function firstEntryOf(docIds, count, internalId) {
8508
+ let low = 0;
8509
+ let high = count;
8510
+ while (low < high) {
8511
+ const middle = low + high >>> 1;
8512
+ if (docIds[middle] < internalId) low = middle + 1;
8513
+ else high = middle;
8514
+ }
8515
+ return low;
8516
+ }
8517
+ function samePairQueryOrderScore(scans, internalId) {
8336
8518
  let score = 0;
8337
8519
  for (const scan2 of scans) {
8338
8520
  const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
8339
8521
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8340
- let low = 0;
8341
- let high = count;
8342
- while (low < high) {
8343
- const middle = low + high >>> 1;
8344
- if (docIds[middle] < internalId) low = middle + 1;
8345
- else high = middle;
8522
+ for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
8523
+ if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
8524
+ score += samePairEntryScore(scan2, entry, internalId);
8346
8525
  }
8347
- for (let entry = low; entry < count && docIds[entry] === internalId; entry++) {
8526
+ }
8527
+ return score;
8528
+ }
8529
+ function fieldPairQueryOrderScore(scans, internalId) {
8530
+ let score = 0;
8531
+ for (const scan2 of scans) {
8532
+ const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
8533
+ if (hasDeleted && deletedDocs.has(internalId)) continue;
8534
+ for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
8348
8535
  if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
8349
- score += entryScore(scan2, entry, internalId);
8536
+ score += fieldPairEntryScore(scan2, entry, internalId);
8350
8537
  }
8351
8538
  }
8352
8539
  return score;
8353
8540
  }
8354
- function pageFromBuffer(scans, buffer, selection, resolver) {
8541
+ var SAME_PAIR_SCORER = {
8542
+ upperBound: samePairUpperBound,
8543
+ scoreEveryDocument: samePairScoreEveryDocument,
8544
+ scoreKnownDocumentsAndCountTheRest: samePairScoreKnownDocuments,
8545
+ queryOrderScore: samePairQueryOrderScore
8546
+ };
8547
+ var FIELD_PAIR_SCORER = {
8548
+ upperBound: fieldPairUpperBound,
8549
+ scoreEveryDocument: fieldPairScoreEveryDocument,
8550
+ scoreKnownDocumentsAndCountTheRest: fieldPairScoreKnownDocuments,
8551
+ queryOrderScore: fieldPairQueryOrderScore
8552
+ };
8553
+ function pageFromBuffer(scorer, scans, buffer, selection, resolver) {
8355
8554
  const approximateKth = kthBestScore(buffer, selection, resolver);
8356
8555
  const gatherFrom = approximateKth - Math.abs(approximateKth) * PRUNING_REJECTION_SLACK;
8357
8556
  const heap = [];
@@ -8363,7 +8562,7 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
8363
8562
  if (scores[internalId] < gatherFrom) continue;
8364
8563
  const externalId = resolver.toExternal(internalId);
8365
8564
  if (externalId === void 0) continue;
8366
- const score = queryOrderScore(scans, internalId);
8565
+ const score = scorer.queryOrderScore(scans, internalId);
8367
8566
  if (full && score < threshold) continue;
8368
8567
  const candidate = { internalId, externalId, score };
8369
8568
  if (!full) {
@@ -8393,18 +8592,28 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
8393
8592
  return scored;
8394
8593
  }
8395
8594
  function multiTermTopK(request) {
8396
- const { terms, docFrequencies, totalDocs, bm25Params, limit, fields, resolver, buffer } = request;
8595
+ const { terms, docFrequencies, totalDocs, limit, fields, resolver, buffer } = request;
8397
8596
  if (!boostsKeepScoresNonNegative(fields)) return null;
8398
- const { k1, b } = resolveBM25Params(bm25Params);
8597
+ const onePair = fields.boundPairs.length === 2;
8598
+ const scorer = onePair ? SAME_PAIR_SCORER : FIELD_PAIR_SCORER;
8599
+ const k1 = onePair ? fields.boundPairs[0] : 0;
8600
+ const b = onePair ? fields.boundPairs[1] : 0;
8399
8601
  const scans = new Array(terms.length);
8400
8602
  for (let term = 0; term < terms.length; term++) {
8401
8603
  const idf = computeIDF(docFrequencies[term], totalDocs);
8402
8604
  if (!(idf >= 0)) return null;
8403
- scans[term] = { columns: postingColumns(terms[term].list), idf, k1, b, scoresAreZero: totalDocs === 0, fields };
8605
+ scans[term] = {
8606
+ columns: postingColumns(terms[term].list),
8607
+ idf,
8608
+ scoresAreZero: totalDocs === 0,
8609
+ fields,
8610
+ k1,
8611
+ b
8612
+ };
8404
8613
  }
8405
8614
  const maxBoost = bestSearchable(fields.searchable, fields.boosts);
8406
8615
  const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
8407
- const upperBounds = scans.map((scan2, term) => termUpperBound(scan2, terms[term].list, maxBoost, maxAverageLength));
8616
+ const upperBounds = scans.map((scan2, term) => scorer.upperBound(scan2, terms[term].list, maxBoost, maxAverageLength));
8408
8617
  const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
8409
8618
  const remainingBound = new Float64Array(order.length + 1);
8410
8619
  for (let position2 = order.length - 1; position2 >= 0; position2--) {
@@ -8419,15 +8628,15 @@ function multiTermTopK(request) {
8419
8628
  const threshold = kthBestScore(buffer, selection, resolver);
8420
8629
  if (remaining < threshold - threshold * PRUNING_REJECTION_SLACK) break;
8421
8630
  }
8422
- scoreEveryDocument(scans[order[position]], buffer);
8631
+ scorer.scoreEveryDocument(scans[order[position]], buffer);
8423
8632
  }
8424
8633
  let countedOnly = 0;
8425
8634
  for (let rest = position; rest < order.length; rest++) {
8426
- countedOnly += scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
8635
+ countedOnly += scorer.scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
8427
8636
  }
8428
8637
  const totalMatched = buffer.touchedCount + countedOnly;
8429
8638
  if (wanted === 0) return { scored: [], totalMatched };
8430
- return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
8639
+ return { scored: pageFromBuffer(scorer, scans, buffer, selection, resolver), totalMatched };
8431
8640
  }
8432
8641
 
8433
8642
  // src/core/partition/single-term-topk.ts
@@ -8455,20 +8664,25 @@ function prunableSingleTermList(params, index, fieldNames) {
8455
8664
  return list;
8456
8665
  }
8457
8666
  function singleTermTopK(request) {
8458
- const { list, docFrequency, totalDocs, bm25Params, limit, resolver } = request;
8667
+ const { list, docFrequency, totalDocs, limit, resolver } = request;
8459
8668
  const {
8460
8669
  searchable: fieldSearchable,
8461
8670
  boosts: fieldBoosts,
8462
8671
  averageLengths: fieldAvgLengths,
8463
- lengthColumns: fieldLengthColumns
8672
+ lengthColumns: fieldLengthColumns,
8673
+ k1s,
8674
+ bs,
8675
+ boundPairs
8464
8676
  } = request.fields;
8677
+ const onePair = boundPairs.length === 2;
8678
+ const onlyK1 = boundPairs[0];
8679
+ const onlyB = boundPairs[1];
8465
8680
  const wanted = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
8466
8681
  const { docIds, termFrequencies, fieldNameIndices, deletedDocs: deleted, hasDeleted } = postingColumns(list);
8467
8682
  const bounds = blockBoundsFor(list, fieldLengthColumns);
8468
8683
  const maxBoost = bestSearchable(fieldSearchable, fieldBoosts);
8469
8684
  const maxAverageLength = bestSearchable(fieldSearchable, fieldAvgLengths);
8470
8685
  const idf = computeIDF(docFrequency, totalDocs);
8471
- const { k1, b } = resolveBM25Params(bm25Params);
8472
8686
  const scoresAreZero = totalDocs === 0;
8473
8687
  const heap = [];
8474
8688
  let full = false;
@@ -8478,14 +8692,12 @@ function singleTermTopK(request) {
8478
8692
  const start = block === 0 ? 0 : bounds.entryEnd[block - 1];
8479
8693
  const end = bounds.entryEnd[block];
8480
8694
  if (full) {
8481
- const bestEntry = computeBM25(
8482
- bounds.maxTermFrequency[block],
8483
- docFrequency,
8484
- totalDocs,
8485
- bounds.minFieldLength[block],
8486
- maxAverageLength,
8487
- bm25Params
8488
- );
8695
+ let bestEntry = 0;
8696
+ if (!scoresAreZero) {
8697
+ const blockMaxTf = bounds.maxTermFrequency[block];
8698
+ const blockMinLength = bounds.minFieldLength[block];
8699
+ bestEntry = onePair ? computeBM25WithIDF(blockMaxTf, idf, blockMinLength, maxAverageLength, onlyK1, onlyB) : blockEntryBound(request.fields, blockMaxTf, idf, blockMinLength, maxAverageLength);
8700
+ }
8489
8701
  if (bestEntry * maxBoost * bounds.maxEntriesPerDocument[block] < threshold) {
8490
8702
  totalMatched += bounds.documentCount[block];
8491
8703
  continue;
@@ -8504,7 +8716,14 @@ function singleTermTopK(request) {
8504
8716
  const fieldIndex = fieldNameIndices[entry];
8505
8717
  if (fieldSearchable[fieldIndex] === 1) {
8506
8718
  const fieldLength = fieldLengthOf(fieldLengthColumns, fieldIndex, internalId, fieldAvgLengths[fieldIndex]);
8507
- score += (scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[entry], idf, fieldLength, fieldAvgLengths[fieldIndex], k1, b)) * fieldBoosts[fieldIndex];
8719
+ score += (scoresAreZero ? 0 : computeBM25WithIDF(
8720
+ termFrequencies[entry],
8721
+ idf,
8722
+ fieldLength,
8723
+ fieldAvgLengths[fieldIndex],
8724
+ onePair ? onlyK1 : k1s[fieldIndex],
8725
+ onePair ? onlyB : bs[fieldIndex]
8726
+ )) * fieldBoosts[fieldIndex];
8508
8727
  scored2 = true;
8509
8728
  }
8510
8729
  entry++;
@@ -8564,15 +8783,14 @@ function searchFulltext(state, params) {
8564
8783
  globalStats,
8565
8784
  maxResults,
8566
8785
  termMatch,
8567
- filterBitset
8786
+ phrases
8568
8787
  } = params;
8569
8788
  const collectComponents = params.collectComponents !== false;
8570
- if (queryTokens.length === 0) {
8789
+ if (queryTokens.length === 0 && phrases === void 0) {
8571
8790
  return { scored: [], totalMatched: 0 };
8572
8791
  }
8573
8792
  const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
8574
8793
  const avgFieldLengths = globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths;
8575
- const { k1, b } = resolveBM25Params(bm25Params);
8576
8794
  const scoresAreZero = totalDocs === 0;
8577
8795
  if (state.scoreBuffer === null) state.scoreBuffer = createScoreBuffer(state.docStore.internalIdCapacity());
8578
8796
  const scoreBuffer = state.scoreBuffer;
@@ -8581,19 +8799,59 @@ function searchFulltext(state, params) {
8581
8799
  const useIntersection = termMatch === "all" && queryTokens.length > 1;
8582
8800
  const fieldNames = state.fieldNameTable.names;
8583
8801
  const resolver = state.docStore.resolver();
8584
- const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths);
8585
- const { searchable, boosts, averageLengths, lengthColumns } = fieldScoring;
8802
+ const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths, bm25Params);
8803
+ const { searchable, boosts, averageLengths, lengthColumns, k1s, bs, boundPairs } = fieldScoring;
8804
+ const onePair = boundPairs.length === 2;
8805
+ const onlyK1 = boundPairs[0];
8806
+ const onlyB = boundPairs[1];
8807
+ const phraseMatches = phrases === void 0 ? null : requireLoadedPhraseCode().matchPhrases(
8808
+ state.invertedIdx,
8809
+ phrases,
8810
+ searchable,
8811
+ params.filterBitset,
8812
+ state.docStore.internalIdCapacity(),
8813
+ params.phraseWork,
8814
+ true
8815
+ );
8816
+ if (phraseMatches !== null && bitsetIsEmpty(phraseMatches.matched)) {
8817
+ return { scored: [], totalMatched: 0 };
8818
+ }
8819
+ const filterBitset = phraseMatches?.matched ?? params.filterBitset;
8586
8820
  function documentFrequency(token, list) {
8587
8821
  return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
8588
8822
  }
8823
+ function scorePhraseRows(rows, matched) {
8824
+ let idf = 0;
8825
+ for (let entry = 0; entry < rows.phrase.tokens.length; entry++) {
8826
+ idf += computeIDF(documentFrequency(rows.phrase.tokens[entry], rows.lists[entry]), totalDocs);
8827
+ }
8828
+ for (let row = 0; row < rows.docIds.length; row++) {
8829
+ const internalId = rows.docIds.values[row];
8830
+ if (!bitsetHas(matched, internalId)) continue;
8831
+ const fieldIndex = rows.fieldIndices.values[row];
8832
+ const frequency = rows.frequencies.values[row];
8833
+ const avgLen = averageLengths[fieldIndex];
8834
+ const fieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
8835
+ const score = scoresAreZero ? 0 : computeBM25WithIDF(
8836
+ frequency,
8837
+ idf,
8838
+ fieldLength,
8839
+ avgLen,
8840
+ onePair ? onlyK1 : k1s[fieldIndex],
8841
+ onePair ? onlyB : bs[fieldIndex]
8842
+ );
8843
+ addScore(scoreBuffer, internalId, score * boosts[fieldIndex]);
8844
+ if (components !== null) {
8845
+ recordComponents(components, internalId, fieldNames[fieldIndex], rows.phrase.key, frequency, fieldLength, idf);
8846
+ }
8847
+ }
8848
+ }
8589
8849
  const groupContext = {
8590
8850
  index: state.invertedIdx,
8591
8851
  fields: fieldScoring,
8592
8852
  partialMarks: partialFieldMarks(fieldNames, partialFields),
8593
8853
  fieldNames,
8594
8854
  totalDocs,
8595
- k1,
8596
- b,
8597
8855
  filterBitset,
8598
8856
  globalFrequency(token) {
8599
8857
  if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
@@ -8619,7 +8877,14 @@ function searchFulltext(state, params) {
8619
8877
  if (searchable[fieldIndex] === 0) continue;
8620
8878
  const avgLen = averageLengths[fieldIndex];
8621
8879
  const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
8622
- const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], match.idf, actualFieldLength, avgLen, k1, b);
8880
+ const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
8881
+ termFrequencies[pi],
8882
+ match.idf,
8883
+ actualFieldLength,
8884
+ avgLen,
8885
+ onePair ? onlyK1 : k1s[fieldIndex],
8886
+ onePair ? onlyB : bs[fieldIndex]
8887
+ );
8623
8888
  addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
8624
8889
  }
8625
8890
  return;
@@ -8634,7 +8899,14 @@ function searchFulltext(state, params) {
8634
8899
  const termFrequency = termFrequencies[pi];
8635
8900
  const avgLen = averageLengths[fieldIndex];
8636
8901
  const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
8637
- const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
8902
+ const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
8903
+ termFrequency,
8904
+ match.idf,
8905
+ actualFieldLength,
8906
+ avgLen,
8907
+ onePair ? onlyK1 : k1s[fieldIndex],
8908
+ onePair ? onlyB : bs[fieldIndex]
8909
+ );
8638
8910
  addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
8639
8911
  if (components !== null) {
8640
8912
  recordComponents(
@@ -8656,25 +8928,23 @@ function searchFulltext(state, params) {
8656
8928
  mergeGroupContribution(internalId, contribution);
8657
8929
  }
8658
8930
  }
8659
- const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
8931
+ const prunableList = phrases === void 0 ? prunableSingleTermList(params, state.invertedIdx, fieldNames) : null;
8660
8932
  if (prunableList !== null && maxResults !== void 0) {
8661
8933
  return singleTermTopK({
8662
8934
  list: prunableList,
8663
8935
  docFrequency: documentFrequency(queryTokens[0].token, prunableList),
8664
8936
  totalDocs,
8665
- bm25Params,
8666
8937
  limit: maxResults,
8667
8938
  fields: fieldScoring,
8668
8939
  resolver
8669
8940
  });
8670
8941
  }
8671
- const prunableLists = prunableMultiTermLists(params, state.invertedIdx);
8942
+ const prunableLists = phrases === void 0 ? prunableMultiTermLists(params, state.invertedIdx) : null;
8672
8943
  if (prunableLists !== null && maxResults !== void 0) {
8673
8944
  const pruned = multiTermTopK({
8674
8945
  terms: prunableLists,
8675
8946
  docFrequencies: prunableLists.map((term) => documentFrequency(term.token, term.list)),
8676
8947
  totalDocs,
8677
- bm25Params,
8678
8948
  limit: maxResults,
8679
8949
  fields: fieldScoring,
8680
8950
  resolver,
@@ -8695,7 +8965,7 @@ function searchFulltext(state, params) {
8695
8965
  }
8696
8966
  resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
8697
8967
  }
8698
- resolved.sort((a, b2) => a.totalPostings - b2.totalPostings);
8968
+ resolved.sort((a, b) => a.totalPostings - b.totalPostings);
8699
8969
  for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
8700
8970
  const onlyScored = tokenIndex > 0;
8701
8971
  for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
@@ -8710,6 +8980,9 @@ function searchFulltext(state, params) {
8710
8980
  if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
8711
8981
  }
8712
8982
  }
8983
+ if (phraseMatches !== null) {
8984
+ for (const rows of phraseMatches.rows) scorePhraseRows(rows, phraseMatches.matched);
8985
+ }
8713
8986
  const totalMatched = scoreBuffer.touchedCount;
8714
8987
  const k = maxResults === void 0 ? totalMatched : Math.max(0, Math.min(maxResults, totalMatched));
8715
8988
  const scored = topKFromBuffer(scoreBuffer, k, resolver, components);
@@ -10367,6 +10640,9 @@ function heapPop(heap) {
10367
10640
  function collectCandidateTerms(subs, params) {
10368
10641
  const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
10369
10642
  const terms = /* @__PURE__ */ new Set();
10643
+ for (const phrase of params.phrases ?? []) {
10644
+ for (const token of phrase.tokens) terms.add(token);
10645
+ }
10370
10646
  for (const qt of queryTokens) {
10371
10647
  const group = termGroupOf(termGroups, qt.token);
10372
10648
  if (group !== void 0) {
@@ -10468,7 +10744,7 @@ function searchSingleSub(subs, layout, params, index) {
10468
10744
  return { ...result, matchedOrdinalBitset };
10469
10745
  }
10470
10746
  function compositeSearchFulltext(subs, layout, params) {
10471
- if (params.queryTokens.length === 0) {
10747
+ if (params.queryTokens.length === 0 && params.phrases === void 0) {
10472
10748
  return { scored: [], totalMatched: 0 };
10473
10749
  }
10474
10750
  const single = onlyPopulatedSub(subs);
@@ -11357,7 +11633,7 @@ var PREPARED_TYPO_TEST_BYTES = Math.ceil(MAX_FILTER_TESTS / TYPO_CLAIM_SLOTS_PER
11357
11633
  function capExceeded(cap) {
11358
11634
  return new NarsilError(
11359
11635
  ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
11360
- `The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
11636
+ `The engine stopped this search before it checked every candidate, because the work on its pattern tests and phrases passed the cap of ${cap} units. Narrow the tests or the phrases, or raise the patternWorkCap engine setting`,
11361
11637
  { cap }
11362
11638
  );
11363
11639
  }
@@ -11588,15 +11864,19 @@ function addPartitionTokens(tokens, addedMembers, partition, term, analysed, exp
11588
11864
  }
11589
11865
  }
11590
11866
  }
11591
- function addReachedTokens(tokens, partitions, term, expansion, language, analysis) {
11867
+ function addReachedTokens(tokens, partitions, term, requested, language, analysis) {
11868
+ if (requested.options.exact === true) return [];
11869
+ const quoted = analyseQueryPhrases(term, language, analysis);
11870
+ const words = quoted?.words ?? term;
11871
+ if (quoted !== null && words.trim().length === 0) return [];
11872
+ const expansion = quoted?.endsWithPhrase === true ? { schema: requested.schema, options: { ...requested.options, prefix: false } } : requested;
11592
11873
  const { options } = expansion;
11593
- if (options.exact === true) return [];
11594
11874
  if (!((options.tolerance ?? 0) > 0 || options.prefix === true || options.partialWords === true)) return [];
11595
- const analysed = queryTokensOf(term, language, analysis);
11875
+ const analysed = queryTokensOf(words, language, analysis);
11596
11876
  if (analysed === null) return [];
11597
11877
  const addedMembers = /* @__PURE__ */ new Set();
11598
11878
  for (const partition of partitions) {
11599
- addPartitionTokens(tokens, addedMembers, partition, term, analysed, expansion, language, analysis);
11879
+ addPartitionTokens(tokens, addedMembers, partition, words, analysed, expansion, language, analysis);
11600
11880
  }
11601
11881
  for (const token of addedMembers) tokens.add(token);
11602
11882
  return [...addedMembers];
@@ -11619,17 +11899,24 @@ function partitionsIn(manager, partitionIds) {
11619
11899
 
11620
11900
  // src/partitioning/distributed-scoring.ts
11621
11901
  function analysedQueryTokens(term, language, analysis) {
11622
- const { tokens } = tokenize(term, language, {
11623
- stem: true,
11624
- removeStopWords: true,
11625
- customTokenizer: analysis.customTokenizer,
11626
- stopWordOverride: analysis.stopWords
11627
- });
11902
+ const quoted = analyseQueryPhrases(term, language, analysis);
11903
+ const words = quoted?.words ?? term;
11628
11904
  const unique = /* @__PURE__ */ new Set();
11629
- for (const { token } of tokens) {
11630
- unique.add(token);
11905
+ if (quoted === null || words.trim().length > 0) {
11906
+ const { tokens } = tokenize(words, language, {
11907
+ stem: true,
11908
+ removeStopWords: true,
11909
+ customTokenizer: analysis.customTokenizer,
11910
+ stopWordOverride: analysis.stopWords
11911
+ });
11912
+ for (const { token } of tokens) {
11913
+ unique.add(token);
11914
+ }
11631
11915
  }
11632
11916
  for (const synonym of analysis.synonyms?.expand(unique)?.tokens ?? []) unique.add(synonym);
11917
+ for (const phrase of quoted?.phrases ?? []) {
11918
+ for (const token of phrase.tokens) unique.add(token);
11919
+ }
11633
11920
  return unique;
11634
11921
  }
11635
11922
  function copyNumericRecord(source) {
@@ -12028,6 +12315,14 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
12028
12315
  }
12029
12316
 
12030
12317
  // src/search/fulltext.ts
12318
+ function analysedWords(term, language, options) {
12319
+ return tokenize(term, language, {
12320
+ stem: true,
12321
+ removeStopWords: true,
12322
+ customTokenizer: options?.customTokenizer,
12323
+ stopWordOverride: options?.stopWords
12324
+ }).tokens;
12325
+ }
12031
12326
  function prepareFulltextQuery(partition, params, language, schema, options) {
12032
12327
  if (!params.term || params.term.trim().length === 0) return null;
12033
12328
  if (params.fields && params.fields.length === 0) return null;
@@ -12035,14 +12330,16 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
12035
12330
  if (params.fields) {
12036
12331
  validateSearchFields(params.fields, flatSchema);
12037
12332
  }
12038
- const queryTokenResult = tokenize(params.term, language, {
12039
- stem: true,
12040
- removeStopWords: true,
12041
- customTokenizer: options?.customTokenizer,
12042
- stopWordOverride: options?.stopWords
12333
+ const quoted = analyseQueryPhrases(params.term, language, {
12334
+ stopWords: options?.stopWords,
12335
+ customTokenizer: options?.customTokenizer
12043
12336
  });
12044
- if (queryTokenResult.tokens.length === 0) return null;
12045
- const queryTokens = deduplicateTokens(queryTokenResult.tokens);
12337
+ const wordParams = quoted === null ? params : requireLoadedPhraseCode().withWordsOnly(params, quoted);
12338
+ const phrases = quoted !== null && quoted.phrases.length > 0 ? quoted.phrases : void 0;
12339
+ const words = wordParams.term ?? "";
12340
+ const analysedTokens = quoted !== null && words.trim().length === 0 ? [] : analysedWords(words, language, options);
12341
+ if (analysedTokens.length === 0 && phrases === void 0) return null;
12342
+ const queryTokens = deduplicateTokens(analysedTokens);
12046
12343
  let filterBitset;
12047
12344
  if (params.filters) {
12048
12345
  filterBitset = partition.applyFiltersBitset(
@@ -12052,16 +12349,19 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
12052
12349
  );
12053
12350
  if (bitsetIsEmpty(filterBitset)) return null;
12054
12351
  }
12055
- const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
12352
+ if (analysedTokens.length === 0) return { queryTokens, phrases, filterBitset };
12353
+ const lastToken = analysedTokens[analysedTokens.length - 1];
12056
12354
  const synonyms = params.exact === true ? void 0 : options?.synonyms?.expand(queryTokens.map((qt) => qt.token));
12057
- const groups = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, options, synonyms);
12058
- return { queryTokens, ...groups, filterBitset };
12355
+ const groups = resolveTermGroups(partition, wordParams, language, schema, queryTokens, lastToken, options, synonyms);
12356
+ return { queryTokens, phrases, ...groups, filterBitset };
12059
12357
  }
12060
12358
  function fulltextMatches(partition, params, language, schema, options) {
12061
12359
  const prepared = prepareFulltextQuery(partition, params, language, schema, options);
12062
12360
  if (prepared === null) return null;
12063
12361
  return partition.searchFulltextMatches({
12064
12362
  queryTokens: prepared.queryTokens,
12363
+ phrases: prepared.phrases,
12364
+ phraseWork: options?.patternWork,
12065
12365
  termGroups: prepared.termGroups,
12066
12366
  partialFields: prepared.partialFields,
12067
12367
  fields: params.fields,
@@ -12076,13 +12376,15 @@ function fulltextSearch(partition, params, language, schema, options) {
12076
12376
  if (prepared === null) {
12077
12377
  return { scored: [], totalMatched: 0 };
12078
12378
  }
12079
- const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
12379
+ const { queryTokens, phrases, termGroups, partialFields, filterBitset } = prepared;
12080
12380
  const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
12081
12381
  const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
12082
12382
  const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
12083
12383
  const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
12084
12384
  const rawResult = partition.searchFulltext({
12085
12385
  queryTokens,
12386
+ phrases,
12387
+ phraseWork: options?.patternWork,
12086
12388
  termGroups,
12087
12389
  partialFields,
12088
12390
  fields: params.fields,
@@ -12100,7 +12402,7 @@ function fulltextSearch(partition, params, language, schema, options) {
12100
12402
  });
12101
12403
  let scored = rawResult.scored;
12102
12404
  const termMatch = params.termMatch ?? "any";
12103
- if (termMatch !== "any") {
12405
+ if (termMatch !== "any" && queryTokens.length > 0) {
12104
12406
  scored = filterByTermCoverage(
12105
12407
  scored,
12106
12408
  queryTokens,
@@ -12109,7 +12411,8 @@ function fulltextSearch(partition, params, language, schema, options) {
12109
12411
  params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
12110
12412
  params.exact ?? false,
12111
12413
  termGroups,
12112
- partialFields ?? []
12414
+ partialFields ?? [],
12415
+ phrases
12113
12416
  );
12114
12417
  }
12115
12418
  if (params.minScore !== void 0 && params.minScore > 0) {
@@ -12151,8 +12454,9 @@ function deduplicateTokens(tokens) {
12151
12454
  }
12152
12455
  return result;
12153
12456
  }
12154
- function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
12457
+ function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields, phrases) {
12155
12458
  const requiredCount = policy === "all" ? queryTokens.length : policy;
12459
+ const phraseKeys = phrases === void 0 ? null : new Set(phrases.map((phrase) => phrase.key));
12156
12460
  if (requiredCount <= 0) return scored;
12157
12461
  if (requiredCount > queryTokens.length) return [];
12158
12462
  const groups = /* @__PURE__ */ new Map();
@@ -12168,12 +12472,22 @@ function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLeng
12168
12472
  });
12169
12473
  }
12170
12474
  return scored.filter((doc) => {
12171
- const matched = countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields);
12475
+ const matched = countDocTermMatches(
12476
+ doc,
12477
+ queryTokens,
12478
+ tolerance,
12479
+ prefixLength,
12480
+ exact,
12481
+ groups,
12482
+ partialFields,
12483
+ phraseKeys
12484
+ );
12172
12485
  return matched >= requiredCount;
12173
12486
  });
12174
12487
  }
12175
- function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
12176
- const indexTokens = Object.keys(doc.idf);
12488
+ function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields, phraseKeys) {
12489
+ const componentKeys = Object.keys(doc.idf);
12490
+ const indexTokens = phraseKeys === null ? componentKeys : componentKeys.filter((key) => !phraseKeys.has(key));
12177
12491
  if (indexTokens.length === 0) return 0;
12178
12492
  let count = 0;
12179
12493
  for (const qt of queryTokens) {
@@ -12328,8 +12642,11 @@ function fanOutMatchCount(manager, params, language, schema, options) {
12328
12642
 
12329
12643
  // src/workers/query-actions.ts
12330
12644
  async function runQueryAction(entry, action) {
12645
+ const loadingPhraseCode = loadPhraseCodeFor(action.params.term);
12646
+ if (loadingPhraseCode !== null) await loadingPhraseCode;
12331
12647
  const patternWork = joinPatternWorkMeter(action.patternWork, action.params.filters);
12332
- const searchOptions = { ...entry.searchOptions, patternWork };
12648
+ const bm25Params = action.type === "query" ? action.bm25 ?? void 0 : entry.config.bm25;
12649
+ const searchOptions = { ...entry.searchOptions, patternWork, bm25Params };
12333
12650
  if (action.type === "query") {
12334
12651
  const result2 = await fanOutQuery(
12335
12652
  entry.manager,
@@ -13447,6 +13764,10 @@ function createFrozenPostingViews(payload, tombstones, counts) {
13447
13764
  if (positionOffsets === null) return 0;
13448
13765
  return positionOffsets[start + row + 1] - positionOffsets[start + row];
13449
13766
  },
13767
+ positionStore: positionValues,
13768
+ positionStartAt(row) {
13769
+ return positionOffsets === null ? 0 : positionOffsets[start + row];
13770
+ },
13450
13771
  docIdSet: {
13451
13772
  get size() {
13452
13773
  return counts.documentFrequency(sortedIndex);
@@ -13565,7 +13886,7 @@ function addForms(target, forms) {
13565
13886
  function createFrozenSurfaceSlot(load, documents) {
13566
13887
  let registry = null;
13567
13888
  let uncounted = null;
13568
- function loaded2() {
13889
+ function loaded3() {
13569
13890
  if (registry === null) {
13570
13891
  const built = createSurfaceRegistry();
13571
13892
  const forms = load();
@@ -13579,26 +13900,26 @@ function createFrozenSurfaceSlot(load, documents) {
13579
13900
  }
13580
13901
  return {
13581
13902
  reader: {
13582
- candidatesForPrefix: (prefix) => loaded2().candidatesForPrefix(prefix),
13583
- stemChangedTotalFor: (token) => loaded2().stemChangedTotalFor(token),
13584
- size: () => loaded2().size(),
13585
- serialize: () => loaded2().serialize()
13903
+ candidatesForPrefix: (prefix) => loaded3().candidatesForPrefix(prefix),
13904
+ stemChangedTotalFor: (token) => loaded3().stemChangedTotalFor(token),
13905
+ size: () => loaded3().size(),
13906
+ serialize: () => loaded3().serialize()
13586
13907
  },
13587
13908
  holdsAny() {
13588
- return loaded2().size() > 0;
13909
+ return loaded3().size() > 0;
13589
13910
  },
13590
13911
  holds(spelling) {
13591
- return loaded2().holds(spelling);
13912
+ return loaded3().holds(spelling);
13592
13913
  },
13593
13914
  subtractSpellings(ordinal, spellings) {
13594
13915
  uncounted ??= new Uint8Array(documents.count);
13595
13916
  uncounted[ordinal] = 1;
13596
13917
  if (spellings.length === 0) return;
13597
- const target = loaded2();
13918
+ const target = loaded3();
13598
13919
  for (const spelling of spellings) target.subtract(spelling, 1);
13599
13920
  },
13600
13921
  addKeptForms(target, mergedOrdinals, analysis) {
13601
- const source = loaded2();
13922
+ const source = loaded3();
13602
13923
  if (analysis === null) {
13603
13924
  addForms(target, source.serialize());
13604
13925
  return;
@@ -14653,36 +14974,6 @@ function mergeSegmentContents(selections, analysis = null, textAnalysis = null)
14653
14974
  return { payload, documents: mergeDocuments2(inputs, documentCount), reusedPartialWords: partialWords.reused };
14654
14975
  }
14655
14976
 
14656
- // src/core/partition/growable-uint32.ts
14657
- var GrowableUint32 = class {
14658
- values;
14659
- length = 0;
14660
- constructor(capacity) {
14661
- this.values = new Uint32Array(capacity);
14662
- }
14663
- growTo(needed, kept) {
14664
- let capacity = this.values.length * 2;
14665
- while (capacity < needed) capacity *= 2;
14666
- const next = new Uint32Array(capacity);
14667
- next.set(this.values.subarray(0, kept));
14668
- this.values = next;
14669
- }
14670
- reserve(extra) {
14671
- const needed = this.length + extra;
14672
- if (needed > this.values.length) this.growTo(needed, this.length);
14673
- }
14674
- push(value) {
14675
- if (this.length === this.values.length) this.growTo(this.length + 1, this.length);
14676
- this.values[this.length++] = value;
14677
- }
14678
- ensureIndex(index) {
14679
- if (index >= this.values.length) this.growTo(index + 1, this.values.length);
14680
- }
14681
- clear() {
14682
- this.length = 0;
14683
- }
14684
- };
14685
-
14686
14977
  // src/core/partition/segment-postings.ts
14687
14978
  var TextPostings = class {
14688
14979
  constructor(trackPositions) {
@@ -14952,8 +15243,10 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
14952
15243
  const spellings = /* @__PURE__ */ new Map();
14953
15244
  let fieldSpellings = null;
14954
15245
  let positionOffset = 0;
15246
+ let lastItemPosition = -1;
14955
15247
  const addToken = (token, position, surface) => {
14956
15248
  postings.addToken(token, positionOffset + position);
15249
+ lastItemPosition = position;
14957
15250
  surfaces.count(surface, token);
14958
15251
  if (fieldSpellings !== null) countPartialSpelling(fieldSpellings, token, surface ?? token, 1);
14959
15252
  };
@@ -14972,8 +15265,9 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
14972
15265
  postings.beginField();
14973
15266
  positionOffset = 0;
14974
15267
  for (const text of fieldType === "string" ? [value] : value) {
15268
+ lastItemPosition = -1;
14975
15269
  visitTokens(text, language, tokenizing, addToken);
14976
- positionOffset = postings.fieldLength;
15270
+ if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
14977
15271
  }
14978
15272
  postings.endField(ordinal, fieldPath);
14979
15273
  lengths.record(ordinal, fieldPath, postings.fieldLength);
@@ -15091,30 +15385,30 @@ function runSegmentAction(entry, action) {
15091
15385
 
15092
15386
  // src/workers/vector-code.ts
15093
15387
  var VECTOR_TYPE_NAME = "vector";
15094
- var loaded = null;
15095
- var loading = null;
15388
+ var loaded2 = null;
15389
+ var loading2 = null;
15096
15390
  function loadedVectorCode() {
15097
- return loaded;
15391
+ return loaded2;
15098
15392
  }
15099
15393
  function requireLoadedVectorCode() {
15100
- if (loaded === null) throw new Error("The engine reached vector code before it loaded that code");
15101
- return loaded;
15394
+ if (loaded2 === null) throw new Error("The engine reached vector code before it loaded that code");
15395
+ return loaded2;
15102
15396
  }
15103
15397
  function loadVectorCode() {
15104
- if (loaded !== null) return Promise.resolve(loaded);
15105
- if (loading === null) {
15106
- loading = import('./module-ACWDC7YI.mjs').then(
15398
+ if (loaded2 !== null) return Promise.resolve(loaded2);
15399
+ if (loading2 === null) {
15400
+ loading2 = import('./module-4HIMOQM5.mjs').then(
15107
15401
  (code) => {
15108
- loaded = code;
15402
+ loaded2 = code;
15109
15403
  return code;
15110
15404
  },
15111
15405
  (err) => {
15112
- loading = null;
15406
+ loading2 = null;
15113
15407
  throw err;
15114
15408
  }
15115
15409
  );
15116
15410
  }
15117
- return loading;
15411
+ return loading2;
15118
15412
  }
15119
15413
  function mayDeclareVectorField(schema, depth) {
15120
15414
  if (!isPlainObject(schema) || depth > MAX_NESTING_DEPTH + 1) return false;
@@ -15126,7 +15420,7 @@ function mayDeclareVectorField(schema, depth) {
15126
15420
  return false;
15127
15421
  }
15128
15422
  function loadVectorCodeFor(schema) {
15129
- if (loaded !== null || !mayDeclareVectorField(schema, 1)) return null;
15423
+ if (loaded2 !== null || !mayDeclareVectorField(schema, 1)) return null;
15130
15424
  return loadVectorCode();
15131
15425
  }
15132
15426
 
@@ -15254,7 +15548,6 @@ function createDirectExecutor(options) {
15254
15548
  config,
15255
15549
  language,
15256
15550
  searchOptions: {
15257
- bm25Params: config.bm25,
15258
15551
  stopWords: manager.analysis.stopWords,
15259
15552
  customTokenizer: manager.analysis.customTokenizer,
15260
15553
  get synonyms() {
@@ -15323,8 +15616,8 @@ function createDirectExecutor(options) {
15323
15616
  );
15324
15617
  }
15325
15618
  case "createIndex": {
15326
- const loading2 = loadVectorCodeFor(action.config.schema);
15327
- if (loading2 !== null) await loading2;
15619
+ const loading3 = loadVectorCodeFor(action.config.schema);
15620
+ if (loading3 !== null) await loading3;
15328
15621
  const language = getLanguage(action.config.language ?? "english");
15329
15622
  const manager = addIndex(
15330
15623
  action.indexName,
@@ -15395,6 +15688,11 @@ function createDirectExecutor(options) {
15395
15688
  if (entry !== void 0) entry.manager.synonyms = lookupOf(entry.config, entry.language, action.synonyms);
15396
15689
  return void 0;
15397
15690
  }
15691
+ case "replaceBm25": {
15692
+ const entry = indexes.get(action.indexName);
15693
+ if (entry !== void 0) entry.config.bm25 = action.bm25;
15694
+ return void 0;
15695
+ }
15398
15696
  case "get": {
15399
15697
  const entry = requireIndex(action.indexName);
15400
15698
  return entry.manager.get(action.docId);
@@ -15688,6 +15986,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
15688
15986
  }
15689
15987
  }
15690
15988
  return {
15989
+ rebuildMode,
15691
15990
  markStale(index) {
15692
15991
  const existing = stale.get(index.indexName);
15693
15992
  if (existing !== void 0 && existing.run !== null) {
@@ -15816,6 +16115,17 @@ function resolveDurabilityTier(config) {
15816
16115
  return { kind: "snapshot", config: {}, adapter: config.persistence };
15817
16116
  }
15818
16117
 
16118
+ // src/engine/array-positions.ts
16119
+ function holdsStringArrayField(schema) {
16120
+ return schemaFieldsOf(schema).some((field) => field.base === "string[]");
16121
+ }
16122
+ function arrayPositionsAreStale(config, storedGap) {
16123
+ return storedGap !== ARRAY_POSITION_GAP && config.trackPositions !== false && holdsStringArrayField(config.schema);
16124
+ }
16125
+ function storedArrayPositionGap(value) {
16126
+ return typeof value === "number" ? value : null;
16127
+ }
16128
+
15819
16129
  // src/engine/recovery-schema.ts
15820
16130
  function unflattenSchema(flat) {
15821
16131
  const root = {};
@@ -15842,7 +16152,11 @@ function reconstructSchemaFromMetadata(metadata) {
15842
16152
  const config = {
15843
16153
  schema: unflattenSchema(metadata.schema),
15844
16154
  language: metadata.language,
15845
- bm25: { k1: metadata.bm25Params.k1, b: metadata.bm25Params.b }
16155
+ bm25: {
16156
+ k1: metadata.bm25Params.k1,
16157
+ b: metadata.bm25Params.b,
16158
+ ...metadata.bm25Params.fields !== void 0 ? { fields: metadata.bm25Params.fields } : {}
16159
+ }
15846
16160
  };
15847
16161
  if (metadata.embedding) {
15848
16162
  config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
@@ -16037,16 +16351,6 @@ function validateWorkerConfig(workers, lifecycle) {
16037
16351
  );
16038
16352
  }
16039
16353
  }
16040
- function validateBM25Params(bm25) {
16041
- if (bm25 === void 0) return;
16042
- const { k1, b } = bm25;
16043
- if (k1 !== void 0 && (!Number.isFinite(k1) || k1 < 0)) {
16044
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.k1 must be a finite number at or above 0", { k1 });
16045
- }
16046
- if (b !== void 0 && (!Number.isFinite(b) || b < 0 || b > 1)) {
16047
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.b must be between 0 and 1", { b });
16048
- }
16049
- }
16050
16354
  function clampLimit(limit) {
16051
16355
  return clampRowCount(limit, DEFAULT_LIMIT);
16052
16356
  }
@@ -16144,11 +16448,12 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
16144
16448
  documentCount: metadata.documentCount ?? 0,
16145
16449
  partitionCount: metadata.partitionCount,
16146
16450
  analysisRevision: metadata.analysisRevision ?? null,
16451
+ arrayPositionGap: metadata.arrayPositionGap ?? null,
16147
16452
  synonyms
16148
16453
  });
16149
16454
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
16150
16455
  else deps.indexState().registerClosed(metadata.indexName);
16151
- if (metadata.analysisRevision !== language.revision) {
16456
+ if (metadata.analysisRevision !== language.revision || arrayPositionsAreStale(indexConfig, metadata.arrayPositionGap ?? null)) {
16152
16457
  deps.analysisRebuild().markStale({
16153
16458
  indexName: metadata.indexName,
16154
16459
  language: language.name,
@@ -17464,10 +17769,10 @@ async function replicateToWorkers(state, action) {
17464
17769
  return;
17465
17770
  }
17466
17771
  const detached = detachCallerDocuments(state, action);
17467
- const loading2 = state.copyLoadBuffers.get(action.indexName);
17468
- if (loading2 !== void 0) {
17772
+ const loading3 = state.copyLoadBuffers.get(action.indexName);
17773
+ if (loading3 !== void 0) {
17469
17774
  queueForCopies(state, detached);
17470
- if (loading2.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop();
17775
+ if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop();
17471
17776
  return;
17472
17777
  }
17473
17778
  const queue = state.replicationQueues.get(action.indexName);
@@ -17478,9 +17783,9 @@ async function replicateToWorkers(state, action) {
17478
17783
  }
17479
17784
  function queueForCopies(state, action) {
17480
17785
  if (!("indexName" in action)) return;
17481
- const loading2 = state.copyLoadBuffers.get(action.indexName);
17482
- if (loading2 !== void 0) {
17483
- loading2.push(action);
17786
+ const loading3 = state.copyLoadBuffers.get(action.indexName);
17787
+ if (loading3 !== void 0) {
17788
+ loading3.push(action);
17484
17789
  return;
17485
17790
  }
17486
17791
  if (state.workerPool === null || !state.scaledOutIndexes.has(action.indexName)) return;
@@ -18796,23 +19101,24 @@ function scoresPerPartition(state, indexName, params, globalStats) {
18796
19101
  if (mode === "local") return true;
18797
19102
  return mode === "broadcast" && globalStats !== void 0;
18798
19103
  }
18799
- function queryAction(indexName, params, globalStats, partitionIds, patternWork) {
19104
+ function queryAction(indexName, params, globalStats, partitionIds, patternWork, bm25) {
18800
19105
  return {
18801
19106
  type: "query",
18802
19107
  indexName,
18803
19108
  params,
18804
19109
  requestId: createRequestId(),
18805
19110
  patternWork,
19111
+ bm25: bm25 ?? null,
18806
19112
  ...partitionIds !== void 0 ? { partitionIds } : {},
18807
19113
  ...globalStats !== void 0 ? { globalStats } : {}
18808
19114
  };
18809
19115
  }
18810
- async function runSplit(leases, scope, indexName, params, globalStats, patternWork) {
19116
+ async function runSplit(leases, scope, indexName, params, globalStats, patternWork, bm25) {
18811
19117
  const assignments = partitionsPerLease(scope, leases.length);
18812
19118
  const workerParams = params.facets !== void 0 ? { ...params, facets: oversampledFacetConfig(params.facets) } : params;
18813
19119
  const outcomes = await Promise.allSettled(
18814
19120
  assignments.map(
18815
- (partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork)).finally(() => leases[at].release())
19121
+ (partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork, bm25)).finally(() => leases[at].release())
18816
19122
  )
18817
19123
  );
18818
19124
  const results = [];
@@ -18848,7 +19154,7 @@ function answersFasterOnMainCopy(state, pool, manager) {
18848
19154
  if (manager.countDocuments() > MAIN_COPY_LONE_QUERY_DOCUMENTS) return false;
18849
19155
  return takeMainCopyTurn(state);
18850
19156
  }
18851
- async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork) {
19157
+ async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25) {
18852
19158
  const pool = state.workerPool;
18853
19159
  if (!pool) return null;
18854
19160
  if (!state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) return null;
@@ -18863,7 +19169,7 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
18863
19169
  if (idle.length >= 2 && scoresPerPartition(state, indexName, params, globalStats)) {
18864
19170
  leases.push(...idle);
18865
19171
  fork = patternWork.fork(leases.length);
18866
- const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared);
19172
+ const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared, bm25);
18867
19173
  fork.settle(merged.patternWorkSpent ?? 0);
18868
19174
  return merged;
18869
19175
  }
@@ -18873,7 +19179,7 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
18873
19179
  leases.push(lease);
18874
19180
  fork = patternWork.fork(1);
18875
19181
  const result = await lease.executor.execute(
18876
- queryAction(indexName, params, globalStats, partitionIds, fork.shared)
19182
+ queryAction(indexName, params, globalStats, partitionIds, fork.shared, bm25)
18877
19183
  );
18878
19184
  fork.settle(result.patternWorkSpent ?? 0);
18879
19185
  return result;
@@ -19134,7 +19440,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
19134
19440
  segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
19135
19441
  holdUnbroadcastSegments: (indexName, segmentIds) => holdUnbroadcastSegments(state, indexName, segmentIds),
19136
19442
  releaseUnbroadcastSegments: (indexName, segmentIds) => releaseUnbroadcastSegments(state, indexName, segmentIds),
19137
- searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork),
19443
+ searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork, bm25) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25),
19138
19444
  hasWorkerPool: () => state.workerPool !== null,
19139
19445
  mainCopyQueries: () => state.mainCopyQueries ?? "lone",
19140
19446
  shareMainThread: () => {
@@ -19146,10 +19452,10 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
19146
19452
  stopRequestThreads: () => stopRequestThreads(state),
19147
19453
  desyncIndex: (indexName) => {
19148
19454
  state.desyncedIndexes.add(indexName);
19149
- const loading2 = state.copyTransitions.get(indexName);
19455
+ const loading3 = state.copyTransitions.get(indexName);
19150
19456
  const wasScaledOut = state.scaledOutIndexes.delete(indexName);
19151
19457
  if (wasScaledOut) withdrawCopies(indexName);
19152
- return wasScaledOut || loading2 !== void 0 && loading2.kind !== "drop";
19458
+ return wasScaledOut || loading3 !== void 0 && loading3.kind !== "drop";
19153
19459
  },
19154
19460
  async releaseIndexCopies(indexName) {
19155
19461
  const hold = state.copyReleaseHolds.get(indexName);
@@ -19198,7 +19504,7 @@ async function emitWriteEventAfter(source, indexName, write) {
19198
19504
  // src/engine/core.ts
19199
19505
  async function createEngineCore(config, hooks, capacityScope = "index") {
19200
19506
  validateWorkerConfig(config?.workers, config?.lifecycle);
19201
- const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-BX6XRFYR.mjs')).resolveRerankSettings(config);
19507
+ const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-3X7ZJ5T3.mjs')).resolveRerankSettings(config);
19202
19508
  const eventHandlers = eventHandlersOf(config?.events);
19203
19509
  const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
19204
19510
  const vectorCopyPolicy = {
@@ -19209,7 +19515,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19209
19515
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
19210
19516
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
19211
19517
  }
19212
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-PUC4QSCR.mjs');
19518
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-FNZSRH4N.mjs');
19213
19519
  const filesystemDurability = durabilityTier?.kind === "wal";
19214
19520
  const executor = createDirectExecutor({
19215
19521
  vectorWorkerCopies: vectorCopyPolicy,
@@ -19375,7 +19681,9 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19375
19681
  resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
19376
19682
  recordCurrentAnalysis: (indexName) => {
19377
19683
  const entry = indexRegistry.get(indexName);
19378
- if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
19684
+ if (entry === void 0 || analysisRebuild.isStale(indexName)) return;
19685
+ entry.analysisRevision = entry.language.revision;
19686
+ entry.arrayPositionGap = ARRAY_POSITION_GAP;
19379
19687
  },
19380
19688
  durabilityManager: durability?.manager ?? null
19381
19689
  });
@@ -19506,7 +19814,7 @@ async function shutdownEngine(core) {
19506
19814
  }
19507
19815
  }
19508
19816
  if (core.rerankAdapters.size > 0) {
19509
- await (await import('./runtime-BX6XRFYR.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19817
+ await (await import('./runtime-3X7ZJ5T3.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19510
19818
  }
19511
19819
  await orchestrator.shutdown();
19512
19820
  await executor.shutdown();
@@ -19539,6 +19847,105 @@ async function resolveRunnableConfig(config) {
19539
19847
  }
19540
19848
  }
19541
19849
 
19850
+ // src/search/bm25-params.ts
19851
+ var BM25_KEYS = /* @__PURE__ */ new Set(["k1", "b", "fields"]);
19852
+ var BM25_FIELD_KEYS = /* @__PURE__ */ new Set(["k1", "b"]);
19853
+ function displayed(value) {
19854
+ if (typeof value === "string") return `"${value.slice(0, BM25_DISPLAYED_NAME_LENGTH)}"`;
19855
+ if (typeof value === "number" || typeof value === "boolean" || value === null) return String(value);
19856
+ return `a value of type ${typeof value}`;
19857
+ }
19858
+ function requireValue(label, value, maximum, ranges) {
19859
+ if (value === void 0) return;
19860
+ const usable = typeof value === "number" && Number.isFinite(value) && (ranges === "stored" || value >= 0 && value <= maximum);
19861
+ if (usable) return;
19862
+ throw new NarsilError(
19863
+ ErrorCodes.CONFIG_INVALID,
19864
+ `${label} must be a finite number from 0 to ${maximum}, and this request sets ${displayed(value)}`,
19865
+ { parameter: label }
19866
+ );
19867
+ }
19868
+ function requireKnownKeys2(label, value, known) {
19869
+ for (const key of Object.keys(value)) {
19870
+ if (known.has(key)) continue;
19871
+ throw new NarsilError(
19872
+ ErrorCodes.CONFIG_INVALID,
19873
+ `${label} takes ${[...known].join(", ")}, and this request sets ${displayed(key)}`,
19874
+ { parameter: label }
19875
+ );
19876
+ }
19877
+ }
19878
+ function requireFieldPair(name, pair, ranges) {
19879
+ const label = `bm25.fields.${name.slice(0, BM25_DISPLAYED_NAME_LENGTH)}`;
19880
+ if (!isPlainObject(pair)) {
19881
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${label} must be an object that sets k1 or b`, {
19882
+ parameter: label
19883
+ });
19884
+ }
19885
+ requireKnownKeys2(label, pair, BM25_FIELD_KEYS);
19886
+ requireValue(`${label}.k1`, pair.k1, MAX_BM25_K1, ranges);
19887
+ requireValue(`${label}.b`, pair.b, MAX_BM25_B, ranges);
19888
+ }
19889
+ function requireScoredField(name, flatSchema, unknownFieldCode) {
19890
+ const fieldType = Object.hasOwn(flatSchema, name) ? flatSchema[name] : void 0;
19891
+ if (typeof fieldType === "string" && isWordIndexedFieldType(fieldType)) return;
19892
+ throw new NarsilError(
19893
+ unknownFieldCode,
19894
+ fieldType === void 0 ? `BM25 parameters apply to a text field that the schema declares, and the schema declares no field ${displayed(name)}` : `BM25 parameters apply to a text field alone, and the schema declares field ${displayed(name)} as ${displayed(fieldType)}`,
19895
+ { field: name.slice(0, BM25_DISPLAYED_NAME_LENGTH) }
19896
+ );
19897
+ }
19898
+ function fieldsOfCheckedSetting(raw, ranges) {
19899
+ if (!isPlainObject(raw)) {
19900
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25 must be an object that sets k1, b, or fields");
19901
+ }
19902
+ requireKnownKeys2("bm25", raw, BM25_KEYS);
19903
+ requireValue("bm25.k1", raw.k1, MAX_BM25_K1, ranges);
19904
+ requireValue("bm25.b", raw.b, MAX_BM25_B, ranges);
19905
+ const fields = raw.fields;
19906
+ if (fields === void 0) return void 0;
19907
+ if (!isPlainObject(fields)) {
19908
+ throw new NarsilError(
19909
+ ErrorCodes.CONFIG_INVALID,
19910
+ "bm25.fields must be an object that maps a field to its parameters"
19911
+ );
19912
+ }
19913
+ return fields;
19914
+ }
19915
+ function validateBM25Params(raw, schema, unknownFieldCode, ranges = "settable") {
19916
+ const fields = fieldsOfCheckedSetting(raw, ranges);
19917
+ if (fields === void 0) return;
19918
+ const names = Object.keys(fields);
19919
+ if (names.length === 0) return;
19920
+ const flatSchema = flattenSchema(schema);
19921
+ for (const name of names) {
19922
+ requireScoredField(name, flatSchema, unknownFieldCode);
19923
+ requireFieldPair(name, fields[name], ranges);
19924
+ }
19925
+ }
19926
+ function presentPair(pair) {
19927
+ return {
19928
+ ...pair.k1 !== void 0 ? { k1: pair.k1 } : {},
19929
+ ...pair.b !== void 0 ? { b: pair.b } : {}
19930
+ };
19931
+ }
19932
+ function storedBM25Params(params) {
19933
+ const stored = presentPair(params);
19934
+ const fields = params.fields;
19935
+ if (fields !== void 0) {
19936
+ const kept = /* @__PURE__ */ Object.create(null);
19937
+ let holdsField = false;
19938
+ for (const name of Object.keys(fields)) {
19939
+ const pair = presentPair(fields[name]);
19940
+ if (pair.k1 === void 0 && pair.b === void 0) continue;
19941
+ kept[name] = pair;
19942
+ holdsField = true;
19943
+ }
19944
+ if (holdsField) stored.fields = kept;
19945
+ }
19946
+ return Object.keys(stored).length === 0 ? void 0 : stored;
19947
+ }
19948
+
19542
19949
  // src/serialization/crc32.ts
19543
19950
  var IEEE_POLYNOMIAL = 3988292384;
19544
19951
  var ALL_BITS = 4294967295;
@@ -19836,6 +20243,7 @@ function indexConfigurationOf(entry) {
19836
20243
  entry.language,
19837
20244
  entry.embeddingAdapterName,
19838
20245
  entry.analysisRevision,
20246
+ entry.arrayPositionGap,
19839
20247
  synonyms
19840
20248
  );
19841
20249
  }
@@ -19845,7 +20253,8 @@ function requireConfigurationWithinSizeLimit(config, language) {
19845
20253
  config,
19846
20254
  language,
19847
20255
  typeof adapter === "string" ? adapter : null,
19848
- language.revision
20256
+ language.revision,
20257
+ ARRAY_POSITION_GAP
19849
20258
  );
19850
20259
  if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
19851
20260
  const bytes = encode(payload).byteLength;
@@ -19856,19 +20265,21 @@ function requireConfigurationWithinSizeLimit(config, language) {
19856
20265
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
19857
20266
  );
19858
20267
  }
19859
- function configurationPayloadOf(config, language, adapterName, analysisRevision, synonyms) {
20268
+ function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
19860
20269
  const bm25 = config.bm25;
19861
20270
  return {
19862
20271
  schema: config.schema,
19863
20272
  language: language.name,
19864
20273
  ...analysisRevision !== null ? { analysisRevision } : {},
20274
+ ...arrayPositionGap !== null ? { arrayPositionGap } : {},
19865
20275
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
19866
20276
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
19867
20277
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
19868
20278
  ...bm25 !== void 0 ? {
19869
20279
  bm25: {
19870
20280
  ...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
19871
- ...bm25.b !== void 0 ? { b: bm25.b } : {}
20281
+ ...bm25.b !== void 0 ? { b: bm25.b } : {},
20282
+ ...bm25.fields !== void 0 ? { fields: bm25.fields } : {}
19872
20283
  }
19873
20284
  } : {},
19874
20285
  surfaceForms: config.surfaceForms !== false,
@@ -19956,14 +20367,21 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
19956
20367
  const schema = validateSchema(requestedConfig.schema);
19957
20368
  if (appliesCreationLimits) requireCreatableSchema(schema);
19958
20369
  const configured = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
19959
- const indexConfig = withoutSynonyms(configured);
20370
+ const indexConfig = { ...withoutSynonyms(configured) };
19960
20371
  requirePatternSearchFor(schema);
19961
20372
  requirePartialWordSearchFor(schema);
19962
20373
  validatePatternValueLimit(indexConfig.patternValueLimit);
19963
20374
  requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
19964
20375
  validateVectorPromotion(indexConfig.vectorPromotion);
19965
20376
  validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
19966
- validateBM25Params(indexConfig.bm25);
20377
+ if (indexConfig.bm25 !== void 0) {
20378
+ validateBM25Params(
20379
+ indexConfig.bm25,
20380
+ schema,
20381
+ ErrorCodes.CONFIG_INVALID,
20382
+ appliesCreationLimits ? "settable" : "stored"
20383
+ );
20384
+ }
19967
20385
  if (indexConfig.partitions) {
19968
20386
  validatePartitionConfig(indexConfig.partitions);
19969
20387
  }
@@ -20030,6 +20448,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
20030
20448
  documentCount: 0,
20031
20449
  partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
20032
20450
  analysisRevision: language.revision,
20451
+ arrayPositionGap: ARRAY_POSITION_GAP,
20033
20452
  synonyms: synonyms?.encoded ?? null
20034
20453
  });
20035
20454
  await core.indexState.registerOpen(name);
@@ -21985,6 +22404,75 @@ function readProcessMemory() {
21985
22404
  }
21986
22405
  }
21987
22406
 
22407
+ // src/engine/serial-index-update.ts
22408
+ var updatingByCore = /* @__PURE__ */ new WeakMap();
22409
+ async function withOpenIndex(core, indexName, action) {
22410
+ core.guardShutdown();
22411
+ core.requireIndex(indexName);
22412
+ const release = await core.indexState.acquire(indexName);
22413
+ try {
22414
+ return await action();
22415
+ } finally {
22416
+ release();
22417
+ }
22418
+ }
22419
+ async function oneAtATime(core, indexName, action) {
22420
+ let chains = updatingByCore.get(core);
22421
+ if (chains === void 0) {
22422
+ chains = /* @__PURE__ */ new Map();
22423
+ updatingByCore.set(core, chains);
22424
+ }
22425
+ const updating = chains;
22426
+ const earlier = updating.get(indexName) ?? Promise.resolve();
22427
+ const current = earlier.catch(() => void 0).then(action);
22428
+ updating.set(indexName, current);
22429
+ try {
22430
+ return await current;
22431
+ } finally {
22432
+ if (updating.get(indexName) === current) updating.delete(indexName);
22433
+ }
22434
+ }
22435
+
22436
+ // src/engine/bm25-update.ts
22437
+ async function replaceNow(core, indexName, bm25) {
22438
+ const { config } = core.requireIndex(indexName);
22439
+ const previous = config.bm25;
22440
+ config.bm25 = bm25;
22441
+ try {
22442
+ if (core.durability) await core.durability.manager.persistMetadata(indexName);
22443
+ } catch (err) {
22444
+ config.bm25 = previous;
22445
+ throw err;
22446
+ }
22447
+ await core.orchestrator.replicateToWorkers({
22448
+ type: "replaceBm25",
22449
+ indexName,
22450
+ requestId: `replace-bm25-${indexName}`,
22451
+ ...bm25 !== void 0 ? { bm25 } : {}
22452
+ });
22453
+ await core.orchestrator.awaitWrites(indexName);
22454
+ }
22455
+ function updateIndexBm25(core, indexName, value) {
22456
+ return withOpenIndex(
22457
+ core,
22458
+ indexName,
22459
+ () => oneAtATime(core, indexName, () => {
22460
+ const entry = core.requireIndex(indexName);
22461
+ validateBM25Params(value, entry.config.schema, ErrorCodes.CONFIG_INVALID);
22462
+ return replaceNow(core, indexName, storedBM25Params(value));
22463
+ })
22464
+ );
22465
+ }
22466
+
22467
+ // src/narsil/bm25.ts
22468
+ function bm25Operations(core) {
22469
+ return {
22470
+ async updateBM25(indexName, bm25) {
22471
+ await updateIndexBm25(core, indexName, bm25);
22472
+ }
22473
+ };
22474
+ }
22475
+
21988
22476
  // src/narsil/internals.ts
21989
22477
  var engineCores = /* @__PURE__ */ new WeakMap();
21990
22478
  function bindEngineCore(engine, core) {
@@ -22005,18 +22493,37 @@ function restoredBm25(raw, reject2) {
22005
22493
  if (!isPlainObject3(raw)) {
22006
22494
  reject2("bm25 must be an object");
22007
22495
  }
22496
+ const fields = restoredBm25Fields(raw.fields, reject2);
22497
+ return {
22498
+ ...restoredBm25Pair(raw, "bm25", reject2),
22499
+ ...fields !== void 0 ? { fields } : {}
22500
+ };
22501
+ }
22502
+ function restoredBm25Pair(raw, label, reject2) {
22008
22503
  const { k1, b } = raw;
22009
22504
  if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
22010
- reject2("bm25 k1 must be a finite number");
22505
+ reject2(`${label} k1 must be a finite number`);
22011
22506
  }
22012
22507
  if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
22013
- reject2("bm25 b must be a finite number");
22508
+ reject2(`${label} b must be a finite number`);
22014
22509
  }
22015
22510
  return {
22016
22511
  ...k1 !== void 0 ? { k1 } : {},
22017
22512
  ...b !== void 0 ? { b } : {}
22018
22513
  };
22019
22514
  }
22515
+ function restoredBm25Fields(raw, reject2) {
22516
+ if (raw === void 0) return void 0;
22517
+ if (!isPlainObject3(raw)) reject2("bm25 fields must be an object");
22518
+ const source = raw;
22519
+ const fields = /* @__PURE__ */ Object.create(null);
22520
+ for (const name of Object.keys(source)) {
22521
+ const pair = source[name];
22522
+ if (!isPlainObject3(pair)) reject2("bm25 fields must map each field to an object");
22523
+ fields[name] = restoredBm25Pair(pair, "bm25 field", reject2);
22524
+ }
22525
+ return fields;
22526
+ }
22020
22527
  function restoredStopWords(name, list, reject2) {
22021
22528
  if (name !== void 0 && typeof name !== "string") {
22022
22529
  reject2("stopWords must be a name");
@@ -22614,7 +23121,7 @@ async function readDocumentExport(engine, indexName, source) {
22614
23121
 
22615
23122
  // src/narsil/portability.ts
22616
23123
  function portabilityOperations(core, config, engine) {
22617
- async function withOpenIndex(indexName, action) {
23124
+ async function withOpenIndex2(indexName, action) {
22618
23125
  core.guardShutdown();
22619
23126
  core.requireIndex(indexName);
22620
23127
  const release = await core.indexState.acquire(indexName);
@@ -22642,25 +23149,25 @@ function portabilityOperations(core, config, engine) {
22642
23149
  }
22643
23150
  return {
22644
23151
  snapshot(indexName) {
22645
- return withOpenIndex(indexName, async () => {
22646
- const { createSnapshot } = await import('./snapshot-U5Y346XG.mjs');
23152
+ return withOpenIndex2(indexName, async () => {
23153
+ const { createSnapshot } = await import('./snapshot-V7VF6BN6.mjs');
22647
23154
  return createSnapshot(snapshotDepsOf(indexName));
22648
23155
  });
22649
23156
  },
22650
23157
  async restore(indexName, data) {
22651
23158
  core.guardShutdown();
22652
23159
  validateIndexName(indexName);
22653
- const restored = import('./restore-queue-MLUEUEAN.mjs').then(
23160
+ const restored = import('./restore-queue-GEZ54CA5.mjs').then(
22654
23161
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
22655
23162
  );
22656
23163
  await emitWriteEventAfter(core, indexName, restored);
22657
23164
  core.heapPressureNotifier.check(indexName);
22658
23165
  },
22659
23166
  exportDocuments(indexName) {
22660
- return withOpenIndex(indexName, async () => {
23167
+ return withOpenIndex2(indexName, async () => {
22661
23168
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
22662
- import('./snapshot-U5Y346XG.mjs'),
22663
- import('./write-GZHR6AVG.mjs')
23169
+ import('./snapshot-V7VF6BN6.mjs'),
23170
+ import('./write-YIC2MKRY.mjs')
22664
23171
  ]);
22665
23172
  const header = indexConfigurationOf(core.requireIndex(indexName));
22666
23173
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -22816,6 +23323,27 @@ function boundRerankOf(rerank) {
22816
23323
  }
22817
23324
  return present;
22818
23325
  }
23326
+ function presentMembersOf(record) {
23327
+ const present = {};
23328
+ for (const [key, value] of Object.entries(record)) {
23329
+ if (value !== void 0) present[key] = value;
23330
+ }
23331
+ return present;
23332
+ }
23333
+ function boundBm25Of(bm25) {
23334
+ const bound = presentMembersOf(bm25);
23335
+ const fields = bm25.fields;
23336
+ if (typeof fields !== "object" || fields === null) return bound;
23337
+ const boundFields = {};
23338
+ for (const [name, pair] of Object.entries(fields)) {
23339
+ const members = typeof pair === "object" && pair !== null ? presentMembersOf(pair) : pair;
23340
+ if (typeof members === "object" && members !== null && Object.keys(members).length === 0) continue;
23341
+ boundFields[name] = members;
23342
+ }
23343
+ if (Object.keys(boundFields).length === 0) delete bound.fields;
23344
+ else bound.fields = boundFields;
23345
+ return bound;
23346
+ }
22819
23347
  var FILTERS_SLOT = 2;
22820
23348
  var BOUND_SLOT_COUNT = 16;
22821
23349
  function bindingOf(values, clock) {
@@ -22846,6 +23374,7 @@ function queryBindingOf(params, clock) {
22846
23374
  params.scoring ?? "local"
22847
23375
  ];
22848
23376
  if (params.rerank !== void 0) values.push(boundRerankOf(params.rerank));
23377
+ if (params.bm25 !== void 0 && bm25SetsValue(params.bm25)) values.push(boundBm25Of(params.bm25));
22849
23378
  return bindingOf(values, clock);
22850
23379
  }
22851
23380
  function lazyQueryBinding(params, clock) {
@@ -22908,6 +23437,7 @@ function requireGroupableFields(fields, flatSchema, strict) {
22908
23437
  }
22909
23438
  }
22910
23439
  function requireUsableQueryFields(params, schema, strict = false) {
23440
+ if (params.bm25 !== void 0) validateBM25Params(params.bm25, schema, ErrorCodes.SEARCH_INVALID_FIELD);
22911
23441
  const groupFields = params.group?.fields;
22912
23442
  const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
22913
23443
  if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
@@ -22986,9 +23516,13 @@ function broadcastStatsForWorker(params, context, scoring) {
22986
23516
  expansion: { options: params, schema: context.config.schema }
22987
23517
  });
22988
23518
  }
22989
- function searchOptionsFor(manager, patternWork) {
23519
+ function bm25SettingFor(params, context) {
23520
+ return mergeBM25Levels(params.bm25, context.indexBm25 ?? context.config.bm25);
23521
+ }
23522
+ function searchOptionsFor(params, context) {
23523
+ const { manager, patternWork } = context;
22990
23524
  return {
22991
- bm25Params: manager.config.bm25,
23525
+ bm25Params: bm25SettingFor(params, context),
22992
23526
  stopWords: manager.analysis.stopWords,
22993
23527
  customTokenizer: manager.analysis.customTokenizer,
22994
23528
  synonyms: manager.synonyms,
@@ -23040,6 +23574,13 @@ function validQueryFilter(filters, schema) {
23040
23574
  if (filters === void 0 || filters === null) return filters;
23041
23575
  return validatedFilterOf(filters, flattenSchema(schema));
23042
23576
  }
23577
+ function requireUsablePhrases(params, context) {
23578
+ if (params.term === void 0) return;
23579
+ const quoted = analyseQueryPhrases(params.term, context.language, context.manager.analysis);
23580
+ if (quoted === null || quoted.keptPhraseCount === 0) return;
23581
+ requireLoadedPhraseCode().requirePhrasesWithinLimits(quoted, context.config.trackPositions !== false);
23582
+ context.requirePhraseReady?.();
23583
+ }
23043
23584
  function requireVectorSearchable(params, context, needsVector) {
23044
23585
  if (!needsVector) return;
23045
23586
  const field = params.vector?.field;
@@ -23445,8 +23986,8 @@ function highlightField(text, queryTokens, language, options) {
23445
23986
  }
23446
23987
 
23447
23988
  // src/engine/query/highlight.ts
23448
- function writtenWordsOf(params, language, analysis) {
23449
- return tokenize(params.term ?? "", language, {
23989
+ function writtenWordsOf(term, language, analysis) {
23990
+ return tokenize(term, language, {
23450
23991
  stem: false,
23451
23992
  removeStopWords: true,
23452
23993
  customTokenizer: analysis.customTokenizer,
@@ -23464,18 +24005,21 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
23464
24005
  highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
23465
24006
  };
23466
24007
  }
23467
- function applyHighlights(hits, params, language, analysis, schema, readStoredDocument, synonymLookup) {
23468
- if (!params.highlight) return;
23469
- const queryTokenResult = tokenize(params.term ?? "", language, {
24008
+ function applyHighlights(hits, request, language, analysis, schema, readStoredDocument, synonymLookup) {
24009
+ if (!request.highlight) return;
24010
+ const quoted = analyseQueryPhrases(request.term ?? "", language, analysis);
24011
+ const params = quoted === null ? request : requireLoadedPhraseCode().withWordsOnly(request, quoted);
24012
+ const words = params.term ?? "";
24013
+ const highlightConfig = request.highlight;
24014
+ const queryTokens = quoted !== null && words.trim().length === 0 ? [] : tokenize(words, language, {
23470
24015
  stem: true,
23471
24016
  removeStopWords: true,
23472
24017
  customTokenizer: analysis.customTokenizer,
23473
24018
  stopWordOverride: analysis.stopWords
23474
- });
23475
- const queryTokens = queryTokenResult.tokens;
24019
+ }).tokens;
23476
24020
  const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
23477
24021
  const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
23478
- const written = usesPrefix || usesPartialWords ? writtenWordsOf(params, language, analysis) : [];
24022
+ const written = usesPrefix || usesPartialWords ? writtenWordsOf(words, language, analysis) : [];
23479
24023
  let prefixToken;
23480
24024
  let completedToken;
23481
24025
  if (usesPrefix) {
@@ -23489,12 +24033,14 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
23489
24033
  }
23490
24034
  }
23491
24035
  const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
23492
- const synonyms = params.exact === true ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
24036
+ const wordSynonyms = params.exact === true || queryTokens.length === 0 ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
24037
+ const phraseTokens = quoted?.phrases.flatMap((phrase) => phrase.tokens) ?? [];
24038
+ const synonyms = phraseTokens.length === 0 ? wordSynonyms : [...wordSynonyms ?? [], ...phraseTokens];
23493
24039
  for (const hit of hits) {
23494
24040
  const highlights = {};
23495
24041
  const projected = hit.document;
23496
24042
  let stored;
23497
- for (const field of params.highlight.fields) {
24043
+ for (const field of highlightConfig.fields) {
23498
24044
  let fieldValue = projected[field];
23499
24045
  if (typeof fieldValue !== "string" && readStoredDocument !== void 0) {
23500
24046
  stored ??= readStoredDocument(hit.id);
@@ -23502,9 +24048,9 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
23502
24048
  }
23503
24049
  if (typeof fieldValue === "string") {
23504
24050
  highlights[field] = highlightField(fieldValue, queryTokens, language, {
23505
- preTag: params.highlight.preTag,
23506
- postTag: params.highlight.postTag,
23507
- maxSnippetLength: params.highlight.maxSnippetLength,
24051
+ preTag: highlightConfig.preTag,
24052
+ postTag: highlightConfig.postTag,
24053
+ maxSnippetLength: highlightConfig.maxSnippetLength,
23508
24054
  prefixToken,
23509
24055
  completedToken,
23510
24056
  tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
@@ -23749,7 +24295,8 @@ async function textLeg(textOnlyParams, context) {
23749
24295
  textOnlyParams,
23750
24296
  broadcastStatsForWorker(textOnlyParams, context, scoring),
23751
24297
  context.partitionIds,
23752
- context.patternWork
24298
+ context.patternWork,
24299
+ bm25SettingFor(textOnlyParams, context)
23753
24300
  );
23754
24301
  if (viaWorker) return viaWorker;
23755
24302
  }
@@ -23759,7 +24306,7 @@ async function textLeg(textOnlyParams, context) {
23759
24306
  language,
23760
24307
  config.schema,
23761
24308
  scoring,
23762
- searchOptionsFor(manager, context.patternWork)
24309
+ searchOptionsFor(textOnlyParams, context)
23763
24310
  );
23764
24311
  }
23765
24312
  async function vectorLeg(vectorConfig, queryVector, context, filtered, k, rescoreWindow) {
@@ -23857,7 +24404,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
23857
24404
  const flatSchema = flattenSchema(schema);
23858
24405
  const fieldTypes = fields.map((field) => flatSchema[field]);
23859
24406
  const modes = sortModesOf(normalized);
23860
- const options = searchOptionsFor(manager, context.patternWork);
24407
+ const options = searchOptionsFor(params, context);
23861
24408
  const partitions = partitionsFor(manager, context.partitionIds);
23862
24409
  const partitionLimit = offset + limit + 1;
23863
24410
  const merged = [];
@@ -23925,13 +24472,17 @@ function requireCursorDepth(cursor, signature, binding) {
23925
24472
  }
23926
24473
  function hybridSearchOf(params, context, limit, offset, cursorDepth) {
23927
24474
  if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
23928
- return import('./runtime-BX6XRFYR.mjs').then(
24475
+ return import('./runtime-3X7ZJ5T3.mjs').then(
23929
24476
  (runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
23930
24477
  );
23931
24478
  }
23932
24479
  function executeQuery(request, callerContext) {
23933
- if (request.rerank === void 0) return runQuery(request, callerContext);
23934
- return import('./runtime-BX6XRFYR.mjs').then(
24480
+ if (request.rerank === void 0) {
24481
+ const loadingPhraseCode = loadPhraseCodeFor(request.term);
24482
+ if (loadingPhraseCode === null) return runQuery(request, callerContext);
24483
+ return loadingPhraseCode.then(() => runQuery(request, callerContext));
24484
+ }
24485
+ return import('./runtime-3X7ZJ5T3.mjs').then(
23935
24486
  (runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
23936
24487
  );
23937
24488
  }
@@ -23958,6 +24509,7 @@ async function runQuery(request, callerContext) {
23958
24509
  }
23959
24510
  const params = validQueryParams(request, config);
23960
24511
  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
24512
+ if (!isVectorOnly) requireUsablePhrases(params, context);
23961
24513
  const projection = resolveProjection(params.document);
23962
24514
  let paginated;
23963
24515
  let nextCursor;
@@ -23988,7 +24540,8 @@ async function runQuery(request, callerContext) {
23988
24540
  params,
23989
24541
  broadcastStatsForWorker(params, context, scoring),
23990
24542
  context.partitionIds,
23991
- context.patternWork
24543
+ context.patternWork,
24544
+ bm25SettingFor(params, context)
23992
24545
  ) : null;
23993
24546
  if (workerResult) {
23994
24547
  fanOutResult = workerResult;
@@ -23999,7 +24552,7 @@ async function runQuery(request, callerContext) {
23999
24552
  language,
24000
24553
  config.schema,
24001
24554
  scoring,
24002
- searchOptionsFor(manager, context.patternWork)
24555
+ searchOptionsFor(params, context)
24003
24556
  );
24004
24557
  }
24005
24558
  }
@@ -24133,6 +24686,9 @@ async function executePreflight(request, callerContext) {
24133
24686
  const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
24134
24687
  const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
24135
24688
  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
24689
+ const loadingPhraseCode = loadPhraseCodeFor(params.term);
24690
+ if (loadingPhraseCode !== null) await loadingPhraseCode;
24691
+ if (!isVectorOnly) requireUsablePhrases(params, context);
24136
24692
  const requestedVectorField = params.vector?.field;
24137
24693
  const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
24138
24694
  let totalMatched;
@@ -24149,7 +24705,7 @@ async function executePreflight(request, callerContext) {
24149
24705
  countExact = result.matchedExact !== false;
24150
24706
  } else if (countsWithoutScores(params)) {
24151
24707
  totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
24152
- searchOptions: searchOptionsFor(manager, context.patternWork),
24708
+ searchOptions: searchOptionsFor(params, context),
24153
24709
  partitionIds: context.partitionIds
24154
24710
  });
24155
24711
  } else {
@@ -24159,7 +24715,8 @@ async function executePreflight(request, callerContext) {
24159
24715
  params,
24160
24716
  broadcastStatsForWorker(params, context, scoring),
24161
24717
  context.partitionIds,
24162
- context.patternWork
24718
+ context.patternWork,
24719
+ bm25SettingFor(params, context)
24163
24720
  ) : null;
24164
24721
  if (workerResult) {
24165
24722
  totalMatched = workerResult.totalMatched;
@@ -24170,7 +24727,7 @@ async function executePreflight(request, callerContext) {
24170
24727
  language,
24171
24728
  config.schema,
24172
24729
  scoring,
24173
- searchOptionsFor(manager, context.patternWork)
24730
+ searchOptionsFor(params, context)
24174
24731
  );
24175
24732
  totalMatched = fanOutResult.totalMatched;
24176
24733
  }
@@ -24292,6 +24849,12 @@ async function readUnderOneAnalysis(core, indexName, read) {
24292
24849
  }
24293
24850
  return { result, analysisStale };
24294
24851
  }
24852
+ function phraseReadinessOf(core, indexName, config) {
24853
+ return () => {
24854
+ if (!core.analysisRebuild.isStale(indexName) || !holdsStringArrayField(config.schema)) return;
24855
+ requireLoadedPhraseCode().refusePhraseUntilRebuilt(indexName, core.analysisRebuild.rebuildMode);
24856
+ };
24857
+ }
24295
24858
  function scopedParams(params, options) {
24296
24859
  {
24297
24860
  return params;
@@ -24312,7 +24875,7 @@ async function runEngineQuery(core, indexName, params, options) {
24312
24875
  const manager = core.requireManager(indexName);
24313
24876
  const scoped = scopedParams(params, options);
24314
24877
  await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
24315
- const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-BX6XRFYR.mjs');
24878
+ const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-3X7ZJ5T3.mjs');
24316
24879
  const resolvedParams = await resolveVectorText(
24317
24880
  rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
24318
24881
  entry.embeddingAdapter,
@@ -24329,9 +24892,11 @@ async function runEngineQuery(core, indexName, params, options) {
24329
24892
  broadcastStats: broadcastStatsFor(core, options),
24330
24893
  partitionIds: options?.partitionIds,
24331
24894
  rescoreWindow: options?.rescoreWindow,
24895
+ indexBm25: options?.indexBm25,
24332
24896
  cursorBinding: lazyQueryBinding(params),
24333
24897
  patternWorkCap: core.patternWorkCap,
24334
- rerank: rerank?.rerankRunnerFor(core)
24898
+ rerank: rerank?.rerankRunnerFor(core),
24899
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24335
24900
  };
24336
24901
  const { result, analysisStale } = await readUnderOneAnalysis(
24337
24902
  core,
@@ -24378,8 +24943,10 @@ async function runEnginePreflight(core, indexName, params, options) {
24378
24943
  indexName,
24379
24944
  broadcastStats: broadcastStatsFor(core, options),
24380
24945
  partitionIds: options?.partitionIds,
24946
+ indexBm25: options?.indexBm25,
24381
24947
  cursorBinding: lazyQueryBinding(params),
24382
- patternWorkCap: core.patternWorkCap
24948
+ patternWorkCap: core.patternWorkCap,
24949
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24383
24950
  };
24384
24951
  const { result, analysisStale } = await readUnderOneAnalysis(
24385
24952
  core,
@@ -24462,7 +25029,7 @@ function createNarsilFromCore(core, config) {
24462
25029
  mutationCtx,
24463
25030
  rebalanceCtx
24464
25031
  } = core;
24465
- async function withOpenIndex(indexName, action) {
25032
+ async function withOpenIndex2(indexName, action) {
24466
25033
  guardShutdown();
24467
25034
  requireIndex(indexName);
24468
25035
  const release = await core.indexState.acquire(indexName);
@@ -24473,7 +25040,7 @@ function createNarsilFromCore(core, config) {
24473
25040
  }
24474
25041
  }
24475
25042
  function withOpenIndexWrite(indexName, action) {
24476
- return emitWriteEventAfter(core, indexName, withOpenIndex(indexName, action));
25043
+ return emitWriteEventAfter(core, indexName, withOpenIndex2(indexName, action));
24477
25044
  }
24478
25045
  const narsil = {
24479
25046
  createIndex(name, indexConfig) {
@@ -24559,12 +25126,12 @@ function createNarsilFromCore(core, config) {
24559
25126
  return orchestrator.awaitWrites(indexName);
24560
25127
  },
24561
25128
  async get(indexName, docId) {
24562
- return withOpenIndex(indexName, () => executor.execute({ type: "get", indexName, docId, requestId: docId }));
25129
+ return withOpenIndex2(indexName, () => executor.execute({ type: "get", indexName, docId, requestId: docId }));
24563
25130
  },
24564
25131
  async getMultiple(indexName, docIds) {
24565
25132
  guardShutdown();
24566
25133
  requireIndex(indexName);
24567
- return withOpenIndex(indexName, async () => {
25134
+ return withOpenIndex2(indexName, async () => {
24568
25135
  const result = /* @__PURE__ */ new Map();
24569
25136
  for (const docId of docIds) {
24570
25137
  const doc = await executor.execute({
@@ -24579,10 +25146,10 @@ function createNarsilFromCore(core, config) {
24579
25146
  });
24580
25147
  },
24581
25148
  async has(indexName, docId) {
24582
- return withOpenIndex(indexName, () => executor.execute({ type: "has", indexName, docId, requestId: docId }));
25149
+ return withOpenIndex2(indexName, () => executor.execute({ type: "has", indexName, docId, requestId: docId }));
24583
25150
  },
24584
25151
  async countDocuments(indexName) {
24585
- return withOpenIndex(indexName, () => executor.execute({ type: "count", indexName, requestId: indexName }));
25152
+ return withOpenIndex2(indexName, () => executor.execute({ type: "count", indexName, requestId: indexName }));
24586
25153
  },
24587
25154
  async listDocuments(indexName, params) {
24588
25155
  return runEngineListDocuments(core, indexName, params ?? {});
@@ -24603,10 +25170,11 @@ function createNarsilFromCore(core, config) {
24603
25170
  },
24604
25171
  ...portabilityOperations(core, config, () => narsil),
24605
25172
  ...synonymOperations(core),
25173
+ ...bm25Operations(core),
24606
25174
  async checkpoint(indexName) {
24607
25175
  guardShutdown();
24608
25176
  requireIndex(indexName);
24609
- await withOpenIndex(indexName, async () => {
25177
+ await withOpenIndex2(indexName, async () => {
24610
25178
  if (durability) await durability.manager.checkpoint(indexName);
24611
25179
  });
24612
25180
  },
@@ -24629,7 +25197,7 @@ function createNarsilFromCore(core, config) {
24629
25197
  },
24630
25198
  async updatePartitionConfig(indexName, partitionConfig) {
24631
25199
  guardShutdown();
24632
- return withOpenIndex(indexName, async () => {
25200
+ return withOpenIndex2(indexName, async () => {
24633
25201
  const entry = requireIndex(indexName);
24634
25202
  const manager = requireManager(indexName);
24635
25203
  if (rebalancingIndexes.has(indexName)) {
@@ -24744,4 +25312,4 @@ async function createNarsil(config) {
24744
25312
  // src/index.ts
24745
25313
  var VERSION = "0.1.0";
24746
25314
 
24747
- export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
25315
+ export { ARRAY_POSITION_GAP, BYTES_TAG, DATE_TAG, DEFAULT_BM25_B, DEFAULT_BM25_K1, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, storedArrayPositionGap, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };