@delali/narsil 0.3.1-next.90 → 0.3.1-next.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-66KYAOIK.mjs} +1 -1
  2. package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-B6XE7KXY.mjs} +11 -3
  3. package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-NOUZ6RH3.mjs} +2219 -2005
  4. package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-F4OGQUSU.mjs} +3 -3
  5. package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-5EJ7I7QY.mjs} +12 -6
  6. package/dist/browser/{write-6SBYTS62.mjs → write-K5V4XQ72.mjs} +2 -2
  7. package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-CvExBThq.d.ts} +11 -3
  8. package/dist/{chunk-MWJIAREY.mjs → chunk-3RDS4UHC.mjs} +2 -2
  9. package/dist/{chunk-QAMUC2BJ.mjs → chunk-3TTYK5A7.mjs} +7 -6
  10. package/dist/{chunk-URGLSNKR.mjs → chunk-43ORORN6.mjs} +201 -15
  11. package/dist/{chunk-A3UB2MXL.mjs → chunk-5ER245FI.mjs} +2 -505
  12. package/dist/{chunk-VY4DXZ5P.mjs → chunk-67Z2V6TC.mjs} +1 -1
  13. package/dist/{chunk-EMXAYPTQ.mjs → chunk-6VZ3Q73N.mjs} +2265 -3461
  14. package/dist/{chunk-S4LQDSUL.mjs → chunk-7OQHSDK6.mjs} +3 -3
  15. package/dist/chunk-7X5JST5L.mjs +114 -0
  16. package/dist/{chunk-R6BMMQXH.mjs → chunk-A4G7JSYO.mjs} +106 -41
  17. package/dist/{chunk-DCK4P5BI.mjs → chunk-BJ2YKRNX.mjs} +16 -7
  18. package/dist/{chunk-J2BPXTD7.mjs → chunk-CO4MRKST.mjs} +1 -1
  19. package/dist/{chunk-CL4OM7GB.mjs → chunk-EONIHG3V.mjs} +8 -4
  20. package/dist/chunk-MKWUXWQH.mjs +35 -0
  21. package/dist/chunk-NVL4ZR3K.mjs +426 -0
  22. package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
  23. package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
  24. package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
  25. package/dist/{chunk-V7YUSKKZ.mjs → chunk-T4DB5YFB.mjs} +7 -3
  26. package/dist/{chunk-XSZVS4S3.mjs → chunk-UXQE2ROB.mjs} +6 -2
  27. package/dist/chunk-X5CV2E2B.mjs +1 -0
  28. package/dist/chunk-XLJ2S2BH.mjs +3572 -0
  29. package/dist/chunk-ZIJ63TBT.mjs +507 -0
  30. package/dist/{client-BMK53IR2.d.ts → client-Cc42b42j.d.ts} +27 -3
  31. package/dist/client.d.ts +9 -8
  32. package/dist/client.mjs +15 -2
  33. package/dist/{config-BUR-7eVQ.d.ts → config-BYne-yCI.d.ts} +2 -2
  34. package/dist/{config-UJx1dyX1.d.ts → config-DyRowvVP.d.ts} +2 -2
  35. package/dist/{direct-executor-EGESD26S.mjs → direct-executor-CHXVCBN7.mjs} +9 -5
  36. package/dist/distribution/coordinator/etcd.d.ts +3 -2
  37. package/dist/distribution/coordinator/etcd.mjs +4 -0
  38. package/dist/distribution/coordinator/in-memory.d.ts +3 -2
  39. package/dist/distribution/coordinator/in-memory.mjs +4 -0
  40. package/dist/distribution/transport/grpc.mjs +2 -2
  41. package/dist/distribution/transport/in-memory.mjs +1 -1
  42. package/dist/distribution/transport/tcp.mjs +2 -2
  43. package/dist/distribution.d.ts +43 -9
  44. package/dist/distribution.mjs +4590 -4263
  45. package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-6AN4QTT3.mjs} +20 -16
  46. package/dist/{engine-9X6W5naj.d.ts → engine-96_kLtSY.d.ts} +43 -7
  47. package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
  48. package/dist/index.browser.mjs +1 -1
  49. package/dist/index.d.ts +44 -25
  50. package/dist/index.mjs +13 -9
  51. package/dist/{memory-CV9NAxLY.d.ts → memory-dd3GESdc.d.ts} +3 -3
  52. package/dist/partial-words.mjs +2 -1
  53. package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
  54. package/dist/persistence/durability/checkpoint-worker.mjs +18 -14
  55. package/dist/react.d.ts +8 -7
  56. package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
  57. package/dist/server.d.ts +11 -10
  58. package/dist/server.mjs +33 -6
  59. package/dist/{snapshot-22CUEYYW.mjs → snapshot-JFZVOGZV.mjs} +28 -18
  60. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  61. package/dist/synonyms.d.ts +22 -0
  62. package/dist/synonyms.mjs +12 -0
  63. package/dist/{tasks-BGaZJyoY.d.ts → tasks-DA1Ff0G4.d.ts} +4 -1
  64. package/dist/{thread-ZP3UMIZT.mjs → thread-VTDLLYXC.mjs} +7 -5
  65. package/dist/vector/search-worker.d.ts +2 -1
  66. package/dist/workers/entry.mjs +4 -4
  67. package/dist/{write-PXDKPG2R.mjs → write-OJR3H4BI.mjs} +14 -10
  68. package/package.json +15 -7
  69. package/dist/chunk-PR4O7CAP.mjs +0 -18
  70. package/dist/chunk-X4SEXL4U.mjs +0 -2310
@@ -1,4 +1,4 @@
1
- import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, toReducedSortValue, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, resolveRequestThreadCount, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, defaultSortMode, holdsUnpairedSurrogate, MIN_SWEEP_INTERVAL_MS, MAX_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_INDEX_CONFIGURATION_BYTES, compareComparableValues, readSortField, exceedsCodePoints, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, compareComparableKeys, sortModeOf, toComparableSortValue, DEFAULT_FUZZY_PREFIX_LENGTH, SORT_VALUE_MAX_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, truncateSortString, MAX_IMPORT_REFUSALS_REPORTED, MIN_OVERSAMPLE, DEFAULT_MAX_PER_GROUP, compareSortValues, isSortMode, SORT_MODES, MIN_DOCUMENTS_FOR_SEGMENTS, SHUTDOWN_TIMEOUT_MS, DEFAULT_PAGE_SIZE, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-FHFFWLEK.mjs';
1
+ import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, toReducedSortValue, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, resolveRequestThreadCount, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, defaultSortMode, holdsUnpairedSurrogate, MIN_SWEEP_INTERVAL_MS, MAX_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_INDEX_CONFIGURATION_BYTES, compareComparableValues, readSortField, exceedsCodePoints, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, compareComparableKeys, sortModeOf, toComparableSortValue, DEFAULT_FUZZY_PREFIX_LENGTH, SORT_VALUE_MAX_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, truncateSortString, MAX_IMPORT_REFUSALS_REPORTED, MIN_OVERSAMPLE, DEFAULT_MAX_PER_GROUP, compareSortValues, isSortMode, SORT_MODES, MIN_DOCUMENTS_FOR_SEGMENTS, SHUTDOWN_TIMEOUT_MS, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-FHFFWLEK.mjs';
2
2
  import { Decoder, encode, decode } from '@msgpack/msgpack';
3
3
 
4
4
  // src/analysis/registry.ts
@@ -503,6 +503,42 @@ function requirePatternSearch(fieldPath) {
503
503
  );
504
504
  }
505
505
 
506
+ // src/core/synonyms/platform.browser.ts
507
+ function platformSynonymSearch() {
508
+ return null;
509
+ }
510
+
511
+ // src/core/synonyms/registry.ts
512
+ var registered3 = null;
513
+ var MODULE_FUNCTIONS3 = ["prepare", "storedLookup", "decodeRules", "replace", "applyLoaded"];
514
+ function isSynonymSearchModule(value) {
515
+ if (value === null || typeof value !== "object" || value.name !== "synonyms") return false;
516
+ return MODULE_FUNCTIONS3.every((name) => typeof Reflect.get(value, name) === "function");
517
+ }
518
+ function registerSynonyms(module) {
519
+ if (!isSynonymSearchModule(module)) {
520
+ throw new NarsilError(
521
+ ErrorCodes.CONFIG_INVALID,
522
+ 'Pass registerSynonyms the synonyms object that "@delali/narsil/synonyms" exports'
523
+ );
524
+ }
525
+ registered3 = module;
526
+ }
527
+ function availableSynonymModule() {
528
+ return registered3 ?? platformSynonymSearch();
529
+ }
530
+ function synonymModule() {
531
+ const module = availableSynonymModule();
532
+ if (module !== null) return module;
533
+ throw new NarsilError(
534
+ ErrorCodes.CONFIG_INVALID,
535
+ "This bundle holds no synonym code, so import { synonyms } from '@delali/narsil/synonyms' and pass it to registerSynonyms before you use a synonym set"
536
+ );
537
+ }
538
+ function isEmptySynonymSet(value) {
539
+ return value === void 0 || Array.isArray(value) && value.length === 0;
540
+ }
541
+
506
542
  // src/core/tokenizer/constants.ts
507
543
  var TOKEN_CACHE_SIZE_FLOOR = 5e4;
508
544
  var TOKEN_CACHE_SIZE_CEILING = 2e6;
@@ -7467,19 +7503,22 @@ function termGroupOf(groups, token) {
7467
7503
  function typoMatchesFor(index, token, group, exact, tolerance, prefixLength) {
7468
7504
  if (group !== void 0) {
7469
7505
  if (!group.typoMatches || exact || tolerance === 0) return [];
7470
- return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token);
7506
+ return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token && group.synonyms?.has(match.token) !== true);
7471
7507
  }
7472
7508
  if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
7473
7509
  const postingList = index.lookup(token);
7474
7510
  return postingList ? [{ token, postingList }] : [];
7475
7511
  }
7476
7512
  function resolveGroupMatches(context, group) {
7477
- let groupDocumentFrequency = 0;
7513
+ const frequencyOf = (token) => context.globalFrequency(token) ?? context.index.documentFrequency(token);
7514
+ const lookup = (token) => context.index.lookup(token);
7515
+ const synonyms = group.synonyms;
7516
+ let groupDocumentFrequency = synonyms?.documentFrequency(frequencyOf) ?? 0;
7478
7517
  const found = [];
7479
7518
  for (const member of group.members) {
7480
- const documents = context.globalFrequency(member.token) ?? context.index.documentFrequency(member.token);
7519
+ const documents = frequencyOf(member.token);
7481
7520
  if (documents > groupDocumentFrequency) groupDocumentFrequency = documents;
7482
- const postingList = context.index.lookup(member.token);
7521
+ const postingList = synonyms !== void 0 && member.token === group.token ? synonyms.postings(lookup, member.token) : lookup(member.token);
7483
7522
  if (postingList === void 0) continue;
7484
7523
  found.push({ token: member.token, factor: member.factor, partialFactor: member.partialFactor, postingList });
7485
7524
  }
@@ -7586,6 +7625,10 @@ function searchFulltextMatches(state, params) {
7586
7625
  if (postingList === void 0) continue;
7587
7626
  markPostingList(matched, postingList, member.factor > 0 ? searched : searchedPartial, filterBitset);
7588
7627
  }
7628
+ for (const synonym of group.synonyms?.tokens ?? []) {
7629
+ const postingList = state.invertedIdx.lookup(synonym);
7630
+ if (postingList !== void 0) markPostingList(matched, postingList, searched, filterBitset);
7631
+ }
7589
7632
  }
7590
7633
  const docStore = state.docStore;
7591
7634
  const resolver = docStore.resolver();
@@ -9730,205 +9773,253 @@ function readHeapStatistics() {
9730
9773
  return null;
9731
9774
  }
9732
9775
 
9733
- // src/core/pattern-index/work-meter.ts
9734
- var COUNTER_BYTES = 8;
9735
- function capExceeded(cap) {
9736
- return new NarsilError(
9737
- ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
9738
- `The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
9739
- { cap }
9740
- );
9741
- }
9742
- function localMeter(cap, spent, matcherStateBytes, flushUnits) {
9743
- let total = spent;
9744
- let sharedCount = null;
9745
- const spentSoFar = () => sharedCount === null ? total : Number(Atomics.load(sharedCount, 0));
9746
- const meter = {
9747
- cap,
9748
- matcherStateBytes,
9749
- flushUnits,
9750
- add(units) {
9751
- if (units <= 0) return;
9752
- if (sharedCount === null) total += units;
9753
- else total = Number(Atomics.add(sharedCount, 0, BigInt(units))) + units;
9754
- if (total > cap) throw capExceeded(cap);
9755
- },
9756
- remaining() {
9757
- return cap - spentSoFar();
9758
- },
9759
- fork(workerCount) {
9760
- const copies = Math.max(1, workerCount);
9761
- const seed = spentSoFar();
9762
- let counter = null;
9763
- if (typeof SharedArrayBuffer === "function") {
9764
- counter = new SharedArrayBuffer(COUNTER_BYTES);
9765
- new BigInt64Array(counter)[0] = BigInt(seed);
9766
- }
9767
- const shared2 = {
9768
- cap,
9769
- spent: seed,
9770
- counter,
9771
- matcherStateBytes: Math.floor(matcherStateBytes / copies),
9772
- flushUnits: Math.max(1, Math.floor(flushUnits / copies))
9773
- };
9774
- return {
9775
- shared: shared2,
9776
- settle(reportedUnits) {
9777
- if (counter === null) {
9778
- meter.add(reportedUnits);
9779
- return;
9780
- }
9781
- const unitsSinceFork = spentSoFar() - seed;
9782
- const view = new BigInt64Array(counter);
9783
- if (unitsSinceFork > 0) Atomics.add(view, 0, BigInt(unitsSinceFork));
9784
- sharedCount = view;
9785
- total = spentSoFar();
9786
- if (total > cap) throw capExceeded(cap);
9787
- }
9788
- };
9789
- }
9790
- };
9791
- return meter;
9792
- }
9793
- function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP) {
9794
- return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS);
9795
- }
9796
- function sharedMeter(work, counter) {
9797
- const view = new BigInt64Array(counter);
9798
- return {
9799
- matcherStateBytes: work.matcherStateBytes,
9800
- flushUnits: work.flushUnits,
9801
- add(units) {
9802
- if (units <= 0) return;
9803
- const total = Number(Atomics.add(view, 0, BigInt(units))) + units;
9804
- if (total > work.cap) throw capExceeded(work.cap);
9805
- },
9806
- remaining() {
9807
- return work.cap - Number(Atomics.load(view, 0));
9808
- }
9809
- };
9810
- }
9811
- function joinPatternWorkMeter(work) {
9812
- const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits) : sharedMeter(work, work.counter);
9813
- let spent = 0;
9814
- return {
9815
- matcherStateBytes: meter.matcherStateBytes,
9816
- flushUnits: meter.flushUnits,
9817
- add(units) {
9818
- if (units <= 0) return;
9819
- spent += units;
9820
- meter.add(units);
9821
- },
9822
- remaining: () => meter.remaining(),
9823
- spent: () => spent
9824
- };
9776
+ // src/search/oversample.ts
9777
+ function oversampledShardSize(size) {
9778
+ return Math.ceil(size * 1.5) + 10;
9825
9779
  }
9826
9780
 
9827
- // src/partitioning/distributed-scoring.ts
9828
- function analysedQueryTokens(term, language, analysis) {
9829
- const { tokens } = tokenize(term, language, {
9830
- stem: true,
9831
- removeStopWords: true,
9832
- customTokenizer: analysis.customTokenizer,
9833
- stopWordOverride: analysis.stopWords
9834
- });
9835
- const unique = /* @__PURE__ */ new Set();
9836
- for (const { token } of tokens) {
9837
- unique.add(token);
9838
- }
9839
- return unique;
9781
+ // src/search/facets.ts
9782
+ function keptValueCount(fieldConfig) {
9783
+ return keptFacetValues(fieldConfig?.limit);
9840
9784
  }
9841
- function copyNumericRecord(source) {
9842
- const copy = /* @__PURE__ */ Object.create(null);
9843
- for (const [key, value] of Object.entries(source)) {
9844
- if (typeof value === "number" && Number.isFinite(value)) {
9845
- copy[key] = value;
9846
- }
9785
+ function everyValueFacetConfig(config) {
9786
+ const widened = {};
9787
+ for (const [field, fieldConfig] of Object.entries(config)) {
9788
+ widened[field] = { ...fieldConfig, limit: void 0 };
9847
9789
  }
9848
- return copy;
9849
- }
9850
- function sanitizeGlobalStats(stats) {
9851
- return {
9852
- totalDocuments: typeof stats.totalDocuments === "number" && Number.isFinite(stats.totalDocuments) ? stats.totalDocuments : 0,
9853
- docFrequencies: copyNumericRecord(stats.docFrequencies),
9854
- totalFieldLengths: copyNumericRecord(stats.totalFieldLengths),
9855
- averageFieldLengths: copyNumericRecord(stats.averageFieldLengths)
9856
- };
9790
+ return widened;
9857
9791
  }
9858
- function pruneStatsToQueryTerms(stats, term, language, analysis) {
9859
- const docFrequencies = /* @__PURE__ */ Object.create(null);
9860
- for (const token of analysedQueryTokens(term, language, analysis)) {
9861
- if (Object.hasOwn(stats.docFrequencies, token)) {
9862
- docFrequencies[token] = stats.docFrequencies[token];
9792
+ function oversampledFacetConfig(config) {
9793
+ const widened = {};
9794
+ for (const [field, fieldConfig] of Object.entries(config)) {
9795
+ if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
9796
+ widened[field] = { ...fieldConfig, limit: void 0 };
9797
+ continue;
9863
9798
  }
9799
+ const kept = keptValueCount(fieldConfig);
9800
+ widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
9864
9801
  }
9865
- return {
9866
- totalDocuments: stats.totalDocuments,
9867
- docFrequencies,
9868
- totalFieldLengths: stats.totalFieldLengths,
9869
- averageFieldLengths: stats.averageFieldLengths
9870
- };
9802
+ return widened;
9871
9803
  }
9872
- function collectQueryTermStats(manager, term, language, analysis, partitionIds) {
9873
- const queryTokens = analysedQueryTokens(term, language, analysis);
9874
- let totalDocuments = 0;
9875
- const docFrequencies = /* @__PURE__ */ Object.create(null);
9876
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
9877
- const statsPartitions = partitionIds === void 0 ? manager.getAllPartitions() : partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
9878
- for (const partition of statsPartitions) {
9879
- const stats = partition.stats;
9880
- totalDocuments += stats.totalDocuments;
9881
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
9882
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
9883
- }
9884
- for (const token of queryTokens) {
9885
- if (Object.hasOwn(stats.docFrequencies, token)) {
9886
- docFrequencies[token] = (docFrequencies[token] ?? 0) + stats.docFrequencies[token];
9804
+ function mergeFacets(partitionFacets, config) {
9805
+ const merged = /* @__PURE__ */ new Map();
9806
+ const bounds = /* @__PURE__ */ new Map();
9807
+ for (const partition of partitionFacets) {
9808
+ for (const [field, facetResult] of Object.entries(partition)) {
9809
+ let fieldMap = merged.get(field);
9810
+ if (!fieldMap) {
9811
+ fieldMap = /* @__PURE__ */ new Map();
9812
+ merged.set(field, fieldMap);
9813
+ }
9814
+ bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
9815
+ for (const [value, count] of Object.entries(facetResult.values)) {
9816
+ fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
9887
9817
  }
9888
9818
  }
9889
9819
  }
9890
- const averageFieldLengths = /* @__PURE__ */ Object.create(null);
9891
- if (totalDocuments > 0) {
9892
- for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
9893
- averageFieldLengths[field] = totalLength / totalDocuments;
9894
- }
9895
- }
9896
- return { totalDocuments, docFrequencies, totalFieldLengths, averageFieldLengths };
9897
- }
9898
- function mergePartitionStats(statsArray) {
9899
- let totalDocuments = 0;
9900
- const docFrequencies = /* @__PURE__ */ Object.create(null);
9901
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
9902
- for (const stats of statsArray) {
9903
- totalDocuments += stats.totalDocuments;
9904
- for (const [term, freq] of Object.entries(stats.docFrequencies)) {
9905
- docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
9906
- }
9907
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
9908
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
9820
+ const result = {};
9821
+ for (const [field, valueMap] of merged) {
9822
+ const fieldConfig = config[field];
9823
+ const ascending = fieldConfig?.sort === "asc";
9824
+ const ordered = Array.from(valueMap.entries());
9825
+ ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
9826
+ const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
9827
+ let largestLeftOut = 0;
9828
+ for (let index = kept; index < ordered.length; index++) {
9829
+ if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
9909
9830
  }
9910
- }
9911
- const averageFieldLengths = /* @__PURE__ */ Object.create(null);
9912
- if (totalDocuments > 0) {
9913
- for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
9914
- averageFieldLengths[field] = totalLength / totalDocuments;
9831
+ const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
9832
+ const values = {};
9833
+ for (let index = 0; index < kept; index++) {
9834
+ values[ordered[index][0]] = ordered[index][1];
9915
9835
  }
9836
+ result[field] = { values, count: kept, errorBound };
9916
9837
  }
9917
- return {
9918
- totalDocuments,
9919
- docFrequencies,
9920
- totalFieldLengths,
9921
- averageFieldLengths
9922
- };
9838
+ return result;
9923
9839
  }
9924
9840
 
9925
- // src/core/partition/scored-merge.ts
9926
- function kWayMerge(arrays) {
9927
- const nonEmpty = arrays.filter((a) => a.length > 0);
9928
- if (nonEmpty.length === 0) return [];
9929
- if (nonEmpty.length === 1) return nonEmpty[0];
9930
- if (nonEmpty.length <= 4) {
9931
- return sequentialMerge(nonEmpty);
9841
+ // src/core/partition/frozen/memory.ts
9842
+ function frozenSegmentBytes(segment) {
9843
+ const arrays = segment.arrays;
9844
+ let bytes = arrays.tokenTable.bytes + arrays.idTable.bytes;
9845
+ bytes += arrays.postingOffsets.byteLength;
9846
+ bytes += arrays.postingDocIds.byteLength;
9847
+ bytes += arrays.postingFrequencies.byteLength;
9848
+ bytes += arrays.postingFieldIndices.byteLength;
9849
+ if (arrays.positionOffsets !== null) bytes += arrays.positionOffsets.byteLength;
9850
+ if (arrays.positionValues !== null) bytes += arrays.positionValues.byteLength;
9851
+ if (arrays.documentTable !== null) {
9852
+ bytes += arrays.documentTable.blob.byteLength + arrays.documentTable.offsets.byteLength;
9853
+ }
9854
+ for (const column of arrays.fieldLengthColumns) bytes += column.byteLength;
9855
+ for (const entry of [...arrays.numeric, ...arrays.sizes]) bytes += entry.docIds.byteLength + entry.values.byteLength;
9856
+ for (const entry of arrays.boolean) bytes += entry.trueDocs.byteLength + entry.falseDocs.byteLength;
9857
+ for (const entry of arrays.enums) bytes += entry.offsets.byteLength + entry.docIds.byteLength;
9858
+ for (const entry of arrays.geo) {
9859
+ bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
9860
+ }
9861
+ for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
9862
+ return bytes + segment.partialWords.bytes() + segment.removalBytes();
9863
+ }
9864
+
9865
+ // src/core/partition/composite/compaction.ts
9866
+ function resolveFrozenSegments(frozen, segmentIds, partitionId) {
9867
+ const byId = new Map(frozen.map((segment) => [segment.segmentId, segment]));
9868
+ return segmentIds.map((segmentId) => {
9869
+ const segment = byId.get(segmentId);
9870
+ if (segment === void 0) {
9871
+ throw new NarsilError(
9872
+ ErrorCodes.PARTITION_CORRUPTED,
9873
+ `Partition ${partitionId} does not hold frozen segment "${segmentId}"`,
9874
+ { partitionId, segmentId }
9875
+ );
9876
+ }
9877
+ return segment;
9878
+ });
9879
+ }
9880
+ function survivorDocumentsOf(sub, payload) {
9881
+ return payload.docIds.map((docId) => {
9882
+ const stored = sub.docStore.get(docId);
9883
+ if (stored === void 0) {
9884
+ throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while compacting a segment`, {
9885
+ docId
9886
+ });
9887
+ }
9888
+ return stored.fields;
9889
+ });
9890
+ }
9891
+ function freezeLiveTailInto(live, liveState, frozen, freeze) {
9892
+ if (live.count() === 0) return null;
9893
+ const payload = live.encodeSegment();
9894
+ const segment = freeze(payload, survivorDocumentsOf(liveState, payload));
9895
+ if (segment === null) return null;
9896
+ live.clear();
9897
+ frozen.push(segment);
9898
+ return segment;
9899
+ }
9900
+ function swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings) {
9901
+ const dropSet = new Set(dropSegmentIds);
9902
+ const dropped = frozen.filter((segment) => dropSet.has(segment.segmentId));
9903
+ const applied = /* @__PURE__ */ new Map();
9904
+ for (const segment of dropped) {
9905
+ for (const docId of segment.tombstonedDocIds()) {
9906
+ if (!replacement.hasDocument(docId)) continue;
9907
+ if (dropped.some((other) => other.hasDocument(docId))) continue;
9908
+ const spellings = removedSpellings?.get(docId) ?? replacement.removedSpellingsOf(docId);
9909
+ if (spellings !== void 0) applied.set(docId, spellings);
9910
+ replacement.tombstoneDocument(docId, spellings);
9911
+ }
9912
+ }
9913
+ const kept = frozen.filter((segment) => !dropSet.has(segment.segmentId));
9914
+ frozen.length = 0;
9915
+ frozen.push(...kept, replacement);
9916
+ return applied;
9917
+ }
9918
+
9919
+ // src/core/partition/composite/documents.ts
9920
+ function compositeDocument(frozen, live, docId, projection) {
9921
+ for (const segment of frozen) {
9922
+ const projected = segment.docStore.projectedDocument(docId, projection);
9923
+ if (projected !== void 0) return projected;
9924
+ }
9925
+ return live.get(docId, projection);
9926
+ }
9927
+ function compositeFieldValues(frozen, live, docId, fieldPaths) {
9928
+ for (const segment of frozen) {
9929
+ const internalId = segment.docStore.getInternalId(docId);
9930
+ if (internalId === void 0) continue;
9931
+ return fieldPaths.map((fieldPath) => segment.docStore.fieldReader(fieldPath)(internalId));
9932
+ }
9933
+ return live.fieldValues(docId, fieldPaths);
9934
+ }
9935
+
9936
+ // src/core/partition/composite/filters.ts
9937
+ var WORD_BITS = 32;
9938
+ function alignToWord(value) {
9939
+ return value + WORD_BITS - 1 & -32;
9940
+ }
9941
+ function computeOrdinalLayout(subs) {
9942
+ const bases = new Array(subs.length);
9943
+ let cursor = 0;
9944
+ for (let i = 0; i < subs.length; i++) {
9945
+ bases[i] = cursor;
9946
+ cursor += alignToWord(subs[i].docStore.internalIdCapacity());
9947
+ }
9948
+ return { bases, totalCapacity: cursor };
9949
+ }
9950
+ function subBitsetView(composite, layout, subIndex) {
9951
+ const startWord = layout.bases[subIndex] >> 5;
9952
+ const endWord = subIndex + 1 < layout.bases.length ? layout.bases[subIndex + 1] >> 5 : Math.ceil(layout.totalCapacity / WORD_BITS);
9953
+ return composite.subarray(startWord, endWord);
9954
+ }
9955
+ function placeSubBitset(composite, layout, subIndex, subBits) {
9956
+ const words = subBitsetView(composite, layout, subIndex).length;
9957
+ if (words === 0) return;
9958
+ composite.set(subBits.length > words ? subBits.subarray(0, words) : subBits, layout.bases[subIndex] >> 5);
9959
+ }
9960
+ function compositeFiltersBitset(subs, layout, filters, schema, meter) {
9961
+ const composite = new Uint32Array(Math.ceil(layout.totalCapacity / WORD_BITS));
9962
+ for (let i = 0; i < subs.length; i++) {
9963
+ placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema, meter));
9964
+ }
9965
+ return composite;
9966
+ }
9967
+ function compositeFilters(subs, filters, schema, meter) {
9968
+ const combined = /* @__PURE__ */ new Set();
9969
+ for (const sub of subs) {
9970
+ for (const docId of applyPartitionFilters(sub, filters, schema, meter)) {
9971
+ combined.add(docId);
9972
+ }
9973
+ }
9974
+ return combined;
9975
+ }
9976
+ function compositeFilterMatches(subs, layout, filters, schema, meter) {
9977
+ const bits = compositeFiltersBitset(subs, layout, filters, schema, meter);
9978
+ let count = 0;
9979
+ for (let i = 0; i < subs.length; i++) {
9980
+ const resolver = subs[i].docStore.resolver();
9981
+ const view = subBitsetView(bits, layout, i);
9982
+ for (let wordIndex = 0; wordIndex < view.length; wordIndex++) {
9983
+ let word = view[wordIndex];
9984
+ if (word === 0) continue;
9985
+ const base = wordIndex << 5;
9986
+ while (word !== 0) {
9987
+ const trailingZeros = Math.clz32(word & -word) ^ 31;
9988
+ if (resolver.isLive(base + trailingZeros)) count++;
9989
+ word &= word - 1;
9990
+ }
9991
+ }
9992
+ }
9993
+ return {
9994
+ count,
9995
+ hasExternal(docId) {
9996
+ for (let i = 0; i < subs.length; i++) {
9997
+ const internalId = subs[i].docStore.getInternalId(docId);
9998
+ if (internalId !== void 0) return bitsetHas(bits, layout.bases[i] + internalId);
9999
+ }
10000
+ return false;
10001
+ },
10002
+ hasInternal(internalId) {
10003
+ return bitsetHas(bits, internalId);
10004
+ }
10005
+ };
10006
+ }
10007
+ function subMatchesView(matches, layout, subIndex) {
10008
+ const base = layout.bases[subIndex];
10009
+ return {
10010
+ count: matches.count,
10011
+ hasExternal: (docId) => matches.hasExternal(docId),
10012
+ hasInternal: (internalId) => matches.hasInternal(base + internalId)
10013
+ };
10014
+ }
10015
+
10016
+ // src/core/partition/scored-merge.ts
10017
+ function kWayMerge(arrays) {
10018
+ const nonEmpty = arrays.filter((a) => a.length > 0);
10019
+ if (nonEmpty.length === 0) return [];
10020
+ if (nonEmpty.length === 1) return nonEmpty[0];
10021
+ if (nonEmpty.length <= 4) {
10022
+ return sequentialMerge(nonEmpty);
9932
10023
  }
9933
10024
  return heapMerge(nonEmpty);
9934
10025
  }
@@ -10039,1921 +10130,1944 @@ function heapPop(heap) {
10039
10130
  return top;
10040
10131
  }
10041
10132
 
10042
- // src/search/oversample.ts
10043
- function oversampledShardSize(size) {
10044
- return Math.ceil(size * 1.5) + 10;
10045
- }
10046
-
10047
- // src/search/facets.ts
10048
- function keptValueCount(fieldConfig) {
10049
- return keptFacetValues(fieldConfig?.limit);
10133
+ // src/core/partition/composite/stats.ts
10134
+ function collectCandidateTerms(subs, params) {
10135
+ const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
10136
+ const terms = /* @__PURE__ */ new Set();
10137
+ for (const qt of queryTokens) {
10138
+ const group = termGroupOf(termGroups, qt.token);
10139
+ if (group !== void 0) {
10140
+ for (const member of group.members) terms.add(member.token);
10141
+ for (const synonym of group.synonyms?.tokens ?? []) terms.add(synonym);
10142
+ }
10143
+ if (group === void 0 && (exact || tolerance === 0)) {
10144
+ terms.add(qt.token);
10145
+ continue;
10146
+ }
10147
+ for (const sub of subs) {
10148
+ for (const match of typoMatchesFor(sub.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
10149
+ terms.add(match.token);
10150
+ }
10151
+ }
10152
+ }
10153
+ return terms;
10050
10154
  }
10051
- function everyValueFacetConfig(config) {
10052
- const widened = {};
10053
- for (const [field, fieldConfig] of Object.entries(config)) {
10054
- widened[field] = { ...fieldConfig, limit: void 0 };
10155
+ function resolveQueryTermFrequencies(subs, params) {
10156
+ const frequencies = /* @__PURE__ */ Object.create(null);
10157
+ for (const term of collectCandidateTerms(subs, params)) {
10158
+ const total = partitionDocumentFrequency(subs, term);
10159
+ if (total > 0) frequencies[term] = total;
10055
10160
  }
10056
- return widened;
10161
+ return frequencies;
10057
10162
  }
10058
- function oversampledFacetConfig(config) {
10059
- const widened = {};
10060
- for (const [field, fieldConfig] of Object.entries(config)) {
10061
- if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
10062
- widened[field] = { ...fieldConfig, limit: void 0 };
10063
- continue;
10163
+ function aggregateFieldStats(subs) {
10164
+ let totalDocuments = 0;
10165
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
10166
+ for (const sub of subs) {
10167
+ totalDocuments += sub.stats.totalDocuments;
10168
+ for (const [field, length] of Object.entries(sub.stats.totalFieldLengths)) {
10169
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
10064
10170
  }
10065
- const kept = keptValueCount(fieldConfig);
10066
- widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
10067
10171
  }
10068
- return widened;
10172
+ const averageFieldLengths = /* @__PURE__ */ Object.create(null);
10173
+ if (totalDocuments > 0) {
10174
+ for (const [field, length] of Object.entries(totalFieldLengths)) {
10175
+ averageFieldLengths[field] = length / totalDocuments;
10176
+ }
10177
+ }
10178
+ return { totalDocuments, totalFieldLengths, averageFieldLengths };
10069
10179
  }
10070
- function mergeFacets(partitionFacets, config) {
10071
- const merged = /* @__PURE__ */ new Map();
10072
- const bounds = /* @__PURE__ */ new Map();
10073
- for (const partition of partitionFacets) {
10074
- for (const [field, facetResult] of Object.entries(partition)) {
10075
- let fieldMap = merged.get(field);
10076
- if (!fieldMap) {
10077
- fieldMap = /* @__PURE__ */ new Map();
10078
- merged.set(field, fieldMap);
10079
- }
10080
- bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
10081
- for (const [value, count] of Object.entries(facetResult.values)) {
10082
- fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
10083
- }
10180
+ function mergeDocFrequencies(subs) {
10181
+ const merged = /* @__PURE__ */ Object.create(null);
10182
+ for (const sub of subs) {
10183
+ for (const [term, frequency] of Object.entries(sub.stats.docFrequencies)) {
10184
+ merged[term] = (merged[term] ?? 0) + frequency;
10084
10185
  }
10085
10186
  }
10086
- const result = {};
10087
- for (const [field, valueMap] of merged) {
10088
- const fieldConfig = config[field];
10089
- const ascending = fieldConfig?.sort === "asc";
10090
- const ordered = Array.from(valueMap.entries());
10091
- ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
10092
- const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
10093
- let largestLeftOut = 0;
10094
- for (let index = kept; index < ordered.length; index++) {
10095
- if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
10096
- }
10097
- const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
10098
- const values = {};
10099
- for (let index = 0; index < kept; index++) {
10100
- values[ordered[index][0]] = ordered[index][1];
10101
- }
10102
- result[field] = { values, count: kept, errorBound };
10103
- }
10104
- return result;
10187
+ return merged;
10105
10188
  }
10106
-
10107
- // src/search/cursor-codec.ts
10108
- function encodeCursorText(text) {
10109
- if (typeof Buffer !== "undefined") return Buffer.from(text, "utf-8").toString("base64");
10110
- const bytes = new TextEncoder().encode(text);
10111
- let binary = "";
10112
- for (const byte of bytes) binary += String.fromCharCode(byte);
10113
- return btoa(binary);
10189
+ function buildAggregateStatsView(subs, docFrequencies) {
10190
+ const fields = aggregateFieldStats(subs);
10191
+ return {
10192
+ ...fields,
10193
+ get docFrequencies() {
10194
+ return docFrequencies();
10195
+ }
10196
+ };
10114
10197
  }
10115
- function decodeCursorText(encoded) {
10116
- if (typeof Buffer !== "undefined") return Buffer.from(encoded, "base64").toString("utf-8");
10117
- const binary = atob(encoded);
10118
- const bytes = new Uint8Array(binary.length);
10119
- for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
10120
- return new TextDecoder().decode(bytes);
10198
+ function compositeQueryStats(subs, params) {
10199
+ const resolved = resolveQueryTermFrequencies(subs, params);
10200
+ const caller = params.globalStats;
10201
+ if (caller === void 0) {
10202
+ const fields = aggregateFieldStats(subs);
10203
+ return { ...fields, docFrequencies: resolved };
10204
+ }
10205
+ return {
10206
+ totalDocuments: caller.totalDocuments,
10207
+ totalFieldLengths: caller.totalFieldLengths,
10208
+ averageFieldLengths: caller.averageFieldLengths,
10209
+ docFrequencies: { ...resolved, ...caller.docFrequencies }
10210
+ };
10121
10211
  }
10122
10212
 
10123
- // src/search/sorting.ts
10124
- function isFieldList(sort) {
10125
- return Array.isArray(sort);
10213
+ // src/core/partition/composite/search.ts
10214
+ function subParams(params, layout, subIndex, globalStats) {
10215
+ return {
10216
+ ...params,
10217
+ globalStats,
10218
+ filterBitset: params.filterBitset === void 0 ? void 0 : subBitsetView(params.filterBitset, layout, subIndex)
10219
+ };
10126
10220
  }
10127
- function normalizeSort(sort) {
10128
- if (sort === void 0) return [];
10129
- if (isFieldList(sort)) {
10130
- return sort.map(
10131
- (entry) => entry.mode === void 0 ? { field: entry.field, direction: entry.direction } : { field: entry.field, direction: entry.direction, mode: entry.mode }
10132
- );
10221
+ function onlyPopulatedSub(subs) {
10222
+ let found = -1;
10223
+ for (let i = 0; i < subs.length; i++) {
10224
+ if (subs[i].stats.totalDocuments === 0) continue;
10225
+ if (found !== -1) return -1;
10226
+ found = i;
10133
10227
  }
10134
- return Object.entries(sort).map(([field, direction]) => ({ field, direction }));
10135
- }
10136
- function sortModesOf(fields) {
10137
- return fields.map(sortModeOf);
10228
+ return found;
10138
10229
  }
10139
- function sortSignatureEntry(entry) {
10140
- const mode = sortModeOf(entry);
10141
- return mode === defaultSortMode(entry.direction) ? [entry.field, entry.direction] : [entry.field, entry.direction, mode];
10230
+ function searchSingleSub(subs, layout, params, index) {
10231
+ const result = searchFulltext(subs[index], subParams(params, layout, index, params.globalStats));
10232
+ if (result.matchedOrdinalBitset === void 0) return result;
10233
+ const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
10234
+ placeSubBitset(matchedOrdinalBitset, layout, index, result.matchedOrdinalBitset);
10235
+ return { ...result, matchedOrdinalBitset };
10142
10236
  }
10143
- function requireSortMode(entry, fieldType) {
10144
- if (entry.mode === void 0) return;
10145
- if (!isSortMode(entry.mode)) {
10146
- throw new NarsilError(
10147
- ErrorCodes.SEARCH_INVALID_MODE,
10148
- `A sort mode is one of ${SORT_MODES.map((mode) => `"${mode}"`).join(", ")}, and the mode on "${entry.field}" is "${String(entry.mode)}"`,
10149
- { field: entry.field, mode: String(entry.mode) }
10150
- );
10237
+ function compositeSearchFulltext(subs, layout, params) {
10238
+ if (params.queryTokens.length === 0) {
10239
+ return { scored: [], totalMatched: 0 };
10151
10240
  }
10152
- const averages = entry.mode === "avg" || entry.mode === "median";
10153
- if (averages && fieldType !== void 0 && fieldType !== "number" && fieldType !== "number[]") {
10154
- throw new NarsilError(
10155
- ErrorCodes.SEARCH_INVALID_FIELD,
10156
- `The engine applies the "${entry.mode}" mode to a number field alone, and "${entry.field}" is a ${fieldType} field`,
10157
- { field: entry.field, fieldType, mode: entry.mode }
10158
- );
10241
+ const single = onlyPopulatedSub(subs);
10242
+ if (single !== -1) {
10243
+ return searchSingleSub(subs, layout, params, single);
10159
10244
  }
10160
- }
10161
- function requireSortableFields(sort, schema, strict = false) {
10162
- const fields = normalizeSort(sort);
10163
- if (fields.length === 0) return;
10164
- const flatSchema = flattenSchema(schema);
10165
- for (const entry of fields) {
10166
- const fieldType = flatSchema[entry.field];
10167
- if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(entry.field, "sort");
10168
- if (entry.direction !== "asc" && entry.direction !== "desc") {
10169
- throw new NarsilError(
10170
- ErrorCodes.SEARCH_INVALID_MODE,
10171
- `A sort direction is "asc" or "desc", and the direction on "${entry.field}" is "${String(entry.direction)}"`,
10172
- { field: entry.field, direction: String(entry.direction) }
10173
- );
10174
- }
10175
- requireSortMode(entry, fieldType);
10176
- const parsed = parsedTypeOf(fieldType);
10177
- if (parsed !== void 0 && isStringValuedBase(parsed.base) && !parsed.sortable) {
10178
- throw new NarsilError(
10179
- ErrorCodes.SEARCH_INVALID_FIELD,
10180
- `The engine sorts by text field "${entry.field}" only where its type includes "sortable", as in "${withSortableOption(parsed)}", because it keeps up to the first ${SORT_VALUE_MAX_CODE_POINTS} code points of every document's value in memory to sort by a text field`,
10181
- { field: entry.field, fieldType }
10182
- );
10183
- }
10184
- if (isGeopointOrVectorType(fieldType)) {
10185
- throw new NarsilError(
10186
- ErrorCodes.SEARCH_INVALID_FIELD,
10187
- `The engine sorts by a number, boolean, enum, or sortable text field, and "${entry.field}" is a ${fieldType} field`,
10188
- { field: entry.field, fieldType }
10189
- );
10245
+ const globalStats = compositeQueryStats(subs, params);
10246
+ const results = subs.map((sub, index) => searchFulltext(sub, subParams(params, layout, index, globalStats)));
10247
+ const scored = kWayMerge(results.map((result) => result.scored));
10248
+ if (params.maxResults !== void 0 && scored.length > params.maxResults) {
10249
+ scored.length = params.maxResults;
10250
+ }
10251
+ let totalMatched = 0;
10252
+ for (const result of results) {
10253
+ totalMatched += result.totalMatched;
10254
+ }
10255
+ if (params.collectMatchedSet === void 0) {
10256
+ return { scored, totalMatched };
10257
+ }
10258
+ if (params.collectMatchedSet === "ordinals") {
10259
+ const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
10260
+ for (let i = 0; i < results.length; i++) {
10261
+ const subBitset = results[i].matchedOrdinalBitset;
10262
+ if (subBitset !== void 0) placeSubBitset(matchedOrdinalBitset, layout, i, subBitset);
10190
10263
  }
10264
+ return { scored, totalMatched, matchedOrdinalBitset };
10191
10265
  }
10192
- }
10193
- function applySorting(hits, sort, sortKeyOf) {
10194
- const normalized = normalizeSort(sort);
10195
- if (normalized.length === 0) return hits;
10196
- const directions = normalized.map((entry) => entry.direction);
10197
- const keyCache = /* @__PURE__ */ new Map();
10198
- for (const hit of hits) {
10199
- keyCache.set(hit.id, sortKeyOf(hit.id));
10266
+ const matchedIds = [];
10267
+ for (const result of results) {
10268
+ if (result.matchedIds !== void 0) matchedIds.push(...result.matchedIds);
10200
10269
  }
10201
- const sorted = hits.slice();
10202
- sorted.sort(
10203
- (a, b) => compareComparableKeys(keyCache.get(a.id) ?? [], keyCache.get(b.id) ?? [], directions) || compareCodePoints(a.id, b.id)
10204
- );
10205
- return sorted;
10206
- }
10207
-
10208
- // src/search/cursor.ts
10209
- var CURSOR_VERSION = 4;
10210
- function invalidCursor(cursor, reason) {
10211
- const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
10212
- return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
10270
+ return { scored, totalMatched, matchedIds };
10213
10271
  }
10214
- function sortSignatureOf(sort) {
10215
- const fields = normalizeSort(sort);
10216
- if (fields.length === 0) return null;
10217
- if (fields.length > MAX_SORT_FIELDS) {
10218
- throw new NarsilError(
10219
- ErrorCodes.SEARCH_INVALID_FIELD,
10220
- `A sort names at most ${MAX_SORT_FIELDS} fields, because the paging cursor carries one value per field`,
10221
- { count: fields.length, limit: MAX_SORT_FIELDS }
10222
- );
10272
+ function compositeSearchMatches(subs, layout, params) {
10273
+ const subMatches = subs.map((sub, index) => searchFulltextMatches(sub, subParams(params, layout, index, void 0)));
10274
+ let count = 0;
10275
+ for (const matches of subMatches) {
10276
+ count += matches.count;
10223
10277
  }
10224
- for (const entry of fields) {
10225
- if (entry.field.length > MAX_SORT_FIELD_NAME_LENGTH) {
10226
- throw new NarsilError(
10227
- ErrorCodes.SEARCH_INVALID_FIELD,
10228
- `A sort field name holds at most ${MAX_SORT_FIELD_NAME_LENGTH} characters, because the paging cursor carries the sort with every page`,
10229
- { field: entry.field.slice(0, 64), length: entry.field.length, limit: MAX_SORT_FIELD_NAME_LENGTH }
10230
- );
10278
+ return {
10279
+ count,
10280
+ hasExternal(docId) {
10281
+ for (const matches of subMatches) {
10282
+ if (matches.hasExternal(docId)) return true;
10283
+ }
10284
+ return false;
10285
+ },
10286
+ hasInternal(internalId) {
10287
+ for (let i = subMatches.length - 1; i >= 0; i--) {
10288
+ if (internalId >= layout.bases[i]) {
10289
+ return subMatches[i].hasInternal(internalId - layout.bases[i]);
10290
+ }
10291
+ }
10292
+ return false;
10293
+ },
10294
+ matchedDocIds() {
10295
+ const combined = /* @__PURE__ */ new Set();
10296
+ for (const matches of subMatches) {
10297
+ for (const docId of matches.matchedDocIds()) {
10298
+ combined.add(docId);
10299
+ }
10300
+ }
10301
+ return combined;
10302
+ },
10303
+ ordinalBitset() {
10304
+ const combined = createBitSet(layout.totalCapacity);
10305
+ for (let i = 0; i < subMatches.length; i++) {
10306
+ placeSubBitset(combined, layout, i, subMatches[i].ordinalBitset());
10307
+ }
10308
+ return combined;
10231
10309
  }
10232
- }
10233
- return JSON.stringify(fields.map(sortSignatureEntry));
10310
+ };
10234
10311
  }
10235
- function encodePageCursor(cursor) {
10236
- const payload = { v: CURSOR_VERSION, a: cursor.anchor };
10237
- if (cursor.sortKey !== null && cursor.sortSignature !== null) {
10238
- payload.k = cursor.sortKey.map(toComparableSortValue);
10239
- payload.o = cursor.sortSignature;
10240
- } else if (cursor.score !== null) {
10241
- payload.s = cursor.score;
10312
+
10313
+ // src/core/partition/composite/sorting.ts
10314
+ function compareEntries(a, b, directions) {
10315
+ for (let i = 0; i < directions.length; i++) {
10316
+ const comparison = compareComparableValues(a.key[i] ?? null, b.key[i] ?? null, directions[i]);
10317
+ if (comparison !== 0) return comparison;
10242
10318
  }
10243
- if (cursor.depth !== void 0 && cursor.depth > 0) payload.d = cursor.depth;
10244
- payload.q = cursor.binding;
10245
- return encodeCursorText(JSON.stringify(payload));
10319
+ return compareCodePoints(a.id, b.id);
10246
10320
  }
10247
- function decodeSortKey(value, cursor) {
10248
- if (!Array.isArray(value)) throw invalidCursor(cursor, '"k" must be an array');
10249
- if (value.length > MAX_SORT_FIELDS) throw invalidCursor(cursor, '"k" carries too many sort values');
10250
- const key = [];
10251
- for (const entry of value) {
10252
- if (entry === null || typeof entry === "boolean") {
10253
- key.push(entry);
10254
- continue;
10255
- }
10256
- if (typeof entry === "number") {
10257
- if (!Number.isFinite(entry)) throw invalidCursor(cursor, '"k" holds a number that is not finite');
10258
- key.push(entry);
10259
- continue;
10260
- }
10261
- if (typeof entry === "string") {
10262
- if (exceedsCodePoints(entry, SORT_VALUE_MAX_CODE_POINTS)) {
10263
- throw invalidCursor(cursor, '"k" holds an oversized sort value');
10264
- }
10265
- key.push(entry);
10266
- continue;
10267
- }
10268
- throw invalidCursor(cursor, '"k" accepts a string, a finite number, a boolean, or null');
10269
- }
10270
- return key;
10321
+ function compositeSortedPage(subs, layout, request) {
10322
+ const pages = subs.map(
10323
+ (sub, index) => sortedPageOf(sub, {
10324
+ ...request,
10325
+ matches: request.matches === null ? null : subMatchesView(request.matches, layout, index)
10326
+ })
10327
+ );
10328
+ const merged = pages.flat();
10329
+ merged.sort((a, b) => compareEntries(a, b, request.directions));
10330
+ if (merged.length > request.limit) merged.length = request.limit;
10331
+ return merged;
10271
10332
  }
10272
- function decodePageCursor(cursor) {
10273
- if (cursor.length > MAX_CURSOR_LENGTH) throw invalidCursor(cursor, "the cursor is too long");
10274
- let json;
10275
- try {
10276
- json = decodeCursorText(cursor);
10277
- } catch {
10278
- throw invalidCursor(cursor, "invalid base64 encoding");
10279
- }
10280
- let parsed;
10281
- try {
10282
- parsed = JSON.parse(json);
10283
- } catch {
10284
- throw invalidCursor(cursor, "invalid JSON");
10285
- }
10286
- if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
10287
- throw invalidCursor(cursor, "expected an object");
10288
- }
10289
- const { v, a, s, k, o, d, q } = parsed;
10290
- if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
10291
- if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
10292
- throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
10293
- }
10294
- if (typeof a !== "string" || a.length === 0) throw invalidCursor(cursor, '"a" must be a non-empty string');
10295
- if (exceedsCodePoints(a, MAX_CURSOR_ANCHOR_CODE_POINTS)) {
10296
- throw invalidCursor(cursor, '"a" is too long to be a document id');
10297
- }
10298
- if (s !== void 0 && (typeof s !== "number" || !Number.isFinite(s))) {
10299
- throw invalidCursor(cursor, '"s" must be a finite number');
10300
- }
10301
- if (s !== void 0 && k !== void 0) throw invalidCursor(cursor, 'a cursor carries "s" or "k", never both');
10302
- if (d !== void 0 && (typeof d !== "number" || !Number.isSafeInteger(d) || d < 1)) {
10303
- throw invalidCursor(cursor, '"d" must be a positive integer');
10304
- }
10305
- const sortKey = k === void 0 ? null : decodeSortKey(k, cursor);
10306
- if (o !== void 0 && typeof o !== "string") throw invalidCursor(cursor, '"o" must be a string');
10307
- if (sortKey === null !== (o === void 0)) {
10308
- throw invalidCursor(cursor, '"k" and "o" have to arrive together');
10333
+ function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
10334
+ for (const sub of subs) {
10335
+ if (sub.docStore.has(docId)) {
10336
+ return sortValuesOf(sub, docId, fields, fieldTypes, modes);
10337
+ }
10309
10338
  }
10310
- return {
10311
- anchor: a,
10312
- score: typeof s === "number" ? s : null,
10313
- sortKey,
10314
- sortSignature: typeof o === "string" ? o : null,
10315
- depth: typeof d === "number" ? d : 0,
10316
- binding: q
10317
- };
10339
+ return fields.map(() => null);
10318
10340
  }
10319
- function requireMatchingCursor(cursor, encoded, signature, scoreAnchored, binding) {
10320
- if (cursor.binding !== binding) {
10321
- throw invalidCursor(encoded, "the cursor belongs to a different query");
10322
- }
10323
- if (signature !== null) {
10324
- if (cursor.sortSignature !== signature) {
10325
- throw invalidCursor(encoded, "the cursor belongs to a different sort order");
10341
+
10342
+ // src/core/partition/composite/suggest.ts
10343
+ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
10344
+ if (limit <= 0) return [];
10345
+ const merged = /* @__PURE__ */ new Map();
10346
+ for (const sub of subs) {
10347
+ for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, Number.MAX_SAFE_INTEGER)) {
10348
+ let entry = merged.get(suggestion.token);
10349
+ if (entry === void 0) {
10350
+ entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
10351
+ merged.set(suggestion.token, entry);
10352
+ }
10353
+ entry.documentFrequency += suggestion.documentFrequency;
10354
+ for (const surface of suggestion.surfaces) {
10355
+ entry.surfaceOccurrences.set(
10356
+ surface.surface,
10357
+ (entry.surfaceOccurrences.get(surface.surface) ?? 0) + surface.occurrences
10358
+ );
10359
+ }
10326
10360
  }
10327
- return;
10328
- }
10329
- if (cursor.sortSignature !== null) {
10330
- throw invalidCursor(encoded, "the cursor belongs to a sorted request");
10331
10361
  }
10332
- if (scoreAnchored && cursor.score === null) {
10333
- throw invalidCursor(encoded, "the cursor carries no score to seek from");
10334
- }
10335
- if (!scoreAnchored && cursor.score !== null) {
10336
- throw invalidCursor(encoded, "the cursor belongs to a search rather than a listing");
10362
+ const suggestions = [];
10363
+ for (const [token, entry] of merged) {
10364
+ const surfaces = Array.from(entry.surfaceOccurrences, ([surface, occurrences]) => ({ surface, occurrences }));
10365
+ surfaces.sort((a, b) => b.occurrences - a.occurrences || compareCodePoints(a.surface, b.surface));
10366
+ suggestions.push({ token, documentFrequency: entry.documentFrequency, surfaces });
10337
10367
  }
10368
+ suggestions.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.token, b.token));
10369
+ if (suggestions.length > limit) suggestions.length = limit;
10370
+ return suggestions;
10338
10371
  }
10339
10372
 
10340
- // src/search/pagination.ts
10341
- function clampRowCount(value, fallback) {
10342
- if (value === void 0 || !Number.isFinite(value)) return fallback;
10343
- return Math.max(0, Math.floor(value));
10344
- }
10345
- function requireWithinResultWindow(limit, offset) {
10346
- const depth = offset + limit;
10347
- if (depth <= RESULT_WINDOW) return;
10348
- throw new NarsilError(
10349
- ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
10350
- `A request reaches the first ${RESULT_WINDOW} results, and offset + limit is ${depth}. Page past that with the cursor each result carries`,
10351
- { limit, offset, window: RESULT_WINDOW }
10352
- );
10373
+ // src/core/partition/composite/index.ts
10374
+ function isCompositePartition(partition) {
10375
+ return "attachFrozenSegment" in partition;
10353
10376
  }
10354
- function lastOrganicResult(sliced, pinnedIds) {
10355
- for (let i = sliced.length - 1; i >= 0; i--) {
10356
- if (pinnedIds === void 0 || !pinnedIds.has(sliced[i].id)) return sliced[i];
10377
+ function createCompositePartition(partitionId, trackPositions = true, existingLive, textAnalysis) {
10378
+ const live = existingLive ?? createPartitionIndex(partitionId, trackPositions);
10379
+ const liveState = partitionStateOf(live);
10380
+ const frozen = [];
10381
+ let docFrequenciesCache = null;
10382
+ function invalidateDocFrequencies() {
10383
+ docFrequenciesCache = null;
10357
10384
  }
10358
- return void 0;
10359
- }
10360
- function ordersAfterAnchor(result, anchor, sort) {
10361
- if (sort !== void 0 && anchor.sortKey !== null) {
10362
- const comparison = compareSortValues(sort.sortKeyOf(result.id), anchor.sortKey, sort.directions) || compareCodePoints(result.id, anchor.anchor);
10363
- return comparison > 0;
10385
+ function bindSegments() {
10386
+ invalidateDocFrequencies();
10387
+ if (textAnalysis === void 0) return;
10388
+ for (const segment of frozen) segment.removals.bind(textAnalysis);
10364
10389
  }
10365
- if (anchor.score === null) return true;
10366
- if (result.score === void 0) return true;
10367
- if (result.score < anchor.score) return true;
10368
- return result.score === anchor.score && compareCodePoints(result.id, anchor.anchor) > 0;
10369
- }
10370
- function applyPagination(results, limit, offset, binding, cursor, sort, pinnedIds) {
10371
- const decoded = cursor ? decodePageCursor(cursor) : null;
10372
- if (decoded !== null && cursor !== void 0) {
10373
- requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding);
10390
+ function cachedDocFrequencies() {
10391
+ if (docFrequenciesCache === null) {
10392
+ docFrequenciesCache = mergeDocFrequencies(subs());
10393
+ }
10394
+ return docFrequenciesCache;
10374
10395
  }
10375
- if (limit <= 0) {
10376
- return { paginated: [] };
10396
+ function subs() {
10397
+ return [...frozen, liveState];
10377
10398
  }
10378
- let startIndex = 0;
10379
- if (decoded !== null) {
10380
- startIndex = results.length;
10381
- for (let i = 0; i < results.length; i++) {
10382
- if (ordersAfterAnchor(results[i], decoded, sort)) {
10383
- startIndex = i;
10384
- break;
10385
- }
10399
+ function composedScratch() {
10400
+ const scratch = createPartitionIndex(partitionId, trackPositions);
10401
+ for (const sub of subs()) {
10402
+ const payload = encodeSegmentState(sub);
10403
+ if (payload.documentCount === 0) continue;
10404
+ scratch.mergeSegmentPayload(payload, survivorDocumentsOf(sub, payload));
10386
10405
  }
10406
+ return scratch;
10387
10407
  }
10388
- const afterOffset = startIndex + offset;
10389
- const sliced = results.slice(afterOffset, afterOffset + limit);
10390
- let nextCursor;
10391
- const hasMore = afterOffset + limit < results.length;
10392
- const lastResult = lastOrganicResult(sliced, pinnedIds);
10393
- const depth = afterOffset + sliced.length;
10394
- if (hasMore && lastResult !== void 0) {
10395
- nextCursor = sort !== void 0 ? encodePageCursor({
10396
- anchor: lastResult.id,
10397
- score: null,
10398
- sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
10399
- sortSignature: sort.signature,
10400
- depth,
10401
- binding
10402
- }) : encodePageCursor({
10403
- anchor: lastResult.id,
10404
- score: lastResult.score ?? null,
10405
- sortKey: null,
10406
- sortSignature: null,
10407
- depth,
10408
- binding
10409
- });
10408
+ function layout() {
10409
+ return computeOrdinalLayout(subs());
10410
10410
  }
10411
- return { paginated: sliced, nextCursor };
10412
- }
10413
-
10414
- // src/search/term-groups.ts
10415
- function writtenWordsByPosition(term, language, analysis) {
10416
- const unstemmed = tokenize(term, language, {
10417
- stem: false,
10418
- removeStopWords: true,
10419
- customTokenizer: analysis?.customTokenizer,
10420
- stopWordOverride: analysis?.stopWords
10421
- });
10422
- const written = /* @__PURE__ */ new Map();
10423
- for (const token of unstemmed.tokens) {
10424
- if (!written.has(token.position)) written.set(token.position, token.token);
10411
+ function frozenOwner(docId) {
10412
+ for (const segment of frozen) {
10413
+ if (segment.hasDocument(docId)) return segment;
10414
+ }
10415
+ return void 0;
10425
10416
  }
10426
- return written;
10427
- }
10428
- function groupFor(groups, token, typoMatches) {
10429
- let group = groups.get(token);
10430
- if (group === void 0) {
10431
- group = { token, members: [{ token, factor: 1, partialFactor: 1 }], typoMatches };
10432
- groups.set(token, group);
10417
+ function frozenStored(docId) {
10418
+ for (const segment of frozen) {
10419
+ const stored = segment.docStore.get(docId);
10420
+ if (stored !== void 0) return stored;
10421
+ }
10422
+ return void 0;
10433
10423
  }
10434
- return group;
10435
- }
10436
- function addCompletions(groups, partition, lastToken, typedPrefix) {
10437
- if (typedPrefix === void 0 || typedPrefix.length === 0) return;
10438
- const group = groupFor(groups, lastToken.token, false);
10439
- const ownLength = codePointCount(lastToken.token);
10440
- for (const completion of partition.expandTermPrefix(typedPrefix, lastToken.token, PREFIX_MAX_EXPANSIONS)) {
10441
- if (completion === lastToken.token) continue;
10442
- const factor = Math.min(1, ownLength / codePointCount(completion));
10443
- group.members.push({ token: completion, factor, partialFactor: factor });
10444
- }
10445
- }
10446
- function partialWordAnalysisOf(fields, language, analysis) {
10447
10424
  return {
10448
- fields,
10449
- language,
10450
- options: {
10451
- stopWordOverride: analysis?.stopWords,
10452
- customTokenizer: analysis?.customTokenizer,
10453
- collectSurfaces: true
10425
+ get partitionId() {
10426
+ return partitionId;
10427
+ },
10428
+ get live() {
10429
+ return live;
10430
+ },
10431
+ get stats() {
10432
+ return buildAggregateStatsView([...frozen, liveState], cachedDocFrequencies);
10433
+ },
10434
+ frozenSegmentCount() {
10435
+ return frozen.length;
10436
+ },
10437
+ frozenSegmentSizes() {
10438
+ return frozen.map((segment) => ({ segmentId: segment.segmentId, liveDocumentCount: segment.liveDocumentCount() }));
10439
+ },
10440
+ frozenSegmentsById(segmentIds) {
10441
+ return resolveFrozenSegments(frozen, segmentIds, partitionId);
10442
+ },
10443
+ swapFrozenSegments(dropSegmentIds, replacement, removedSpellings) {
10444
+ if (textAnalysis !== void 0) replacement.removals.bind(textAnalysis);
10445
+ const applied = swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings);
10446
+ bindSegments();
10447
+ return applied;
10448
+ },
10449
+ freezeLiveTail(freeze) {
10450
+ const segment = freezeLiveTailInto(live, liveState, frozen, freeze);
10451
+ if (segment !== null) bindSegments();
10452
+ return segment;
10453
+ },
10454
+ replaceLiveTail(segment) {
10455
+ live.clear();
10456
+ frozen.push(segment);
10457
+ bindSegments();
10458
+ },
10459
+ attachFrozenSegment(segment) {
10460
+ frozen.push(segment);
10461
+ bindSegments();
10462
+ },
10463
+ mergeSegment(segment) {
10464
+ invalidateDocFrequencies();
10465
+ live.mergeSegment(segment);
10466
+ },
10467
+ mergeSegmentPayload(payload, documents) {
10468
+ invalidateDocFrequencies();
10469
+ live.mergeSegmentPayload(payload, documents);
10470
+ },
10471
+ encodeSegment() {
10472
+ if (frozen.length === 0) return live.encodeSegment();
10473
+ return composedScratch().encodeSegment();
10474
+ },
10475
+ serialize(indexName, totalPartitions, language, schema) {
10476
+ if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
10477
+ return composedScratch().serialize(indexName, totalPartitions, language, schema);
10478
+ },
10479
+ deserialize(data, schema) {
10480
+ frozen.length = 0;
10481
+ invalidateDocFrequencies();
10482
+ live.deserialize(data, schema);
10483
+ },
10484
+ clear() {
10485
+ frozen.length = 0;
10486
+ invalidateDocFrequencies();
10487
+ live.clear();
10488
+ },
10489
+ insert(docId, document, schema, language, options) {
10490
+ if (frozenOwner(docId) !== void 0) {
10491
+ throw new NarsilError(
10492
+ ErrorCodes.DOC_ALREADY_EXISTS,
10493
+ `Document "${docId}" already exists in partition ${partitionId}`,
10494
+ { docId, partitionId }
10495
+ );
10496
+ }
10497
+ invalidateDocFrequencies();
10498
+ live.insert(docId, document, schema, language, options);
10499
+ },
10500
+ update(docId, document, schema, language, options, removedSpellings) {
10501
+ invalidateDocFrequencies();
10502
+ const owner = frozenOwner(docId);
10503
+ if (owner === void 0) {
10504
+ live.update(docId, document, schema, language, options);
10505
+ return;
10506
+ }
10507
+ live.insert(docId, document, schema, language, options);
10508
+ owner.tombstoneDocument(docId, removedSpellings);
10509
+ },
10510
+ remove(docId, schema, language, options, removedSpellings) {
10511
+ if (live.has(docId)) {
10512
+ invalidateDocFrequencies();
10513
+ live.remove(docId, schema, language, options);
10514
+ return;
10515
+ }
10516
+ const owner = frozenOwner(docId);
10517
+ if (owner === void 0) {
10518
+ throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in partition ${partitionId}`, {
10519
+ docId,
10520
+ partitionId
10521
+ });
10522
+ }
10523
+ invalidateDocFrequencies();
10524
+ owner.tombstoneDocument(docId, removedSpellings);
10525
+ },
10526
+ removedSpellingsOf(docId) {
10527
+ return live.has(docId) ? void 0 : frozenOwner(docId)?.removedSpellingsOf(docId);
10528
+ },
10529
+ beginBatch() {
10530
+ live.beginBatch();
10531
+ },
10532
+ endBatch() {
10533
+ live.endBatch();
10534
+ },
10535
+ searchFulltext(params) {
10536
+ return compositeSearchFulltext(subs(), layout(), params);
10537
+ },
10538
+ searchFulltextMatches(params) {
10539
+ return compositeSearchMatches(subs(), layout(), params);
10540
+ },
10541
+ applyFilters(filters, schema, meter) {
10542
+ return compositeFilters(subs(), filters, schema, meter);
10543
+ },
10544
+ applyFiltersBitset(filters, schema, meter) {
10545
+ return compositeFiltersBitset(subs(), layout(), filters, schema, meter);
10546
+ },
10547
+ filterMatches(filters, schema, meter) {
10548
+ return compositeFilterMatches(subs(), layout(), filters, schema, meter);
10549
+ },
10550
+ computeFacets(matched, config, schema) {
10551
+ const everyValue = everyValueFacetConfig(config);
10552
+ if ("ordinalBitset" in matched) {
10553
+ const ordinalLayout = layout();
10554
+ return mergeFacets(
10555
+ subs().map(
10556
+ (sub, index) => computeFacets(
10557
+ sub,
10558
+ { ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
10559
+ everyValue,
10560
+ schema
10561
+ )
10562
+ ),
10563
+ config
10564
+ );
10565
+ }
10566
+ return mergeFacets(
10567
+ subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
10568
+ config
10569
+ );
10570
+ },
10571
+ sortedPage(request) {
10572
+ return compositeSortedPage(subs(), layout(), request);
10573
+ },
10574
+ sortValues(docId, fields, fieldTypes, modes) {
10575
+ return compositeSortValues(subs(), docId, fields, fieldTypes, modes);
10576
+ },
10577
+ suggestTerms(surfacePrefix, stemmedPrefix, limit) {
10578
+ return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
10579
+ },
10580
+ expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
10581
+ return expandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
10582
+ },
10583
+ expandPartialWords(analysis, requests) {
10584
+ return expandPartialWords(subs(), analysis, requests);
10585
+ },
10586
+ preparePartialWords(analysis) {
10587
+ expandPartialWords(subs(), analysis, []);
10588
+ },
10589
+ get(docId, projection) {
10590
+ return compositeDocument(frozen, live, docId, projection);
10591
+ },
10592
+ fieldValues(docId, fieldPaths) {
10593
+ return compositeFieldValues(frozen, live, docId, fieldPaths);
10594
+ },
10595
+ getRef(docId) {
10596
+ const stored = frozenStored(docId);
10597
+ if (stored !== void 0) return stored.fields;
10598
+ return live.getRef(docId);
10599
+ },
10600
+ has(docId) {
10601
+ return live.has(docId) || frozenOwner(docId) !== void 0;
10602
+ },
10603
+ count() {
10604
+ let total = live.count();
10605
+ for (const segment of frozen) {
10606
+ total += segment.liveDocumentCount();
10607
+ }
10608
+ return total;
10609
+ },
10610
+ *docIds() {
10611
+ for (const segment of frozen) {
10612
+ for (const internalId of segment.docStore.allInternalIds()) {
10613
+ const docId = segment.docStore.getExternalId(internalId);
10614
+ if (docId !== void 0) yield docId;
10615
+ }
10616
+ }
10617
+ yield* live.docIds();
10618
+ },
10619
+ sortedDocIds() {
10620
+ const merged = [];
10621
+ for (const segment of frozen) {
10622
+ merged.push(...segment.docStore.sortedDocIds());
10623
+ }
10624
+ merged.push(...live.sortedDocIds());
10625
+ merged.sort(compareCodePoints);
10626
+ return merged;
10627
+ },
10628
+ releaseSortedDocIds() {
10629
+ for (const segment of frozen) {
10630
+ segment.docStore.releaseSortedDocIds();
10631
+ }
10632
+ live.releaseSortedDocIds();
10633
+ },
10634
+ estimateMemoryBytes() {
10635
+ let total = live.estimateMemoryBytes();
10636
+ for (const segment of frozen) {
10637
+ total += frozenSegmentBytes(segment);
10638
+ }
10639
+ return total;
10454
10640
  }
10455
10641
  };
10456
10642
  }
10457
- function searchesAPartialField(params, partialFields) {
10458
- if (params.fields === void 0) return true;
10459
- for (const field of params.fields) {
10460
- if (partialFields.includes(field)) return true;
10643
+
10644
+ // src/partitioning/insert-options.ts
10645
+ function resolvePartialWordAnalysis(config, language, analysis) {
10646
+ const fields = partialFieldsOf(config.schema);
10647
+ if (fields.length === 0) return null;
10648
+ return { fields, language, options: resolvePartitionInsertOptions(config, analysis) };
10649
+ }
10650
+ function resolvePartitionInsertOptions(config, analysis, options) {
10651
+ const applyStrict = config.strict === true;
10652
+ const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
10653
+ const applySurfaces = config.surfaceForms !== false;
10654
+ const applyPatternLimit = config.patternValueLimit !== void 0;
10655
+ if (!applyStrict && !applyAnalyzer && !applySurfaces && !applyPatternLimit) return options;
10656
+ const resolved = { ...options };
10657
+ if (applyStrict) resolved.strict = true;
10658
+ if (applyPatternLimit) resolved.patternValueLimit = config.patternValueLimit;
10659
+ if (applyAnalyzer) {
10660
+ resolved.stopWordOverride = options?.stopWordOverride ?? analysis.stopWords;
10661
+ resolved.customTokenizer = options?.customTokenizer ?? analysis.customTokenizer;
10461
10662
  }
10462
- return false;
10663
+ if (applySurfaces) resolved.collectSurfaces = true;
10664
+ return resolved;
10463
10665
  }
10464
- function resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis) {
10465
- const term = params.term;
10466
- if (term === void 0 || params.exact === true) return {};
10467
- const usesPrefix = params.prefix === true;
10468
- const partialFields = params.partialWords === true ? partialFieldsOf(schema) : [];
10469
- const usesPartialWords = partialFields.length > 0 && searchesAPartialField(params, partialFields);
10470
- if (!usesPrefix && !usesPartialWords) return {};
10471
- const written = writtenWordsByPosition(term, language, analysis);
10472
- const groups = /* @__PURE__ */ new Map();
10473
- if (usesPrefix) addCompletions(groups, partition, lastToken, written.get(lastToken.position));
10474
- const prefixToken = groups.has(lastToken.token) ? lastToken.token : void 0;
10475
- if (usesPartialWords) {
10476
- addPartialWordMembers(
10477
- {
10478
- queryTokens,
10479
- written,
10480
- prefixToken,
10481
- tolerance: params.tolerance ?? 0,
10482
- prefixLength: params.prefixLength,
10483
- analysis: partialWordAnalysisOf(partialFields, language, analysis)
10484
- },
10485
- (partialAnalysis, requests) => partition.expandPartialWords(partialAnalysis, requests),
10486
- (token, typoMatches) => groupFor(groups, token, typoMatches)
10487
- );
10488
- }
10489
- const termGroups = [...groups.values()];
10490
- if (termGroups.length === 0) return {};
10491
- return usesPartialWords ? { termGroups, partialFields: [...partialFields] } : { termGroups };
10666
+
10667
+ // src/partitioning/manager/capacity.ts
10668
+ function createCapacityChecks(source) {
10669
+ const { indexName, config } = source;
10670
+ return {
10671
+ assertCapacity(pendingWrites = 0, partitionCountCap) {
10672
+ const maxDocsPerPartition = config.partitions?.maxDocsPerPartition;
10673
+ if (maxDocsPerPartition === void 0) return;
10674
+ const partitionCount = source.partitionCount();
10675
+ const effectivePartitionCount = partitionCountCap === void 0 ? partitionCount : Math.min(partitionCount, partitionCountCap);
10676
+ const totalCapacity = maxDocsPerPartition * effectivePartitionCount;
10677
+ const currentCount = source.documentCount() + pendingWrites;
10678
+ if (currentCount < totalCapacity) return;
10679
+ throw new NarsilError(
10680
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
10681
+ `Index "${indexName}" has reached its capacity of ${totalCapacity} documents (${maxDocsPerPartition} per partition \xD7 ${effectivePartitionCount} partitions)`,
10682
+ {
10683
+ indexName,
10684
+ currentCount,
10685
+ totalCapacity,
10686
+ maxDocsPerPartition,
10687
+ partitionCount: effectivePartitionCount
10688
+ }
10689
+ );
10690
+ },
10691
+ assertPartitionCapacity(partitionId, pendingWrites = 0) {
10692
+ const maxDocsPerPartition = config.partitions?.maxDocsPerPartition;
10693
+ if (maxDocsPerPartition === void 0) return;
10694
+ const currentCount = source.getPartition(partitionId).count() + pendingWrites;
10695
+ if (currentCount < maxDocsPerPartition) return;
10696
+ throw new NarsilError(
10697
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
10698
+ `Partition ${partitionId} of index "${indexName}" has reached its capacity of ${maxDocsPerPartition} documents`,
10699
+ { indexName, partitionId, currentCount, maxDocsPerPartition }
10700
+ );
10701
+ }
10702
+ };
10492
10703
  }
10493
10704
 
10494
- // src/search/fulltext.ts
10495
- function prepareFulltextQuery(partition, params, language, schema, options) {
10496
- if (!params.term || params.term.trim().length === 0) return null;
10497
- if (params.fields && params.fields.length === 0) return null;
10498
- const flatSchema = flattenSchema(schema);
10499
- if (params.fields) {
10500
- validateSearchFields(params.fields, flatSchema);
10705
+ // src/partitioning/manager/nested-values.ts
10706
+ function setNestedValue(obj, path, value) {
10707
+ if (!path.includes(".")) {
10708
+ obj[path] = value;
10709
+ return;
10501
10710
  }
10502
- const queryTokenResult = tokenize(params.term, language, {
10503
- stem: true,
10504
- removeStopWords: true,
10505
- customTokenizer: options?.customTokenizer,
10506
- stopWordOverride: options?.stopWords
10507
- });
10508
- if (queryTokenResult.tokens.length === 0) return null;
10509
- const queryTokens = deduplicateTokens(queryTokenResult.tokens);
10510
- let filterBitset;
10511
- if (params.filters) {
10512
- filterBitset = partition.applyFiltersBitset(
10513
- params.filters,
10514
- schema,
10515
- options?.patternWork ?? createPatternWorkMeter()
10516
- );
10517
- if (bitsetIsEmpty(filterBitset)) return null;
10711
+ const segments = path.split(".");
10712
+ let current = obj;
10713
+ for (let i = 0; i < segments.length - 1; i++) {
10714
+ let next = current[segments[i]];
10715
+ if (next === null || next === void 0 || typeof next !== "object") {
10716
+ next = {};
10717
+ current[segments[i]] = next;
10718
+ }
10719
+ current = next;
10518
10720
  }
10519
- const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
10520
- const groups = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, options);
10521
- return { queryTokens, ...groups, filterBitset };
10522
- }
10523
- function fulltextMatches(partition, params, language, schema, options) {
10524
- const prepared = prepareFulltextQuery(partition, params, language, schema, options);
10525
- if (prepared === null) return null;
10526
- return partition.searchFulltextMatches({
10527
- queryTokens: prepared.queryTokens,
10528
- termGroups: prepared.termGroups,
10529
- partialFields: prepared.partialFields,
10530
- fields: params.fields,
10531
- tolerance: params.tolerance ?? 0,
10532
- prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
10533
- exact: params.exact ?? false,
10534
- filterBitset: prepared.filterBitset
10535
- });
10721
+ current[segments[segments.length - 1]] = value;
10536
10722
  }
10537
- function fulltextSearch(partition, params, language, schema, options) {
10538
- const prepared = prepareFulltextQuery(partition, params, language, schema, options);
10539
- if (prepared === null) {
10540
- return { scored: [], totalMatched: 0 };
10541
- }
10542
- const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
10543
- const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
10544
- const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
10545
- const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
10546
- const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
10547
- const rawResult = partition.searchFulltext({
10548
- queryTokens,
10549
- termGroups,
10550
- partialFields,
10551
- fields: params.fields,
10552
- boost: params.boost,
10553
- tolerance: params.tolerance ?? 0,
10554
- prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
10555
- exact: params.exact ?? false,
10556
- bm25Params: options?.bm25Params,
10557
- globalStats: options?.globalStats,
10558
- maxResults,
10559
- termMatch: params.termMatch,
10560
- filterBitset,
10561
- collectComponents,
10562
- collectMatchedSet
10563
- });
10564
- let scored = rawResult.scored;
10565
- const termMatch = params.termMatch ?? "any";
10566
- if (termMatch !== "any") {
10567
- scored = filterByTermCoverage(
10568
- scored,
10569
- queryTokens,
10570
- termMatch,
10571
- params.tolerance ?? 0,
10572
- params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
10573
- params.exact ?? false,
10574
- termGroups,
10575
- partialFields ?? []
10576
- );
10577
- }
10578
- if (params.minScore !== void 0 && params.minScore > 0) {
10579
- const threshold = params.minScore;
10580
- scored = scored.filter((doc) => doc.score >= threshold);
10581
- }
10582
- const totalMatched = needsAllResults ? scored.length : rawResult.totalMatched;
10583
- if (params.facets === void 0) {
10584
- return { scored, totalMatched };
10585
- }
10586
- if (needsAllResults) {
10587
- return { scored, totalMatched, matchedIds: scored.map((doc) => doc.docId) };
10723
+ function copyVectorsInto(document, docId, vectorIndexes, projection) {
10724
+ for (const [fieldPath, vectorIndex] of vectorIndexes) {
10725
+ if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
10726
+ const vector = vectorIndex.getVector(docId);
10727
+ if (vector) setNestedValue(document, fieldPath, vector);
10588
10728
  }
10589
- return { scored, totalMatched, matchedOrdinalBitset: rawResult.matchedOrdinalBitset };
10590
10729
  }
10591
- function validateSearchFields(fields, flatSchema) {
10592
- for (const field of fields) {
10593
- const fieldType = flatSchema[field];
10594
- if (!fieldType) {
10595
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_FIELD, `Field "${field}" does not exist in the schema`, { field });
10596
- }
10597
- if (!isWordIndexedFieldType(fieldType)) {
10598
- throw new NarsilError(
10599
- ErrorCodes.SEARCH_INVALID_FIELD,
10600
- `Field "${field}" has type "${fieldType}" which cannot be used for full-text search`,
10601
- { field, fieldType }
10602
- );
10603
- }
10730
+
10731
+ // src/partitioning/manager/segments.ts
10732
+ function requireNewSegmentDocuments(segment, docPartitionMap) {
10733
+ for (const internalId of segment.docStore.allInternalIds()) {
10734
+ const docId = segment.docStore.getExternalId(internalId);
10735
+ if (docId === void 0 || !docPartitionMap.has(docId)) continue;
10736
+ throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
10737
+ docId,
10738
+ partitionId: docPartitionMap.get(docId)
10739
+ });
10604
10740
  }
10605
10741
  }
10606
- function deduplicateTokens(tokens) {
10607
- const seen = /* @__PURE__ */ new Set();
10608
- const result = [];
10609
- for (const t of tokens) {
10610
- if (!seen.has(t.token)) {
10611
- seen.add(t.token);
10612
- result.push(t);
10742
+
10743
+ // src/partitioning/manager/stats.ts
10744
+ function aggregateStatsOf(partitions) {
10745
+ let totalDocuments = 0;
10746
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
10747
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
10748
+ for (let i = 0; i < partitions.length; i++) {
10749
+ const stats = partitions[i].stats;
10750
+ totalDocuments += stats.totalDocuments;
10751
+ for (const [term, freq] of Object.entries(stats.docFrequencies)) {
10752
+ docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
10613
10753
  }
10614
- }
10615
- return result;
10616
- }
10617
- function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
10618
- const requiredCount = policy === "all" ? queryTokens.length : policy;
10619
- if (requiredCount <= 0) return scored;
10620
- if (requiredCount > queryTokens.length) return [];
10621
- const groups = /* @__PURE__ */ new Map();
10622
- for (const group of termGroups ?? []) {
10623
- const partialOnly = /* @__PURE__ */ new Set();
10624
- for (const member of group.members) {
10625
- if (member.factor === 0) partialOnly.add(member.token);
10754
+ for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
10755
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
10626
10756
  }
10627
- groups.set(group.token, {
10628
- members: new Set(group.members.map((member) => member.token)),
10629
- partialOnly,
10630
- typoMatches: group.typoMatches
10631
- });
10632
10757
  }
10633
- return scored.filter((doc) => {
10634
- const matched = countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields);
10635
- return matched >= requiredCount;
10636
- });
10758
+ return { totalDocuments, docFrequencies, totalFieldLengths };
10637
10759
  }
10638
- function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
10639
- const indexTokens = Object.keys(doc.idf);
10640
- if (indexTokens.length === 0) return 0;
10641
- let count = 0;
10642
- for (const qt of queryTokens) {
10643
- const group = groups.get(qt.token);
10644
- const satisfied = group === void 0 ? queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact) : groupSatisfied(group, doc, indexTokens, partialFields) || group.typoMatches && queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact);
10645
- if (satisfied) count++;
10760
+ function estimateBytesOf(partitions, vecIndexes) {
10761
+ let total = 0;
10762
+ for (let i = 0; i < partitions.length; i++) {
10763
+ total += partitions[i].estimateMemoryBytes();
10646
10764
  }
10647
- return count;
10648
- }
10649
- function matchedInPartialField(doc, token, partialFields) {
10650
- for (const field of partialFields) {
10651
- if (doc.termFrequencies[`${field}:${token}`] !== void 0) return true;
10765
+ for (const [, vecIndex] of vecIndexes) {
10766
+ total += vecIndex.estimateMemoryBytes();
10652
10767
  }
10653
- return false;
10768
+ return total;
10654
10769
  }
10655
- function groupSatisfied(group, doc, indexTokens, partialFields) {
10656
- for (const indexToken of indexTokens) {
10657
- if (!group.members.has(indexToken)) continue;
10658
- if (!group.partialOnly.has(indexToken) || matchedInPartialField(doc, indexToken, partialFields)) return true;
10659
- }
10660
- return false;
10661
- }
10662
- function queryTermSatisfied(queryToken, indexTokens, tolerance, prefixLength, exact) {
10663
- if (exact || tolerance === 0) {
10664
- return indexTokens.includes(queryToken);
10665
- }
10666
- for (const indexToken of indexTokens) {
10667
- if (fuzzyTermMatches(queryToken, indexToken, tolerance, prefixLength)) {
10668
- return true;
10669
- }
10670
- }
10671
- return false;
10770
+ function partitionStatsOf(partitions) {
10771
+ return partitions.map((p, i) => ({
10772
+ partitionId: i,
10773
+ documentCount: p.count(),
10774
+ estimatedMemoryBytes: p.estimateMemoryBytes()
10775
+ }));
10672
10776
  }
10673
10777
 
10674
- // src/partitioning/partition-selection.ts
10675
- function partitionsIn(manager, partitionIds) {
10676
- if (partitionIds === void 0) {
10677
- return manager.getAllPartitions();
10678
- }
10679
- const partitions = [];
10680
- for (const partitionId of partitionIds) {
10681
- const partition = manager.partitionAt(partitionId);
10682
- if (partition !== void 0) {
10683
- partitions.push(partition);
10778
+ // src/partitioning/manager/write-log.ts
10779
+ function createWriteLogRecorder() {
10780
+ let current = null;
10781
+ return {
10782
+ open() {
10783
+ const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
10784
+ current = log;
10785
+ return {
10786
+ get partitionsReplaced() {
10787
+ return log.replaced;
10788
+ },
10789
+ get pendingDocuments() {
10790
+ return log.docIds.size;
10791
+ },
10792
+ takeWrittenDocIds() {
10793
+ const taken = log.docIds;
10794
+ log.docIds = /* @__PURE__ */ new Set();
10795
+ return taken;
10796
+ },
10797
+ close() {
10798
+ if (current === log) current = null;
10799
+ }
10800
+ };
10801
+ },
10802
+ documentWritten(docId) {
10803
+ if (current !== null) current.docIds.add(docId);
10804
+ },
10805
+ segmentWritten(segment) {
10806
+ if (current === null) return;
10807
+ for (const internalId of segment.docStore.allInternalIds()) {
10808
+ const docId = segment.docStore.getExternalId(internalId);
10809
+ if (docId !== void 0) current.docIds.add(docId);
10810
+ }
10811
+ },
10812
+ partitionsReplaced() {
10813
+ if (current !== null) current.replaced = true;
10684
10814
  }
10685
- }
10686
- return partitions;
10815
+ };
10687
10816
  }
10688
10817
 
10689
- // src/partitioning/fan-out.ts
10690
- async function fanOutQuery(manager, params, language, schema, config, searchOptions) {
10691
- const partitions = partitionsIn(manager, config.partitionIds);
10692
- if (partitions.length === 0) {
10693
- return { scored: [], totalMatched: 0 };
10818
+ // src/partitioning/manager/index.ts
10819
+ function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
10820
+ requirePatternSearchFor(config.schema);
10821
+ requirePartialWordSearchFor(config.schema);
10822
+ const count = initialPartitionCount ?? 1;
10823
+ const analysis = resolveIndexAnalysis(config);
10824
+ const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
10825
+ const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
10826
+ const trackPositions = config.trackPositions ?? true;
10827
+ const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
10828
+ let partitions = [];
10829
+ const docPartitionMap = /* @__PURE__ */ new Map();
10830
+ const writeLog = createWriteLogRecorder();
10831
+ for (let i = 0; i < count; i++) {
10832
+ partitions.push(createPartitionIndex(i, trackPositions));
10694
10833
  }
10695
- let globalStats;
10696
- const effectiveMode = resolveEffectiveMode(config);
10697
- if (effectiveMode === "dfs") {
10698
- if (params.term !== void 0 && params.term.trim().length > 0) {
10699
- globalStats = collectQueryTermStats(
10700
- manager,
10701
- params.term,
10702
- language,
10703
- {
10704
- stopWords: searchOptions?.stopWords,
10705
- customTokenizer: searchOptions?.customTokenizer
10706
- },
10707
- config.partitionIds
10834
+ function validatePartitionId(partitionId) {
10835
+ if (partitionId < 0 || partitionId >= partitions.length) {
10836
+ throw new NarsilError(
10837
+ ErrorCodes.INDEX_NOT_FOUND,
10838
+ `Partition ${partitionId} is out of bounds (0..${partitions.length - 1})`,
10839
+ { partitionId, partitionCount: partitions.length }
10708
10840
  );
10709
10841
  }
10710
- } else if (effectiveMode === "broadcast") {
10711
- globalStats = config.globalStats;
10712
- }
10713
- const options = buildSearchOptions(searchOptions, globalStats);
10714
- if (partitions.length === 1 && !config.dispatcher) {
10715
- const result = dispatchSinglePartition(partitions[0], params, language, schema, options);
10716
- let facets2;
10717
- if (params.facets) {
10718
- facets2 = partitions[0].computeFacets(facetMatchSetOf(result), params.facets, schema);
10719
- }
10720
- return { scored: result.scored, totalMatched: result.totalMatched, facets: facets2 };
10721
- }
10722
- let outcomes;
10723
- let facets;
10724
- if (config.dispatcher) {
10725
- outcomes = await dispatchWithDispatcher(partitions, params, language, schema, options, config.dispatcher);
10726
- if (params.facets) {
10727
- facets = collectAndMergeFacets(outcomes, params.facets, schema);
10728
- }
10729
- } else {
10730
- outcomes = [];
10731
- const partitionFacets = [];
10732
- const everyValue = params.facets ? everyValueFacetConfig(params.facets) : void 0;
10733
- for (const partition of partitions) {
10734
- const result = dispatchSinglePartition(partition, params, language, schema, options);
10735
- if (everyValue) {
10736
- partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), everyValue, schema));
10737
- }
10738
- outcomes.push({ result, partition });
10739
- }
10740
- if (params.facets) {
10741
- facets = mergeFacets(partitionFacets, params.facets);
10742
- }
10743
10842
  }
10744
- const allScoredArrays = outcomes.map((o) => o.result.scored);
10745
- const merged = kWayMerge(allScoredArrays);
10746
- let totalMatched = 0;
10747
- for (const outcome of outcomes) {
10748
- totalMatched += outcome.result.totalMatched;
10843
+ function resolveInsertOptions(options) {
10844
+ return resolvePartitionInsertOptions(config, analysis, options);
10749
10845
  }
10750
- return { scored: merged, totalMatched, facets };
10751
- }
10752
- function facetMatchSetOf(result) {
10753
- if (result.matchedOrdinalBitset !== void 0) {
10754
- return { ordinalBitset: result.matchedOrdinalBitset };
10846
+ function preparePartialWords(partitionId) {
10847
+ if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
10755
10848
  }
10756
- return new Set(result.matchedIds ?? []);
10757
- }
10758
- function resolveEffectiveMode(config) {
10759
- if (config.scoringMode === "broadcast" && !config.globalStats) {
10760
- return "dfs";
10849
+ function locateDocument(docId) {
10850
+ return docPartitionMap.get(docId);
10761
10851
  }
10762
- return config.scoringMode;
10763
- }
10764
- function buildSearchOptions(base, globalStats) {
10765
- return {
10766
- ...base,
10767
- globalStats: globalStats ?? base?.globalStats,
10768
- patternWork: base?.patternWork ?? createPatternWorkMeter()
10769
- };
10770
- }
10771
- function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
10772
- const promises = partitions.map(
10773
- (partition) => dispatcher.dispatch(partition, params, language, schema, options).then((result) => ({ result, partition }))
10774
- );
10775
- return Promise.all(promises);
10776
- }
10777
- function dispatchSinglePartition(partition, params, language, schema, options) {
10778
- return fulltextSearch(partition, params, language, schema, options);
10779
- }
10780
- function collectAndMergeFacets(outcomes, facetConfig, schema) {
10781
- const everyValue = everyValueFacetConfig(facetConfig);
10782
- const partitionFacets = [];
10783
- for (const outcome of outcomes) {
10784
- partitionFacets.push(outcome.partition.computeFacets(facetMatchSetOf(outcome.result), everyValue, schema));
10852
+ function asCompositePartition(partitionId) {
10853
+ const existing = partitions[partitionId];
10854
+ if (isCompositePartition(existing)) return existing;
10855
+ const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
10856
+ partitions[partitionId] = composite;
10857
+ return composite;
10785
10858
  }
10786
- return mergeFacets(partitionFacets, facetConfig);
10787
- }
10788
-
10789
- // src/core/partition/frozen/memory.ts
10790
- function frozenSegmentBytes(segment) {
10791
- const arrays = segment.arrays;
10792
- let bytes = arrays.tokenTable.bytes + arrays.idTable.bytes;
10793
- bytes += arrays.postingOffsets.byteLength;
10794
- bytes += arrays.postingDocIds.byteLength;
10795
- bytes += arrays.postingFrequencies.byteLength;
10796
- bytes += arrays.postingFieldIndices.byteLength;
10797
- if (arrays.positionOffsets !== null) bytes += arrays.positionOffsets.byteLength;
10798
- if (arrays.positionValues !== null) bytes += arrays.positionValues.byteLength;
10799
- if (arrays.documentTable !== null) {
10800
- bytes += arrays.documentTable.blob.byteLength + arrays.documentTable.offsets.byteLength;
10859
+ function rebuildDocPartitionMap() {
10860
+ docPartitionMap.clear();
10861
+ for (let i = 0; i < partitions.length; i++) {
10862
+ for (const docId of partitions[i].docIds()) {
10863
+ docPartitionMap.set(docId, i);
10864
+ }
10865
+ }
10801
10866
  }
10802
- for (const column of arrays.fieldLengthColumns) bytes += column.byteLength;
10803
- for (const entry of [...arrays.numeric, ...arrays.sizes]) bytes += entry.docIds.byteLength + entry.values.byteLength;
10804
- for (const entry of arrays.boolean) bytes += entry.trueDocs.byteLength + entry.falseDocs.byteLength;
10805
- for (const entry of arrays.enums) bytes += entry.offsets.byteLength + entry.docIds.byteLength;
10806
- for (const entry of arrays.geo) {
10807
- bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
10867
+ function registerSegmentDocuments(partitionId, segment) {
10868
+ for (const internalId of segment.docStore.allInternalIds()) {
10869
+ const docId = segment.docStore.getExternalId(internalId);
10870
+ if (docId !== void 0) docPartitionMap.set(docId, partitionId);
10871
+ }
10808
10872
  }
10809
- for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
10810
- return bytes + segment.partialWords.bytes() + segment.removalBytes();
10811
- }
10812
-
10813
- // src/core/partition/composite/compaction.ts
10814
- function resolveFrozenSegments(frozen, segmentIds, partitionId) {
10815
- const byId = new Map(frozen.map((segment) => [segment.segmentId, segment]));
10816
- return segmentIds.map((segmentId) => {
10817
- const segment = byId.get(segmentId);
10818
- if (segment === void 0) {
10819
- throw new NarsilError(
10820
- ErrorCodes.PARTITION_CORRUPTED,
10821
- `Partition ${partitionId} does not hold frozen segment "${segmentId}"`,
10822
- { partitionId, segmentId }
10823
- );
10873
+ function rebuildDocPartitionMapForPartition(partitionId) {
10874
+ for (const [docId, pid] of docPartitionMap) {
10875
+ if (pid === partitionId) {
10876
+ docPartitionMap.delete(docId);
10877
+ }
10824
10878
  }
10825
- return segment;
10826
- });
10827
- }
10828
- function survivorDocumentsOf(sub, payload) {
10829
- return payload.docIds.map((docId) => {
10830
- const stored = sub.docStore.get(docId);
10831
- if (stored === void 0) {
10832
- throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while compacting a segment`, {
10833
- docId
10834
- });
10835
- }
10836
- return stored.fields;
10837
- });
10838
- }
10839
- function freezeLiveTailInto(live, liveState, frozen, freeze) {
10840
- if (live.count() === 0) return null;
10841
- const payload = live.encodeSegment();
10842
- const segment = freeze(payload, survivorDocumentsOf(liveState, payload));
10843
- if (segment === null) return null;
10844
- live.clear();
10845
- frozen.push(segment);
10846
- return segment;
10847
- }
10848
- function swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings) {
10849
- const dropSet = new Set(dropSegmentIds);
10850
- const dropped = frozen.filter((segment) => dropSet.has(segment.segmentId));
10851
- const applied = /* @__PURE__ */ new Map();
10852
- for (const segment of dropped) {
10853
- for (const docId of segment.tombstonedDocIds()) {
10854
- if (!replacement.hasDocument(docId)) continue;
10855
- if (dropped.some((other) => other.hasDocument(docId))) continue;
10856
- const spellings = removedSpellings?.get(docId) ?? replacement.removedSpellingsOf(docId);
10857
- if (spellings !== void 0) applied.set(docId, spellings);
10858
- replacement.tombstoneDocument(docId, spellings);
10859
- }
10860
- }
10861
- const kept = frozen.filter((segment) => !dropSet.has(segment.segmentId));
10862
- frozen.length = 0;
10863
- frozen.push(...kept, replacement);
10864
- return applied;
10865
- }
10866
-
10867
- // src/core/partition/composite/documents.ts
10868
- function compositeDocument(frozen, live, docId, projection) {
10869
- for (const segment of frozen) {
10870
- const projected = segment.docStore.projectedDocument(docId, projection);
10871
- if (projected !== void 0) return projected;
10872
- }
10873
- return live.get(docId, projection);
10874
- }
10875
- function compositeFieldValues(frozen, live, docId, fieldPaths) {
10876
- for (const segment of frozen) {
10877
- const internalId = segment.docStore.getInternalId(docId);
10878
- if (internalId === void 0) continue;
10879
- return fieldPaths.map((fieldPath) => segment.docStore.fieldReader(fieldPath)(internalId));
10880
- }
10881
- return live.fieldValues(docId, fieldPaths);
10882
- }
10883
-
10884
- // src/core/partition/composite/filters.ts
10885
- var WORD_BITS = 32;
10886
- function alignToWord(value) {
10887
- return value + WORD_BITS - 1 & -32;
10888
- }
10889
- function computeOrdinalLayout(subs) {
10890
- const bases = new Array(subs.length);
10891
- let cursor = 0;
10892
- for (let i = 0; i < subs.length; i++) {
10893
- bases[i] = cursor;
10894
- cursor += alignToWord(subs[i].docStore.internalIdCapacity());
10895
- }
10896
- return { bases, totalCapacity: cursor };
10897
- }
10898
- function subBitsetView(composite, layout, subIndex) {
10899
- const startWord = layout.bases[subIndex] >> 5;
10900
- const endWord = subIndex + 1 < layout.bases.length ? layout.bases[subIndex + 1] >> 5 : Math.ceil(layout.totalCapacity / WORD_BITS);
10901
- return composite.subarray(startWord, endWord);
10902
- }
10903
- function placeSubBitset(composite, layout, subIndex, subBits) {
10904
- const words = subBitsetView(composite, layout, subIndex).length;
10905
- if (words === 0) return;
10906
- composite.set(subBits.length > words ? subBits.subarray(0, words) : subBits, layout.bases[subIndex] >> 5);
10907
- }
10908
- function compositeFiltersBitset(subs, layout, filters, schema, meter) {
10909
- const composite = new Uint32Array(Math.ceil(layout.totalCapacity / WORD_BITS));
10910
- for (let i = 0; i < subs.length; i++) {
10911
- placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema, meter));
10912
- }
10913
- return composite;
10914
- }
10915
- function compositeFilters(subs, filters, schema, meter) {
10916
- const combined = /* @__PURE__ */ new Set();
10917
- for (const sub of subs) {
10918
- for (const docId of applyPartitionFilters(sub, filters, schema, meter)) {
10919
- combined.add(docId);
10879
+ for (const docId of partitions[partitionId].docIds()) {
10880
+ docPartitionMap.set(docId, partitionId);
10920
10881
  }
10921
10882
  }
10922
- return combined;
10923
- }
10924
- function compositeFilterMatches(subs, layout, filters, schema, meter) {
10925
- const bits = compositeFiltersBitset(subs, layout, filters, schema, meter);
10926
- let count = 0;
10927
- for (let i = 0; i < subs.length; i++) {
10928
- const resolver = subs[i].docStore.resolver();
10929
- const view = subBitsetView(bits, layout, i);
10930
- for (let wordIndex = 0; wordIndex < view.length; wordIndex++) {
10931
- let word = view[wordIndex];
10932
- if (word === 0) continue;
10933
- const base = wordIndex << 5;
10934
- while (word !== 0) {
10935
- const trailingZeros = Math.clz32(word & -word) ^ 31;
10936
- if (resolver.isLive(base + trailingZeros)) count++;
10937
- word &= word - 1;
10883
+ const manager = {
10884
+ get partitionCount() {
10885
+ return partitions.length;
10886
+ },
10887
+ get indexName() {
10888
+ return indexName;
10889
+ },
10890
+ get schema() {
10891
+ return config.schema;
10892
+ },
10893
+ get language() {
10894
+ return language;
10895
+ },
10896
+ get config() {
10897
+ return config;
10898
+ },
10899
+ get analysis() {
10900
+ return analysis;
10901
+ },
10902
+ get partialWordAnalysis() {
10903
+ return partialWordAnalysis;
10904
+ },
10905
+ synonyms: null,
10906
+ get textAnalysis() {
10907
+ return textAnalysis;
10908
+ },
10909
+ getPartition(partitionId) {
10910
+ validatePartitionId(partitionId);
10911
+ return partitions[partitionId];
10912
+ },
10913
+ partitionAt(index) {
10914
+ return partitions[index];
10915
+ },
10916
+ getAllPartitions() {
10917
+ return [...partitions];
10918
+ },
10919
+ setPartitions(newPartitions) {
10920
+ writeLog.partitionsReplaced();
10921
+ partitions = [...newPartitions];
10922
+ rebuildDocPartitionMap();
10923
+ },
10924
+ openWriteLog() {
10925
+ return writeLog.open();
10926
+ },
10927
+ adoptRebuiltPartitions(rebuilt) {
10928
+ if (rebuilt.length !== partitions.length) {
10929
+ throw new NarsilError(
10930
+ ErrorCodes.PARTITION_CORRUPTED,
10931
+ `Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
10932
+ { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
10933
+ );
10938
10934
  }
10939
- }
10940
- }
10941
- return {
10942
- count,
10943
- hasExternal(docId) {
10944
- for (let i = 0; i < subs.length; i++) {
10945
- const internalId = subs[i].docStore.getInternalId(docId);
10946
- if (internalId !== void 0) return bitsetHas(bits, layout.bases[i] + internalId);
10935
+ partitions = [...rebuilt];
10936
+ },
10937
+ addPartition() {
10938
+ const currentMaxPartitions = config.partitions?.maxPartitions;
10939
+ if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
10940
+ throw new NarsilError(
10941
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
10942
+ `Cannot add partition: maximum of ${currentMaxPartitions} partitions reached`,
10943
+ { maxPartitions: currentMaxPartitions, currentCount: partitions.length }
10944
+ );
10947
10945
  }
10948
- return false;
10946
+ const partition = createPartitionIndex(partitions.length, trackPositions);
10947
+ writeLog.partitionsReplaced();
10948
+ partitions.push(partition);
10949
+ return partition;
10949
10950
  },
10950
- hasInternal(internalId) {
10951
- return bitsetHas(bits, internalId);
10952
- }
10953
- };
10954
- }
10955
- function subMatchesView(matches, layout, subIndex) {
10956
- const base = layout.bases[subIndex];
10957
- return {
10958
- count: matches.count,
10959
- hasExternal: (docId) => matches.hasExternal(docId),
10960
- hasInternal: (internalId) => matches.hasInternal(base + internalId)
10961
- };
10962
- }
10963
-
10964
- // src/core/partition/composite/stats.ts
10965
- function collectCandidateTerms(subs, params) {
10966
- const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
10967
- const terms = /* @__PURE__ */ new Set();
10968
- for (const qt of queryTokens) {
10969
- const group = termGroupOf(termGroups, qt.token);
10970
- if (group !== void 0) {
10971
- for (const member of group.members) terms.add(member.token);
10972
- }
10973
- if (group === void 0 && (exact || tolerance === 0)) {
10974
- terms.add(qt.token);
10975
- continue;
10976
- }
10977
- for (const sub of subs) {
10978
- for (const match of typoMatchesFor(sub.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
10979
- terms.add(match.token);
10951
+ removePartition(partitionId) {
10952
+ validatePartitionId(partitionId);
10953
+ writeLog.partitionsReplaced();
10954
+ partitions.splice(partitionId, 1);
10955
+ rebuildDocPartitionMap();
10956
+ },
10957
+ trimPartitions(newCount) {
10958
+ if (newCount < 1) {
10959
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Cannot trim index "${indexName}" below one partition`, {
10960
+ indexName,
10961
+ requestedCount: newCount
10962
+ });
10980
10963
  }
10981
- }
10982
- }
10983
- return terms;
10984
- }
10985
- function resolveQueryTermFrequencies(subs, params) {
10986
- const frequencies = /* @__PURE__ */ Object.create(null);
10987
- for (const term of collectCandidateTerms(subs, params)) {
10988
- const total = partitionDocumentFrequency(subs, term);
10989
- if (total > 0) frequencies[term] = total;
10990
- }
10991
- return frequencies;
10992
- }
10993
- function aggregateFieldStats(subs) {
10994
- let totalDocuments = 0;
10995
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
10996
- for (const sub of subs) {
10997
- totalDocuments += sub.stats.totalDocuments;
10998
- for (const [field, length] of Object.entries(sub.stats.totalFieldLengths)) {
10999
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
11000
- }
11001
- }
11002
- const averageFieldLengths = /* @__PURE__ */ Object.create(null);
11003
- if (totalDocuments > 0) {
11004
- for (const [field, length] of Object.entries(totalFieldLengths)) {
11005
- averageFieldLengths[field] = length / totalDocuments;
11006
- }
11007
- }
11008
- return { totalDocuments, totalFieldLengths, averageFieldLengths };
11009
- }
11010
- function mergeDocFrequencies(subs) {
11011
- const merged = /* @__PURE__ */ Object.create(null);
11012
- for (const sub of subs) {
11013
- for (const [term, frequency] of Object.entries(sub.stats.docFrequencies)) {
11014
- merged[term] = (merged[term] ?? 0) + frequency;
11015
- }
11016
- }
11017
- return merged;
11018
- }
11019
- function buildAggregateStatsView(subs, docFrequencies) {
11020
- const fields = aggregateFieldStats(subs);
11021
- return {
11022
- ...fields,
11023
- get docFrequencies() {
11024
- return docFrequencies();
11025
- }
11026
- };
11027
- }
11028
- function compositeQueryStats(subs, params) {
11029
- const resolved = resolveQueryTermFrequencies(subs, params);
11030
- const caller = params.globalStats;
11031
- if (caller === void 0) {
11032
- const fields = aggregateFieldStats(subs);
11033
- return { ...fields, docFrequencies: resolved };
11034
- }
11035
- return {
11036
- totalDocuments: caller.totalDocuments,
11037
- totalFieldLengths: caller.totalFieldLengths,
11038
- averageFieldLengths: caller.averageFieldLengths,
11039
- docFrequencies: { ...resolved, ...caller.docFrequencies }
11040
- };
11041
- }
11042
-
11043
- // src/core/partition/composite/search.ts
11044
- function subParams(params, layout, subIndex, globalStats) {
11045
- return {
11046
- ...params,
11047
- globalStats,
11048
- filterBitset: params.filterBitset === void 0 ? void 0 : subBitsetView(params.filterBitset, layout, subIndex)
11049
- };
11050
- }
11051
- function onlyPopulatedSub(subs) {
11052
- let found = -1;
11053
- for (let i = 0; i < subs.length; i++) {
11054
- if (subs[i].stats.totalDocuments === 0) continue;
11055
- if (found !== -1) return -1;
11056
- found = i;
11057
- }
11058
- return found;
11059
- }
11060
- function searchSingleSub(subs, layout, params, index) {
11061
- const result = searchFulltext(subs[index], subParams(params, layout, index, params.globalStats));
11062
- if (result.matchedOrdinalBitset === void 0) return result;
11063
- const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
11064
- placeSubBitset(matchedOrdinalBitset, layout, index, result.matchedOrdinalBitset);
11065
- return { ...result, matchedOrdinalBitset };
11066
- }
11067
- function compositeSearchFulltext(subs, layout, params) {
11068
- if (params.queryTokens.length === 0) {
11069
- return { scored: [], totalMatched: 0 };
11070
- }
11071
- const single = onlyPopulatedSub(subs);
11072
- if (single !== -1) {
11073
- return searchSingleSub(subs, layout, params, single);
11074
- }
11075
- const globalStats = compositeQueryStats(subs, params);
11076
- const results = subs.map((sub, index) => searchFulltext(sub, subParams(params, layout, index, globalStats)));
11077
- const scored = kWayMerge(results.map((result) => result.scored));
11078
- if (params.maxResults !== void 0 && scored.length > params.maxResults) {
11079
- scored.length = params.maxResults;
11080
- }
11081
- let totalMatched = 0;
11082
- for (const result of results) {
11083
- totalMatched += result.totalMatched;
11084
- }
11085
- if (params.collectMatchedSet === void 0) {
11086
- return { scored, totalMatched };
11087
- }
11088
- if (params.collectMatchedSet === "ordinals") {
11089
- const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
11090
- for (let i = 0; i < results.length; i++) {
11091
- const subBitset = results[i].matchedOrdinalBitset;
11092
- if (subBitset !== void 0) placeSubBitset(matchedOrdinalBitset, layout, i, subBitset);
11093
- }
11094
- return { scored, totalMatched, matchedOrdinalBitset };
11095
- }
11096
- const matchedIds = [];
11097
- for (const result of results) {
11098
- if (result.matchedIds !== void 0) matchedIds.push(...result.matchedIds);
11099
- }
11100
- return { scored, totalMatched, matchedIds };
11101
- }
11102
- function compositeSearchMatches(subs, layout, params) {
11103
- const subMatches = subs.map((sub, index) => searchFulltextMatches(sub, subParams(params, layout, index, void 0)));
11104
- let count = 0;
11105
- for (const matches of subMatches) {
11106
- count += matches.count;
11107
- }
11108
- return {
11109
- count,
11110
- hasExternal(docId) {
11111
- for (const matches of subMatches) {
11112
- if (matches.hasExternal(docId)) return true;
10964
+ if (newCount >= partitions.length) {
10965
+ return;
11113
10966
  }
11114
- return false;
11115
- },
11116
- hasInternal(internalId) {
11117
- for (let i = subMatches.length - 1; i >= 0; i--) {
11118
- if (internalId >= layout.bases[i]) {
11119
- return subMatches[i].hasInternal(internalId - layout.bases[i]);
10967
+ writeLog.partitionsReplaced();
10968
+ for (const [docId, pid] of docPartitionMap) {
10969
+ if (pid >= newCount) {
10970
+ docPartitionMap.delete(docId);
11120
10971
  }
11121
10972
  }
11122
- return false;
10973
+ partitions.length = newCount;
11123
10974
  },
11124
- matchedDocIds() {
11125
- const combined = /* @__PURE__ */ new Set();
11126
- for (const matches of subMatches) {
11127
- for (const docId of matches.matchedDocIds()) {
11128
- combined.add(docId);
11129
- }
10975
+ ...createCapacityChecks({
10976
+ indexName,
10977
+ config,
10978
+ partitionCount: () => partitions.length,
10979
+ documentCount: () => manager.countDocuments(),
10980
+ getPartition: (partitionId) => manager.getPartition(partitionId)
10981
+ }),
10982
+ routePartition(docId) {
10983
+ return router.route(docId, partitions.length);
10984
+ },
10985
+ insert(docId, document, options) {
10986
+ const pid = router.route(docId, partitions.length);
10987
+ const insertOpts = resolveInsertOptions(options);
10988
+ partitions[pid].insert(docId, document, config.schema, language, insertOpts);
10989
+ docPartitionMap.set(docId, pid);
10990
+ writeLog.documentWritten(docId);
10991
+ },
10992
+ removedSpellingsOf(docId) {
10993
+ const pid = locateDocument(docId);
10994
+ const partition = pid === void 0 ? void 0 : partitions[pid];
10995
+ return partition !== void 0 && isCompositePartition(partition) ? partition.removedSpellingsOf(docId) : void 0;
10996
+ },
10997
+ remove(docId, removedSpellings) {
10998
+ const pid = locateDocument(docId);
10999
+ if (pid === void 0) {
11000
+ throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
11130
11001
  }
11131
- return combined;
11002
+ partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
11003
+ docPartitionMap.delete(docId);
11004
+ writeLog.documentWritten(docId);
11132
11005
  },
11133
- ordinalBitset() {
11134
- const combined = createBitSet(layout.totalCapacity);
11135
- for (let i = 0; i < subMatches.length; i++) {
11136
- placeSubBitset(combined, layout, i, subMatches[i].ordinalBitset());
11006
+ beginBatchRemove() {
11007
+ for (let i = 0; i < partitions.length; i++) {
11008
+ partitions[i].beginBatch();
11137
11009
  }
11138
- return combined;
11139
- }
11140
- };
11141
- }
11142
-
11143
- // src/core/partition/composite/sorting.ts
11144
- function compareEntries(a, b, directions) {
11145
- for (let i = 0; i < directions.length; i++) {
11146
- const comparison = compareComparableValues(a.key[i] ?? null, b.key[i] ?? null, directions[i]);
11147
- if (comparison !== 0) return comparison;
11148
- }
11149
- return compareCodePoints(a.id, b.id);
11150
- }
11151
- function compositeSortedPage(subs, layout, request) {
11152
- const pages = subs.map(
11153
- (sub, index) => sortedPageOf(sub, {
11154
- ...request,
11155
- matches: request.matches === null ? null : subMatchesView(request.matches, layout, index)
11156
- })
11157
- );
11158
- const merged = pages.flat();
11159
- merged.sort((a, b) => compareEntries(a, b, request.directions));
11160
- if (merged.length > request.limit) merged.length = request.limit;
11161
- return merged;
11162
- }
11163
- function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
11164
- for (const sub of subs) {
11165
- if (sub.docStore.has(docId)) {
11166
- return sortValuesOf(sub, docId, fields, fieldTypes, modes);
11167
- }
11168
- }
11169
- return fields.map(() => null);
11170
- }
11171
-
11172
- // src/core/partition/composite/suggest.ts
11173
- function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
11174
- if (limit <= 0) return [];
11175
- const merged = /* @__PURE__ */ new Map();
11176
- for (const sub of subs) {
11177
- for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, Number.MAX_SAFE_INTEGER)) {
11178
- let entry = merged.get(suggestion.token);
11179
- if (entry === void 0) {
11180
- entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
11181
- merged.set(suggestion.token, entry);
11010
+ },
11011
+ endBatchRemove() {
11012
+ for (let i = 0; i < partitions.length; i++) {
11013
+ partitions[i].endBatch();
11182
11014
  }
11183
- entry.documentFrequency += suggestion.documentFrequency;
11184
- for (const surface of suggestion.surfaces) {
11185
- entry.surfaceOccurrences.set(
11186
- surface.surface,
11187
- (entry.surfaceOccurrences.get(surface.surface) ?? 0) + surface.occurrences
11188
- );
11015
+ },
11016
+ update(docId, document, options, removedSpellings) {
11017
+ const pid = locateDocument(docId);
11018
+ if (pid === void 0) {
11019
+ throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
11189
11020
  }
11190
- }
11191
- }
11192
- const suggestions = [];
11193
- for (const [token, entry] of merged) {
11194
- const surfaces = Array.from(entry.surfaceOccurrences, ([surface, occurrences]) => ({ surface, occurrences }));
11195
- surfaces.sort((a, b) => b.occurrences - a.occurrences || compareCodePoints(a.surface, b.surface));
11196
- suggestions.push({ token, documentFrequency: entry.documentFrequency, surfaces });
11197
- }
11198
- suggestions.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.token, b.token));
11199
- if (suggestions.length > limit) suggestions.length = limit;
11200
- return suggestions;
11201
- }
11202
-
11203
- // src/core/partition/composite/index.ts
11204
- function isCompositePartition(partition) {
11205
- return "attachFrozenSegment" in partition;
11206
- }
11207
- function createCompositePartition(partitionId, trackPositions = true, existingLive, textAnalysis) {
11208
- const live = existingLive ?? createPartitionIndex(partitionId, trackPositions);
11209
- const liveState = partitionStateOf(live);
11210
- const frozen = [];
11211
- let docFrequenciesCache = null;
11212
- function invalidateDocFrequencies() {
11213
- docFrequenciesCache = null;
11214
- }
11215
- function bindSegments() {
11216
- invalidateDocFrequencies();
11217
- if (textAnalysis === void 0) return;
11218
- for (const segment of frozen) segment.removals.bind(textAnalysis);
11219
- }
11220
- function cachedDocFrequencies() {
11221
- if (docFrequenciesCache === null) {
11222
- docFrequenciesCache = mergeDocFrequencies(subs());
11223
- }
11224
- return docFrequenciesCache;
11225
- }
11226
- function subs() {
11227
- return [...frozen, liveState];
11228
- }
11229
- function composedScratch() {
11230
- const scratch = createPartitionIndex(partitionId, trackPositions);
11231
- for (const sub of subs()) {
11232
- const payload = encodeSegmentState(sub);
11233
- if (payload.documentCount === 0) continue;
11234
- scratch.mergeSegmentPayload(payload, survivorDocumentsOf(sub, payload));
11235
- }
11236
- return scratch;
11237
- }
11238
- function layout() {
11239
- return computeOrdinalLayout(subs());
11240
- }
11241
- function frozenOwner(docId) {
11242
- for (const segment of frozen) {
11243
- if (segment.hasDocument(docId)) return segment;
11244
- }
11245
- return void 0;
11246
- }
11247
- function frozenStored(docId) {
11248
- for (const segment of frozen) {
11249
- const stored = segment.docStore.get(docId);
11250
- if (stored !== void 0) return stored;
11251
- }
11252
- return void 0;
11253
- }
11254
- return {
11255
- get partitionId() {
11256
- return partitionId;
11257
- },
11258
- get live() {
11259
- return live;
11260
- },
11261
- get stats() {
11262
- return buildAggregateStatsView([...frozen, liveState], cachedDocFrequencies);
11263
- },
11264
- frozenSegmentCount() {
11265
- return frozen.length;
11266
- },
11267
- frozenSegmentSizes() {
11268
- return frozen.map((segment) => ({ segmentId: segment.segmentId, liveDocumentCount: segment.liveDocumentCount() }));
11269
- },
11270
- frozenSegmentsById(segmentIds) {
11271
- return resolveFrozenSegments(frozen, segmentIds, partitionId);
11272
- },
11273
- swapFrozenSegments(dropSegmentIds, replacement, removedSpellings) {
11274
- if (textAnalysis !== void 0) replacement.removals.bind(textAnalysis);
11275
- const applied = swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings);
11276
- bindSegments();
11277
- return applied;
11278
- },
11279
- freezeLiveTail(freeze) {
11280
- const segment = freezeLiveTailInto(live, liveState, frozen, freeze);
11281
- if (segment !== null) bindSegments();
11282
- return segment;
11283
- },
11284
- replaceLiveTail(segment) {
11285
- live.clear();
11286
- frozen.push(segment);
11287
- bindSegments();
11288
- },
11289
- attachFrozenSegment(segment) {
11290
- frozen.push(segment);
11291
- bindSegments();
11021
+ const updateOpts = resolveInsertOptions(options);
11022
+ partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
11023
+ docPartitionMap.set(docId, pid);
11024
+ writeLog.documentWritten(docId);
11292
11025
  },
11293
- mergeSegment(segment) {
11294
- invalidateDocFrequencies();
11295
- live.mergeSegment(segment);
11026
+ get(docId, projection) {
11027
+ const pid = locateDocument(docId);
11028
+ if (pid === void 0) return void 0;
11029
+ const doc = partitions[pid].get(docId, projection);
11030
+ if (doc) copyVectorsInto(doc, docId, vecIndexes, projection);
11031
+ return doc;
11296
11032
  },
11297
- mergeSegmentPayload(payload, documents) {
11298
- invalidateDocFrequencies();
11299
- live.mergeSegmentPayload(payload, documents);
11033
+ getRef(docId) {
11034
+ const pid = locateDocument(docId);
11035
+ if (pid === void 0) return void 0;
11036
+ return partitions[pid].getRef(docId);
11300
11037
  },
11301
- encodeSegment() {
11302
- if (frozen.length === 0) return live.encodeSegment();
11303
- return composedScratch().encodeSegment();
11038
+ fieldValues(docId, fieldPaths) {
11039
+ const pid = locateDocument(docId);
11040
+ return pid === void 0 ? void 0 : partitions[pid].fieldValues(docId, fieldPaths);
11304
11041
  },
11305
- serialize(indexName, totalPartitions, language, schema) {
11306
- if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
11307
- return composedScratch().serialize(indexName, totalPartitions, language, schema);
11042
+ sortValues(docId, fields, fieldTypes, modes) {
11043
+ const pid = locateDocument(docId);
11044
+ if (pid === void 0) return fields.map(() => null);
11045
+ return partitions[pid].sortValues(docId, fields, fieldTypes, modes);
11308
11046
  },
11309
- deserialize(data, schema) {
11310
- frozen.length = 0;
11311
- invalidateDocFrequencies();
11312
- live.deserialize(data, schema);
11047
+ has(docId) {
11048
+ return locateDocument(docId) !== void 0;
11313
11049
  },
11314
- clear() {
11315
- frozen.length = 0;
11316
- invalidateDocFrequencies();
11317
- live.clear();
11050
+ partitionIdOf(docId) {
11051
+ return locateDocument(docId);
11318
11052
  },
11319
- insert(docId, document, schema, language, options) {
11320
- if (frozenOwner(docId) !== void 0) {
11321
- throw new NarsilError(
11322
- ErrorCodes.DOC_ALREADY_EXISTS,
11323
- `Document "${docId}" already exists in partition ${partitionId}`,
11324
- { docId, partitionId }
11325
- );
11053
+ countDocuments() {
11054
+ let total = 0;
11055
+ for (let i = 0; i < partitions.length; i++) {
11056
+ total += partitions[i].count();
11326
11057
  }
11327
- invalidateDocFrequencies();
11328
- live.insert(docId, document, schema, language, options);
11058
+ return total;
11329
11059
  },
11330
- update(docId, document, schema, language, options, removedSpellings) {
11331
- invalidateDocFrequencies();
11332
- const owner = frozenOwner(docId);
11333
- if (owner === void 0) {
11334
- live.update(docId, document, schema, language, options);
11335
- return;
11336
- }
11337
- live.insert(docId, document, schema, language, options);
11338
- owner.tombstoneDocument(docId, removedSpellings);
11060
+ serializePartition(partitionId) {
11061
+ validatePartitionId(partitionId);
11062
+ return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
11339
11063
  },
11340
- remove(docId, schema, language, options, removedSpellings) {
11341
- if (live.has(docId)) {
11342
- invalidateDocFrequencies();
11343
- live.remove(docId, schema, language, options);
11344
- return;
11345
- }
11346
- const owner = frozenOwner(docId);
11347
- if (owner === void 0) {
11348
- throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in partition ${partitionId}`, {
11349
- docId,
11350
- partitionId
11351
- });
11064
+ mergeSegment(partitionId, payload, documents) {
11065
+ validatePartitionId(partitionId);
11066
+ const partition = partitions[partitionId];
11067
+ partition.beginBatch();
11068
+ partition.mergeSegmentPayload(payload, documents);
11069
+ partition.endBatch();
11070
+ for (const docId of payload.docIds) {
11071
+ docPartitionMap.set(docId, partitionId);
11072
+ writeLog.documentWritten(docId);
11352
11073
  }
11353
- invalidateDocFrequencies();
11354
- owner.tombstoneDocument(docId, removedSpellings);
11355
- },
11356
- removedSpellingsOf(docId) {
11357
- return live.has(docId) ? void 0 : frozenOwner(docId)?.removedSpellingsOf(docId);
11358
- },
11359
- beginBatch() {
11360
- live.beginBatch();
11361
- },
11362
- endBatch() {
11363
- live.endBatch();
11364
- },
11365
- searchFulltext(params) {
11366
- return compositeSearchFulltext(subs(), layout(), params);
11367
- },
11368
- searchFulltextMatches(params) {
11369
- return compositeSearchMatches(subs(), layout(), params);
11370
- },
11371
- applyFilters(filters, schema, meter) {
11372
- return compositeFilters(subs(), filters, schema, meter);
11373
11074
  },
11374
- applyFiltersBitset(filters, schema, meter) {
11375
- return compositeFiltersBitset(subs(), layout(), filters, schema, meter);
11376
- },
11377
- filterMatches(filters, schema, meter) {
11378
- return compositeFilterMatches(subs(), layout(), filters, schema, meter);
11075
+ attachFrozenSegment(partitionId, segment) {
11076
+ validatePartitionId(partitionId);
11077
+ requireNewSegmentDocuments(segment, docPartitionMap);
11078
+ asCompositePartition(partitionId).attachFrozenSegment(segment);
11079
+ registerSegmentDocuments(partitionId, segment);
11080
+ writeLog.segmentWritten(segment);
11379
11081
  },
11380
- computeFacets(matched, config, schema) {
11381
- const everyValue = everyValueFacetConfig(config);
11382
- if ("ordinalBitset" in matched) {
11383
- const ordinalLayout = layout();
11384
- return mergeFacets(
11385
- subs().map(
11386
- (sub, index) => computeFacets(
11387
- sub,
11388
- { ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
11389
- everyValue,
11390
- schema
11391
- )
11392
- ),
11393
- config
11394
- );
11395
- }
11396
- return mergeFacets(
11397
- subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
11398
- config
11082
+ swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
11083
+ validatePartitionId(partitionId);
11084
+ const applied = asCompositePartition(partitionId).swapFrozenSegments(
11085
+ dropSegmentIds,
11086
+ replacement,
11087
+ removedSpellings
11399
11088
  );
11089
+ registerSegmentDocuments(partitionId, replacement);
11090
+ return applied;
11400
11091
  },
11401
- sortedPage(request) {
11402
- return compositeSortedPage(subs(), layout(), request);
11403
- },
11404
- sortValues(docId, fields, fieldTypes, modes) {
11405
- return compositeSortValues(subs(), docId, fields, fieldTypes, modes);
11406
- },
11407
- suggestTerms(surfacePrefix, stemmedPrefix, limit) {
11408
- return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
11409
- },
11410
- expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
11411
- return expandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
11092
+ freezeLiveTail(partitionId, freeze) {
11093
+ validatePartitionId(partitionId);
11094
+ return asCompositePartition(partitionId).freezeLiveTail(freeze);
11412
11095
  },
11413
- expandPartialWords(analysis, requests) {
11414
- return expandPartialWords(subs(), analysis, requests);
11096
+ replaceLiveTail(partitionId, segment) {
11097
+ validatePartitionId(partitionId);
11098
+ asCompositePartition(partitionId).replaceLiveTail(segment);
11099
+ registerSegmentDocuments(partitionId, segment);
11415
11100
  },
11416
- preparePartialWords(analysis) {
11417
- expandPartialWords(subs(), analysis, []);
11101
+ deserializePartition(partitionId, data) {
11102
+ validatePartitionId(partitionId);
11103
+ writeLog.partitionsReplaced();
11104
+ partitions[partitionId].deserialize(data, config.schema);
11105
+ preparePartialWords(partitionId);
11106
+ rebuildDocPartitionMapForPartition(partitionId);
11418
11107
  },
11419
- get(docId, projection) {
11420
- return compositeDocument(frozen, live, docId, projection);
11108
+ getAggregateStats() {
11109
+ return aggregateStatsOf(partitions);
11421
11110
  },
11422
- fieldValues(docId, fieldPaths) {
11423
- return compositeFieldValues(frozen, live, docId, fieldPaths);
11111
+ estimateMemoryBytes() {
11112
+ return estimateBytesOf(partitions, vecIndexes);
11424
11113
  },
11425
- getRef(docId) {
11426
- const stored = frozenStored(docId);
11427
- if (stored !== void 0) return stored.fields;
11428
- return live.getRef(docId);
11114
+ getPartitionStats() {
11115
+ return partitionStatsOf(partitions);
11429
11116
  },
11430
- has(docId) {
11431
- return live.has(docId) || frozenOwner(docId) !== void 0;
11117
+ getVectorIndexes() {
11118
+ return vecIndexes;
11432
11119
  },
11433
- count() {
11434
- let total = live.count();
11435
- for (const segment of frozen) {
11436
- total += segment.liveDocumentCount();
11120
+ resetVectorIndexes(newIndexes) {
11121
+ vecIndexes.clear();
11122
+ for (const [key, value] of newIndexes) {
11123
+ vecIndexes.set(key, value);
11437
11124
  }
11438
- return total;
11439
- },
11440
- *docIds() {
11441
- for (const segment of frozen) {
11442
- for (const internalId of segment.docStore.allInternalIds()) {
11443
- const docId = segment.docStore.getExternalId(internalId);
11444
- if (docId !== void 0) yield docId;
11445
- }
11446
- }
11447
- yield* live.docIds();
11448
- },
11449
- sortedDocIds() {
11450
- const merged = [];
11451
- for (const segment of frozen) {
11452
- merged.push(...segment.docStore.sortedDocIds());
11453
- }
11454
- merged.push(...live.sortedDocIds());
11455
- merged.sort(compareCodePoints);
11456
- return merged;
11125
+ }
11126
+ };
11127
+ return manager;
11128
+ }
11129
+
11130
+ // src/core/pattern-index/work-meter.ts
11131
+ var COUNTER_BYTES = 8;
11132
+ function capExceeded(cap) {
11133
+ return new NarsilError(
11134
+ ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
11135
+ `The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
11136
+ { cap }
11137
+ );
11138
+ }
11139
+ function localMeter(cap, spent, matcherStateBytes, flushUnits) {
11140
+ let total = spent;
11141
+ let sharedCount = null;
11142
+ const spentSoFar = () => sharedCount === null ? total : Number(Atomics.load(sharedCount, 0));
11143
+ const meter = {
11144
+ cap,
11145
+ matcherStateBytes,
11146
+ flushUnits,
11147
+ add(units) {
11148
+ if (units <= 0) return;
11149
+ if (sharedCount === null) total += units;
11150
+ else total = Number(Atomics.add(sharedCount, 0, BigInt(units))) + units;
11151
+ if (total > cap) throw capExceeded(cap);
11457
11152
  },
11458
- releaseSortedDocIds() {
11459
- for (const segment of frozen) {
11460
- segment.docStore.releaseSortedDocIds();
11461
- }
11462
- live.releaseSortedDocIds();
11153
+ remaining() {
11154
+ return cap - spentSoFar();
11463
11155
  },
11464
- estimateMemoryBytes() {
11465
- let total = live.estimateMemoryBytes();
11466
- for (const segment of frozen) {
11467
- total += frozenSegmentBytes(segment);
11156
+ fork(workerCount) {
11157
+ const copies = Math.max(1, workerCount);
11158
+ const seed = spentSoFar();
11159
+ let counter = null;
11160
+ if (typeof SharedArrayBuffer === "function") {
11161
+ counter = new SharedArrayBuffer(COUNTER_BYTES);
11162
+ new BigInt64Array(counter)[0] = BigInt(seed);
11468
11163
  }
11469
- return total;
11164
+ const shared2 = {
11165
+ cap,
11166
+ spent: seed,
11167
+ counter,
11168
+ matcherStateBytes: Math.floor(matcherStateBytes / copies),
11169
+ flushUnits: Math.max(1, Math.floor(flushUnits / copies))
11170
+ };
11171
+ return {
11172
+ shared: shared2,
11173
+ settle(reportedUnits) {
11174
+ if (counter === null) {
11175
+ meter.add(reportedUnits);
11176
+ return;
11177
+ }
11178
+ const unitsSinceFork = spentSoFar() - seed;
11179
+ const view = new BigInt64Array(counter);
11180
+ if (unitsSinceFork > 0) Atomics.add(view, 0, BigInt(unitsSinceFork));
11181
+ sharedCount = view;
11182
+ total = spentSoFar();
11183
+ if (total > cap) throw capExceeded(cap);
11184
+ }
11185
+ };
11470
11186
  }
11471
11187
  };
11188
+ return meter;
11472
11189
  }
11473
-
11474
- // src/partitioning/insert-options.ts
11475
- function resolvePartialWordAnalysis(config, language, analysis) {
11476
- const fields = partialFieldsOf(config.schema);
11477
- if (fields.length === 0) return null;
11478
- return { fields, language, options: resolvePartitionInsertOptions(config, analysis) };
11479
- }
11480
- function resolvePartitionInsertOptions(config, analysis, options) {
11481
- const applyStrict = config.strict === true;
11482
- const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
11483
- const applySurfaces = config.surfaceForms !== false;
11484
- const applyPatternLimit = config.patternValueLimit !== void 0;
11485
- if (!applyStrict && !applyAnalyzer && !applySurfaces && !applyPatternLimit) return options;
11486
- const resolved = { ...options };
11487
- if (applyStrict) resolved.strict = true;
11488
- if (applyPatternLimit) resolved.patternValueLimit = config.patternValueLimit;
11489
- if (applyAnalyzer) {
11490
- resolved.stopWordOverride = options?.stopWordOverride ?? analysis.stopWords;
11491
- resolved.customTokenizer = options?.customTokenizer ?? analysis.customTokenizer;
11492
- }
11493
- if (applySurfaces) resolved.collectSurfaces = true;
11494
- return resolved;
11190
+ function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP) {
11191
+ return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS);
11495
11192
  }
11496
-
11497
- // src/partitioning/manager/capacity.ts
11498
- function createCapacityChecks(source) {
11499
- const { indexName, config } = source;
11193
+ function sharedMeter(work, counter) {
11194
+ const view = new BigInt64Array(counter);
11500
11195
  return {
11501
- assertCapacity(pendingWrites = 0, partitionCountCap) {
11502
- const maxDocsPerPartition = config.partitions?.maxDocsPerPartition;
11503
- if (maxDocsPerPartition === void 0) return;
11504
- const partitionCount = source.partitionCount();
11505
- const effectivePartitionCount = partitionCountCap === void 0 ? partitionCount : Math.min(partitionCount, partitionCountCap);
11506
- const totalCapacity = maxDocsPerPartition * effectivePartitionCount;
11507
- const currentCount = source.documentCount() + pendingWrites;
11508
- if (currentCount < totalCapacity) return;
11509
- throw new NarsilError(
11510
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
11511
- `Index "${indexName}" has reached its capacity of ${totalCapacity} documents (${maxDocsPerPartition} per partition \xD7 ${effectivePartitionCount} partitions)`,
11512
- {
11513
- indexName,
11514
- currentCount,
11515
- totalCapacity,
11516
- maxDocsPerPartition,
11517
- partitionCount: effectivePartitionCount
11518
- }
11519
- );
11196
+ matcherStateBytes: work.matcherStateBytes,
11197
+ flushUnits: work.flushUnits,
11198
+ add(units) {
11199
+ if (units <= 0) return;
11200
+ const total = Number(Atomics.add(view, 0, BigInt(units))) + units;
11201
+ if (total > work.cap) throw capExceeded(work.cap);
11520
11202
  },
11521
- assertPartitionCapacity(partitionId, pendingWrites = 0) {
11522
- const maxDocsPerPartition = config.partitions?.maxDocsPerPartition;
11523
- if (maxDocsPerPartition === void 0) return;
11524
- const currentCount = source.getPartition(partitionId).count() + pendingWrites;
11525
- if (currentCount < maxDocsPerPartition) return;
11526
- throw new NarsilError(
11527
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
11528
- `Partition ${partitionId} of index "${indexName}" has reached its capacity of ${maxDocsPerPartition} documents`,
11529
- { indexName, partitionId, currentCount, maxDocsPerPartition }
11530
- );
11203
+ remaining() {
11204
+ return work.cap - Number(Atomics.load(view, 0));
11531
11205
  }
11532
11206
  };
11533
11207
  }
11208
+ function joinPatternWorkMeter(work) {
11209
+ const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits) : sharedMeter(work, work.counter);
11210
+ let spent = 0;
11211
+ return {
11212
+ matcherStateBytes: meter.matcherStateBytes,
11213
+ flushUnits: meter.flushUnits,
11214
+ add(units) {
11215
+ if (units <= 0) return;
11216
+ spent += units;
11217
+ meter.add(units);
11218
+ },
11219
+ remaining: () => meter.remaining(),
11220
+ spent: () => spent
11221
+ };
11222
+ }
11534
11223
 
11535
- // src/partitioning/manager/nested-values.ts
11536
- function setNestedValue(obj, path, value) {
11537
- if (!path.includes(".")) {
11538
- obj[path] = value;
11539
- return;
11540
- }
11541
- const segments = path.split(".");
11542
- let current = obj;
11543
- for (let i = 0; i < segments.length - 1; i++) {
11544
- let next = current[segments[i]];
11545
- if (next === null || next === void 0 || typeof next !== "object") {
11546
- next = {};
11547
- current[segments[i]] = next;
11548
- }
11549
- current = next;
11224
+ // src/search/term-groups.ts
11225
+ function writtenWordsByPosition(term, language, analysis) {
11226
+ const unstemmed = tokenize(term, language, {
11227
+ stem: false,
11228
+ removeStopWords: true,
11229
+ customTokenizer: analysis?.customTokenizer,
11230
+ stopWordOverride: analysis?.stopWords
11231
+ });
11232
+ const written = /* @__PURE__ */ new Map();
11233
+ for (const token of unstemmed.tokens) {
11234
+ if (!written.has(token.position)) written.set(token.position, token.token);
11550
11235
  }
11551
- current[segments[segments.length - 1]] = value;
11236
+ return written;
11552
11237
  }
11553
- function copyVectorsInto(document, docId, vectorIndexes, projection) {
11554
- for (const [fieldPath, vectorIndex] of vectorIndexes) {
11555
- if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
11556
- const vector = vectorIndex.getVector(docId);
11557
- if (vector) setNestedValue(document, fieldPath, vector);
11238
+ function groupFor(groups, token, typoMatches) {
11239
+ let group = groups.get(token);
11240
+ if (group === void 0) {
11241
+ group = { token, members: [{ token, factor: 1, partialFactor: 1 }], typoMatches };
11242
+ groups.set(token, group);
11558
11243
  }
11244
+ return group;
11559
11245
  }
11560
-
11561
- // src/partitioning/manager/segments.ts
11562
- function requireNewSegmentDocuments(segment, docPartitionMap) {
11563
- for (const internalId of segment.docStore.allInternalIds()) {
11564
- const docId = segment.docStore.getExternalId(internalId);
11565
- if (docId === void 0 || !docPartitionMap.has(docId)) continue;
11566
- throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
11567
- docId,
11568
- partitionId: docPartitionMap.get(docId)
11569
- });
11246
+ function addCompletions(groups, partition, lastToken, typedPrefix) {
11247
+ if (typedPrefix === void 0 || typedPrefix.length === 0) return;
11248
+ const group = groupFor(groups, lastToken.token, false);
11249
+ const ownLength = codePointCount(lastToken.token);
11250
+ for (const completion of partition.expandTermPrefix(typedPrefix, lastToken.token, PREFIX_MAX_EXPANSIONS)) {
11251
+ if (completion === lastToken.token) continue;
11252
+ const factor = Math.min(1, ownLength / codePointCount(completion));
11253
+ group.members.push({ token: completion, factor, partialFactor: factor });
11570
11254
  }
11571
11255
  }
11572
-
11573
- // src/partitioning/manager/stats.ts
11574
- function aggregateStatsOf(partitions) {
11575
- let totalDocuments = 0;
11576
- const docFrequencies = /* @__PURE__ */ Object.create(null);
11577
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
11578
- for (let i = 0; i < partitions.length; i++) {
11579
- const stats = partitions[i].stats;
11580
- totalDocuments += stats.totalDocuments;
11581
- for (const [term, freq] of Object.entries(stats.docFrequencies)) {
11582
- docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
11583
- }
11584
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
11585
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
11256
+ function partialWordAnalysisOf(fields, language, analysis) {
11257
+ return {
11258
+ fields,
11259
+ language,
11260
+ options: {
11261
+ stopWordOverride: analysis?.stopWords,
11262
+ customTokenizer: analysis?.customTokenizer,
11263
+ collectSurfaces: true
11586
11264
  }
11587
- }
11588
- return { totalDocuments, docFrequencies, totalFieldLengths };
11265
+ };
11589
11266
  }
11590
- function estimateBytesOf(partitions, vecIndexes) {
11591
- let total = 0;
11592
- for (let i = 0; i < partitions.length; i++) {
11593
- total += partitions[i].estimateMemoryBytes();
11594
- }
11595
- for (const [, vecIndex] of vecIndexes) {
11596
- total += vecIndex.estimateMemoryBytes();
11267
+ function searchesAPartialField(params, partialFields) {
11268
+ if (params.fields === void 0) return true;
11269
+ for (const field of params.fields) {
11270
+ if (partialFields.includes(field)) return true;
11597
11271
  }
11598
- return total;
11272
+ return false;
11599
11273
  }
11600
- function partitionStatsOf(partitions) {
11601
- return partitions.map((p, i) => ({
11602
- partitionId: i,
11603
- documentCount: p.count(),
11604
- estimatedMemoryBytes: p.estimateMemoryBytes()
11605
- }));
11274
+ function resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis, synonyms) {
11275
+ const term = params.term;
11276
+ if (term === void 0 || params.exact === true) return {};
11277
+ const usesPrefix = params.prefix === true;
11278
+ const partialFields = params.partialWords === true ? partialFieldsOf(schema) : [];
11279
+ const usesPartialWords = partialFields.length > 0 && searchesAPartialField(params, partialFields);
11280
+ if (!usesPrefix && !usesPartialWords && synonyms === void 0) return {};
11281
+ const written = usesPrefix || usesPartialWords ? writtenWordsByPosition(term, language, analysis) : /* @__PURE__ */ new Map();
11282
+ const groups = /* @__PURE__ */ new Map();
11283
+ if (usesPrefix) addCompletions(groups, partition, lastToken, written.get(lastToken.position));
11284
+ const prefixToken = groups.has(lastToken.token) ? lastToken.token : void 0;
11285
+ if (usesPartialWords) {
11286
+ addPartialWordMembers(
11287
+ {
11288
+ queryTokens,
11289
+ written,
11290
+ prefixToken,
11291
+ tolerance: params.tolerance ?? 0,
11292
+ prefixLength: params.prefixLength,
11293
+ analysis: partialWordAnalysisOf(partialFields, language, analysis)
11294
+ },
11295
+ (partialAnalysis, requests) => partition.expandPartialWords(partialAnalysis, requests),
11296
+ (token, typoMatches) => groupFor(groups, token, typoMatches)
11297
+ );
11298
+ }
11299
+ synonyms?.addTo((token) => groupFor(groups, token, true));
11300
+ const termGroups = [...groups.values()];
11301
+ if (termGroups.length === 0) return {};
11302
+ return usesPartialWords ? { termGroups, partialFields: [...partialFields] } : { termGroups };
11606
11303
  }
11607
11304
 
11608
- // src/partitioning/manager/write-log.ts
11609
- function createWriteLogRecorder() {
11610
- let current = null;
11611
- return {
11612
- open() {
11613
- const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
11614
- current = log;
11615
- return {
11616
- get partitionsReplaced() {
11617
- return log.replaced;
11618
- },
11619
- get pendingDocuments() {
11620
- return log.docIds.size;
11621
- },
11622
- takeWrittenDocIds() {
11623
- const taken = log.docIds;
11624
- log.docIds = /* @__PURE__ */ new Set();
11625
- return taken;
11626
- },
11627
- close() {
11628
- if (current === log) current = null;
11629
- }
11630
- };
11631
- },
11632
- documentWritten(docId) {
11633
- if (current !== null) current.docIds.add(docId);
11634
- },
11635
- segmentWritten(segment) {
11636
- if (current === null) return;
11637
- for (const internalId of segment.docStore.allInternalIds()) {
11638
- const docId = segment.docStore.getExternalId(internalId);
11639
- if (docId !== void 0) current.docIds.add(docId);
11640
- }
11641
- },
11642
- partitionsReplaced() {
11643
- if (current !== null) current.replaced = true;
11644
- }
11305
+ // src/partitioning/expanded-tokens.ts
11306
+ function queryTokensOf(term, language, analysis) {
11307
+ const { tokens } = tokenize(term, language, {
11308
+ stem: true,
11309
+ removeStopWords: true,
11310
+ customTokenizer: analysis.customTokenizer,
11311
+ stopWordOverride: analysis.stopWords
11312
+ });
11313
+ const seen = /* @__PURE__ */ new Set();
11314
+ const unique = [];
11315
+ for (const token of tokens) {
11316
+ if (seen.has(token.token)) continue;
11317
+ seen.add(token.token);
11318
+ unique.push(token);
11319
+ }
11320
+ return tokens.length === 0 ? null : { unique, last: tokens[tokens.length - 1] };
11321
+ }
11322
+ function readStatesOf(partition) {
11323
+ if (!isCompositePartition(partition)) return [partitionStateOf(partition)];
11324
+ const segmentIds = partition.frozenSegmentSizes().map((segment) => segment.segmentId);
11325
+ return [...partition.frozenSegmentsById(segmentIds), partitionStateOf(partition.live)];
11326
+ }
11327
+ function addPartitionTokens(tokens, addedMembers, partition, term, analysed, expansion, language, analysis) {
11328
+ const { options, schema } = expansion;
11329
+ const { unique: queryTokens, last: lastToken } = analysed;
11330
+ const tolerance = options.tolerance ?? 0;
11331
+ const prefixLength = options.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH;
11332
+ if (tolerance > 0) {
11333
+ const states = readStatesOf(partition);
11334
+ for (const { token } of queryTokens) {
11335
+ for (const state of states) {
11336
+ for (const match of state.invertedIdx.fuzzyLookup(token, tolerance, prefixLength)) tokens.add(match.token);
11337
+ }
11338
+ }
11339
+ }
11340
+ const params = {
11341
+ term,
11342
+ tolerance,
11343
+ prefixLength,
11344
+ prefix: options.prefix ?? void 0,
11345
+ partialWords: options.partialWords ?? void 0,
11346
+ fields: options.fields ?? void 0
11645
11347
  };
11348
+ const { termGroups } = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis);
11349
+ for (const group of termGroups ?? []) {
11350
+ for (const member of group.members) {
11351
+ if (member.token !== group.token) addedMembers.add(member.token);
11352
+ }
11353
+ }
11354
+ }
11355
+ function addReachedTokens(tokens, partitions, term, expansion, language, analysis) {
11356
+ const { options } = expansion;
11357
+ if (options.exact === true) return [];
11358
+ if (!((options.tolerance ?? 0) > 0 || options.prefix === true || options.partialWords === true)) return [];
11359
+ const analysed = queryTokensOf(term, language, analysis);
11360
+ if (analysed === null) return [];
11361
+ const addedMembers = /* @__PURE__ */ new Set();
11362
+ for (const partition of partitions) {
11363
+ addPartitionTokens(tokens, addedMembers, partition, term, analysed, expansion, language, analysis);
11364
+ }
11365
+ for (const token of addedMembers) tokens.add(token);
11366
+ return [...addedMembers];
11646
11367
  }
11647
11368
 
11648
- // src/partitioning/manager/index.ts
11649
- function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
11650
- requirePatternSearchFor(config.schema);
11651
- requirePartialWordSearchFor(config.schema);
11652
- const count = initialPartitionCount ?? 1;
11653
- const analysis = resolveIndexAnalysis(config);
11654
- const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
11655
- const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
11656
- const trackPositions = config.trackPositions ?? true;
11657
- const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
11658
- let partitions = [];
11659
- const docPartitionMap = /* @__PURE__ */ new Map();
11660
- const writeLog = createWriteLogRecorder();
11661
- for (let i = 0; i < count; i++) {
11662
- partitions.push(createPartitionIndex(i, trackPositions));
11369
+ // src/partitioning/partition-selection.ts
11370
+ function partitionsIn(manager, partitionIds) {
11371
+ if (partitionIds === void 0) {
11372
+ return manager.getAllPartitions();
11663
11373
  }
11664
- function validatePartitionId(partitionId) {
11665
- if (partitionId < 0 || partitionId >= partitions.length) {
11666
- throw new NarsilError(
11667
- ErrorCodes.INDEX_NOT_FOUND,
11668
- `Partition ${partitionId} is out of bounds (0..${partitions.length - 1})`,
11669
- { partitionId, partitionCount: partitions.length }
11670
- );
11374
+ const partitions = [];
11375
+ for (const partitionId of partitionIds) {
11376
+ const partition = manager.partitionAt(partitionId);
11377
+ if (partition !== void 0) {
11378
+ partitions.push(partition);
11671
11379
  }
11672
11380
  }
11673
- function resolveInsertOptions(options) {
11674
- return resolvePartitionInsertOptions(config, analysis, options);
11675
- }
11676
- function preparePartialWords(partitionId) {
11677
- if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
11381
+ return partitions;
11382
+ }
11383
+
11384
+ // src/partitioning/distributed-scoring.ts
11385
+ function analysedQueryTokens(term, language, analysis) {
11386
+ const { tokens } = tokenize(term, language, {
11387
+ stem: true,
11388
+ removeStopWords: true,
11389
+ customTokenizer: analysis.customTokenizer,
11390
+ stopWordOverride: analysis.stopWords
11391
+ });
11392
+ const unique = /* @__PURE__ */ new Set();
11393
+ for (const { token } of tokens) {
11394
+ unique.add(token);
11678
11395
  }
11679
- function locateDocument(docId) {
11680
- return docPartitionMap.get(docId);
11396
+ for (const synonym of analysis.synonyms?.expand(unique)?.tokens ?? []) unique.add(synonym);
11397
+ return unique;
11398
+ }
11399
+ function copyNumericRecord(source) {
11400
+ const copy = /* @__PURE__ */ Object.create(null);
11401
+ for (const [key, value] of Object.entries(source)) {
11402
+ if (typeof value === "number" && Number.isFinite(value)) {
11403
+ copy[key] = value;
11404
+ }
11681
11405
  }
11682
- function asCompositePartition(partitionId) {
11683
- const existing = partitions[partitionId];
11684
- if (isCompositePartition(existing)) return existing;
11685
- const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
11686
- partitions[partitionId] = composite;
11687
- return composite;
11406
+ return copy;
11407
+ }
11408
+ function sanitizeGlobalStats(stats) {
11409
+ return {
11410
+ totalDocuments: typeof stats.totalDocuments === "number" && Number.isFinite(stats.totalDocuments) ? stats.totalDocuments : 0,
11411
+ docFrequencies: copyNumericRecord(stats.docFrequencies),
11412
+ totalFieldLengths: copyNumericRecord(stats.totalFieldLengths),
11413
+ averageFieldLengths: copyNumericRecord(stats.averageFieldLengths)
11414
+ };
11415
+ }
11416
+ function pruneStatsToQueryTerms(stats, term, language, analysis, reached) {
11417
+ const tokens = analysedQueryTokens(term, language, analysis);
11418
+ if (reached !== void 0) addReachedTokens(tokens, reached.partitions, term, reached.expansion, language, analysis);
11419
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
11420
+ for (const token of tokens) {
11421
+ if (Object.hasOwn(stats.docFrequencies, token)) {
11422
+ docFrequencies[token] = stats.docFrequencies[token];
11423
+ }
11688
11424
  }
11689
- function rebuildDocPartitionMap() {
11690
- docPartitionMap.clear();
11691
- for (let i = 0; i < partitions.length; i++) {
11692
- for (const docId of partitions[i].docIds()) {
11693
- docPartitionMap.set(docId, i);
11425
+ return {
11426
+ totalDocuments: stats.totalDocuments,
11427
+ docFrequencies,
11428
+ totalFieldLengths: stats.totalFieldLengths,
11429
+ averageFieldLengths: stats.averageFieldLengths
11430
+ };
11431
+ }
11432
+ function collectQueryTermStats(manager, term, language, analysis, partitionIds, scope) {
11433
+ const queryTokens = analysedQueryTokens(term, language, analysis);
11434
+ let totalDocuments = 0;
11435
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
11436
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
11437
+ const statsPartitions = partitionsIn(manager, partitionIds);
11438
+ for (const token of scope?.tokens ?? []) queryTokens.add(token);
11439
+ const expanded = scope?.expansion === void 0 ? [] : addReachedTokens(queryTokens, statsPartitions, term, scope.expansion, language, analysis);
11440
+ for (const partition of statsPartitions) {
11441
+ const stats = partition.stats;
11442
+ totalDocuments += stats.totalDocuments;
11443
+ for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
11444
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
11445
+ }
11446
+ for (const token of queryTokens) {
11447
+ if (Object.hasOwn(stats.docFrequencies, token)) {
11448
+ docFrequencies[token] = (docFrequencies[token] ?? 0) + stats.docFrequencies[token];
11694
11449
  }
11695
11450
  }
11696
11451
  }
11697
- function registerSegmentDocuments(partitionId, segment) {
11698
- for (const internalId of segment.docStore.allInternalIds()) {
11699
- const docId = segment.docStore.getExternalId(internalId);
11700
- if (docId !== void 0) docPartitionMap.set(docId, partitionId);
11452
+ const averageFieldLengths = /* @__PURE__ */ Object.create(null);
11453
+ if (totalDocuments > 0) {
11454
+ for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
11455
+ averageFieldLengths[field] = totalLength / totalDocuments;
11701
11456
  }
11702
11457
  }
11703
- function rebuildDocPartitionMapForPartition(partitionId) {
11704
- for (const [docId, pid] of docPartitionMap) {
11705
- if (pid === partitionId) {
11706
- docPartitionMap.delete(docId);
11707
- }
11458
+ return { totalDocuments, docFrequencies, totalFieldLengths, averageFieldLengths, expanded };
11459
+ }
11460
+ function mergePartitionStats(statsArray) {
11461
+ let totalDocuments = 0;
11462
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
11463
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
11464
+ for (const stats of statsArray) {
11465
+ totalDocuments += stats.totalDocuments;
11466
+ for (const [term, freq] of Object.entries(stats.docFrequencies)) {
11467
+ docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
11708
11468
  }
11709
- for (const docId of partitions[partitionId].docIds()) {
11710
- docPartitionMap.set(docId, partitionId);
11469
+ for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
11470
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
11711
11471
  }
11712
11472
  }
11713
- const manager = {
11714
- get partitionCount() {
11715
- return partitions.length;
11716
- },
11717
- get indexName() {
11718
- return indexName;
11719
- },
11720
- get schema() {
11721
- return config.schema;
11722
- },
11723
- get language() {
11724
- return language;
11725
- },
11726
- get config() {
11727
- return config;
11728
- },
11729
- get analysis() {
11730
- return analysis;
11731
- },
11732
- get partialWordAnalysis() {
11733
- return partialWordAnalysis;
11734
- },
11735
- get textAnalysis() {
11736
- return textAnalysis;
11737
- },
11738
- getPartition(partitionId) {
11739
- validatePartitionId(partitionId);
11740
- return partitions[partitionId];
11741
- },
11742
- partitionAt(index) {
11743
- return partitions[index];
11744
- },
11745
- getAllPartitions() {
11746
- return [...partitions];
11747
- },
11748
- setPartitions(newPartitions) {
11749
- writeLog.partitionsReplaced();
11750
- partitions = [...newPartitions];
11751
- rebuildDocPartitionMap();
11752
- },
11753
- openWriteLog() {
11754
- return writeLog.open();
11755
- },
11756
- adoptRebuiltPartitions(rebuilt) {
11757
- if (rebuilt.length !== partitions.length) {
11758
- throw new NarsilError(
11759
- ErrorCodes.PARTITION_CORRUPTED,
11760
- `Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
11761
- { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
11762
- );
11763
- }
11764
- partitions = [...rebuilt];
11765
- },
11766
- addPartition() {
11767
- const currentMaxPartitions = config.partitions?.maxPartitions;
11768
- if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
11769
- throw new NarsilError(
11770
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
11771
- `Cannot add partition: maximum of ${currentMaxPartitions} partitions reached`,
11772
- { maxPartitions: currentMaxPartitions, currentCount: partitions.length }
11773
- );
11774
- }
11775
- const partition = createPartitionIndex(partitions.length, trackPositions);
11776
- writeLog.partitionsReplaced();
11777
- partitions.push(partition);
11778
- return partition;
11779
- },
11780
- removePartition(partitionId) {
11781
- validatePartitionId(partitionId);
11782
- writeLog.partitionsReplaced();
11783
- partitions.splice(partitionId, 1);
11784
- rebuildDocPartitionMap();
11785
- },
11786
- trimPartitions(newCount) {
11787
- if (newCount < 1) {
11788
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Cannot trim index "${indexName}" below one partition`, {
11789
- indexName,
11790
- requestedCount: newCount
11791
- });
11792
- }
11793
- if (newCount >= partitions.length) {
11794
- return;
11795
- }
11796
- writeLog.partitionsReplaced();
11797
- for (const [docId, pid] of docPartitionMap) {
11798
- if (pid >= newCount) {
11799
- docPartitionMap.delete(docId);
11800
- }
11801
- }
11802
- partitions.length = newCount;
11803
- },
11804
- ...createCapacityChecks({
11805
- indexName,
11806
- config,
11807
- partitionCount: () => partitions.length,
11808
- documentCount: () => manager.countDocuments(),
11809
- getPartition: (partitionId) => manager.getPartition(partitionId)
11810
- }),
11811
- routePartition(docId) {
11812
- return router.route(docId, partitions.length);
11813
- },
11814
- insert(docId, document, options) {
11815
- const pid = router.route(docId, partitions.length);
11816
- const insertOpts = resolveInsertOptions(options);
11817
- partitions[pid].insert(docId, document, config.schema, language, insertOpts);
11818
- docPartitionMap.set(docId, pid);
11819
- writeLog.documentWritten(docId);
11820
- },
11821
- removedSpellingsOf(docId) {
11822
- const pid = locateDocument(docId);
11823
- const partition = pid === void 0 ? void 0 : partitions[pid];
11824
- return partition !== void 0 && isCompositePartition(partition) ? partition.removedSpellingsOf(docId) : void 0;
11825
- },
11826
- remove(docId, removedSpellings) {
11827
- const pid = locateDocument(docId);
11828
- if (pid === void 0) {
11829
- throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
11830
- }
11831
- partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
11832
- docPartitionMap.delete(docId);
11833
- writeLog.documentWritten(docId);
11834
- },
11835
- beginBatchRemove() {
11836
- for (let i = 0; i < partitions.length; i++) {
11837
- partitions[i].beginBatch();
11838
- }
11839
- },
11840
- endBatchRemove() {
11841
- for (let i = 0; i < partitions.length; i++) {
11842
- partitions[i].endBatch();
11843
- }
11844
- },
11845
- update(docId, document, options, removedSpellings) {
11846
- const pid = locateDocument(docId);
11847
- if (pid === void 0) {
11848
- throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
11849
- }
11850
- const updateOpts = resolveInsertOptions(options);
11851
- partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
11852
- docPartitionMap.set(docId, pid);
11853
- writeLog.documentWritten(docId);
11854
- },
11855
- get(docId, projection) {
11856
- const pid = locateDocument(docId);
11857
- if (pid === void 0) return void 0;
11858
- const doc = partitions[pid].get(docId, projection);
11859
- if (doc) copyVectorsInto(doc, docId, vecIndexes, projection);
11860
- return doc;
11861
- },
11862
- getRef(docId) {
11863
- const pid = locateDocument(docId);
11864
- if (pid === void 0) return void 0;
11865
- return partitions[pid].getRef(docId);
11866
- },
11867
- fieldValues(docId, fieldPaths) {
11868
- const pid = locateDocument(docId);
11869
- return pid === void 0 ? void 0 : partitions[pid].fieldValues(docId, fieldPaths);
11870
- },
11871
- sortValues(docId, fields, fieldTypes, modes) {
11872
- const pid = locateDocument(docId);
11873
- if (pid === void 0) return fields.map(() => null);
11874
- return partitions[pid].sortValues(docId, fields, fieldTypes, modes);
11875
- },
11876
- has(docId) {
11877
- return locateDocument(docId) !== void 0;
11878
- },
11879
- partitionIdOf(docId) {
11880
- return locateDocument(docId);
11881
- },
11882
- countDocuments() {
11883
- let total = 0;
11884
- for (let i = 0; i < partitions.length; i++) {
11885
- total += partitions[i].count();
11473
+ const averageFieldLengths = /* @__PURE__ */ Object.create(null);
11474
+ if (totalDocuments > 0) {
11475
+ for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
11476
+ averageFieldLengths[field] = totalLength / totalDocuments;
11477
+ }
11478
+ }
11479
+ return {
11480
+ totalDocuments,
11481
+ docFrequencies,
11482
+ totalFieldLengths,
11483
+ averageFieldLengths
11484
+ };
11485
+ }
11486
+
11487
+ // src/search/cursor-codec.ts
11488
+ function encodeCursorText(text) {
11489
+ if (typeof Buffer !== "undefined") return Buffer.from(text, "utf-8").toString("base64");
11490
+ const bytes = new TextEncoder().encode(text);
11491
+ let binary = "";
11492
+ for (const byte of bytes) binary += String.fromCharCode(byte);
11493
+ return btoa(binary);
11494
+ }
11495
+ function decodeCursorText(encoded) {
11496
+ if (typeof Buffer !== "undefined") return Buffer.from(encoded, "base64").toString("utf-8");
11497
+ const binary = atob(encoded);
11498
+ const bytes = new Uint8Array(binary.length);
11499
+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
11500
+ return new TextDecoder().decode(bytes);
11501
+ }
11502
+
11503
+ // src/search/sorting.ts
11504
+ function isFieldList(sort) {
11505
+ return Array.isArray(sort);
11506
+ }
11507
+ function normalizeSort(sort) {
11508
+ if (sort === void 0) return [];
11509
+ if (isFieldList(sort)) {
11510
+ return sort.map(
11511
+ (entry) => entry.mode === void 0 ? { field: entry.field, direction: entry.direction } : { field: entry.field, direction: entry.direction, mode: entry.mode }
11512
+ );
11513
+ }
11514
+ return Object.entries(sort).map(([field, direction]) => ({ field, direction }));
11515
+ }
11516
+ function sortModesOf(fields) {
11517
+ return fields.map(sortModeOf);
11518
+ }
11519
+ function sortSignatureEntry(entry) {
11520
+ const mode = sortModeOf(entry);
11521
+ return mode === defaultSortMode(entry.direction) ? [entry.field, entry.direction] : [entry.field, entry.direction, mode];
11522
+ }
11523
+ function requireSortMode(entry, fieldType) {
11524
+ if (entry.mode === void 0) return;
11525
+ if (!isSortMode(entry.mode)) {
11526
+ throw new NarsilError(
11527
+ ErrorCodes.SEARCH_INVALID_MODE,
11528
+ `A sort mode is one of ${SORT_MODES.map((mode) => `"${mode}"`).join(", ")}, and the mode on "${entry.field}" is "${String(entry.mode)}"`,
11529
+ { field: entry.field, mode: String(entry.mode) }
11530
+ );
11531
+ }
11532
+ const averages = entry.mode === "avg" || entry.mode === "median";
11533
+ if (averages && fieldType !== void 0 && fieldType !== "number" && fieldType !== "number[]") {
11534
+ throw new NarsilError(
11535
+ ErrorCodes.SEARCH_INVALID_FIELD,
11536
+ `The engine applies the "${entry.mode}" mode to a number field alone, and "${entry.field}" is a ${fieldType} field`,
11537
+ { field: entry.field, fieldType, mode: entry.mode }
11538
+ );
11539
+ }
11540
+ }
11541
+ function requireSortableFields(sort, schema, strict = false) {
11542
+ const fields = normalizeSort(sort);
11543
+ if (fields.length === 0) return;
11544
+ const flatSchema = flattenSchema(schema);
11545
+ for (const entry of fields) {
11546
+ const fieldType = flatSchema[entry.field];
11547
+ if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(entry.field, "sort");
11548
+ if (entry.direction !== "asc" && entry.direction !== "desc") {
11549
+ throw new NarsilError(
11550
+ ErrorCodes.SEARCH_INVALID_MODE,
11551
+ `A sort direction is "asc" or "desc", and the direction on "${entry.field}" is "${String(entry.direction)}"`,
11552
+ { field: entry.field, direction: String(entry.direction) }
11553
+ );
11554
+ }
11555
+ requireSortMode(entry, fieldType);
11556
+ const parsed = parsedTypeOf(fieldType);
11557
+ if (parsed !== void 0 && isStringValuedBase(parsed.base) && !parsed.sortable) {
11558
+ throw new NarsilError(
11559
+ ErrorCodes.SEARCH_INVALID_FIELD,
11560
+ `The engine sorts by text field "${entry.field}" only where its type includes "sortable", as in "${withSortableOption(parsed)}", because it keeps up to the first ${SORT_VALUE_MAX_CODE_POINTS} code points of every document's value in memory to sort by a text field`,
11561
+ { field: entry.field, fieldType }
11562
+ );
11563
+ }
11564
+ if (isGeopointOrVectorType(fieldType)) {
11565
+ throw new NarsilError(
11566
+ ErrorCodes.SEARCH_INVALID_FIELD,
11567
+ `The engine sorts by a number, boolean, enum, or sortable text field, and "${entry.field}" is a ${fieldType} field`,
11568
+ { field: entry.field, fieldType }
11569
+ );
11570
+ }
11571
+ }
11572
+ }
11573
+ function applySorting(hits, sort, sortKeyOf) {
11574
+ const normalized = normalizeSort(sort);
11575
+ if (normalized.length === 0) return hits;
11576
+ const directions = normalized.map((entry) => entry.direction);
11577
+ const keyCache = /* @__PURE__ */ new Map();
11578
+ for (const hit of hits) {
11579
+ keyCache.set(hit.id, sortKeyOf(hit.id));
11580
+ }
11581
+ const sorted = hits.slice();
11582
+ sorted.sort(
11583
+ (a, b) => compareComparableKeys(keyCache.get(a.id) ?? [], keyCache.get(b.id) ?? [], directions) || compareCodePoints(a.id, b.id)
11584
+ );
11585
+ return sorted;
11586
+ }
11587
+
11588
+ // src/search/cursor.ts
11589
+ var CURSOR_VERSION = 4;
11590
+ function invalidCursor(cursor, reason) {
11591
+ const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
11592
+ return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
11593
+ }
11594
+ function sortSignatureOf(sort) {
11595
+ const fields = normalizeSort(sort);
11596
+ if (fields.length === 0) return null;
11597
+ if (fields.length > MAX_SORT_FIELDS) {
11598
+ throw new NarsilError(
11599
+ ErrorCodes.SEARCH_INVALID_FIELD,
11600
+ `A sort names at most ${MAX_SORT_FIELDS} fields, because the paging cursor carries one value per field`,
11601
+ { count: fields.length, limit: MAX_SORT_FIELDS }
11602
+ );
11603
+ }
11604
+ for (const entry of fields) {
11605
+ if (entry.field.length > MAX_SORT_FIELD_NAME_LENGTH) {
11606
+ throw new NarsilError(
11607
+ ErrorCodes.SEARCH_INVALID_FIELD,
11608
+ `A sort field name holds at most ${MAX_SORT_FIELD_NAME_LENGTH} characters, because the paging cursor carries the sort with every page`,
11609
+ { field: entry.field.slice(0, 64), length: entry.field.length, limit: MAX_SORT_FIELD_NAME_LENGTH }
11610
+ );
11611
+ }
11612
+ }
11613
+ return JSON.stringify(fields.map(sortSignatureEntry));
11614
+ }
11615
+ function encodePageCursor(cursor) {
11616
+ const payload = { v: CURSOR_VERSION, a: cursor.anchor };
11617
+ if (cursor.sortKey !== null && cursor.sortSignature !== null) {
11618
+ payload.k = cursor.sortKey.map(toComparableSortValue);
11619
+ payload.o = cursor.sortSignature;
11620
+ } else if (cursor.score !== null) {
11621
+ payload.s = cursor.score;
11622
+ }
11623
+ if (cursor.depth !== void 0 && cursor.depth > 0) payload.d = cursor.depth;
11624
+ payload.q = cursor.binding;
11625
+ return encodeCursorText(JSON.stringify(payload));
11626
+ }
11627
+ function decodeSortKey(value, cursor) {
11628
+ if (!Array.isArray(value)) throw invalidCursor(cursor, '"k" must be an array');
11629
+ if (value.length > MAX_SORT_FIELDS) throw invalidCursor(cursor, '"k" carries too many sort values');
11630
+ const key = [];
11631
+ for (const entry of value) {
11632
+ if (entry === null || typeof entry === "boolean") {
11633
+ key.push(entry);
11634
+ continue;
11635
+ }
11636
+ if (typeof entry === "number") {
11637
+ if (!Number.isFinite(entry)) throw invalidCursor(cursor, '"k" holds a number that is not finite');
11638
+ key.push(entry);
11639
+ continue;
11640
+ }
11641
+ if (typeof entry === "string") {
11642
+ if (exceedsCodePoints(entry, SORT_VALUE_MAX_CODE_POINTS)) {
11643
+ throw invalidCursor(cursor, '"k" holds an oversized sort value');
11886
11644
  }
11887
- return total;
11888
- },
11889
- serializePartition(partitionId) {
11890
- validatePartitionId(partitionId);
11891
- return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
11892
- },
11893
- mergeSegment(partitionId, payload, documents) {
11894
- validatePartitionId(partitionId);
11895
- const partition = partitions[partitionId];
11896
- partition.beginBatch();
11897
- partition.mergeSegmentPayload(payload, documents);
11898
- partition.endBatch();
11899
- for (const docId of payload.docIds) {
11900
- docPartitionMap.set(docId, partitionId);
11901
- writeLog.documentWritten(docId);
11645
+ key.push(entry);
11646
+ continue;
11647
+ }
11648
+ throw invalidCursor(cursor, '"k" accepts a string, a finite number, a boolean, or null');
11649
+ }
11650
+ return key;
11651
+ }
11652
+ function decodePageCursor(cursor) {
11653
+ if (cursor.length > MAX_CURSOR_LENGTH) throw invalidCursor(cursor, "the cursor is too long");
11654
+ let json;
11655
+ try {
11656
+ json = decodeCursorText(cursor);
11657
+ } catch {
11658
+ throw invalidCursor(cursor, "invalid base64 encoding");
11659
+ }
11660
+ let parsed;
11661
+ try {
11662
+ parsed = JSON.parse(json);
11663
+ } catch {
11664
+ throw invalidCursor(cursor, "invalid JSON");
11665
+ }
11666
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
11667
+ throw invalidCursor(cursor, "expected an object");
11668
+ }
11669
+ const { v, a, s, k, o, d, q } = parsed;
11670
+ if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
11671
+ if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
11672
+ throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
11673
+ }
11674
+ if (typeof a !== "string" || a.length === 0) throw invalidCursor(cursor, '"a" must be a non-empty string');
11675
+ if (exceedsCodePoints(a, MAX_CURSOR_ANCHOR_CODE_POINTS)) {
11676
+ throw invalidCursor(cursor, '"a" is too long to be a document id');
11677
+ }
11678
+ if (s !== void 0 && (typeof s !== "number" || !Number.isFinite(s))) {
11679
+ throw invalidCursor(cursor, '"s" must be a finite number');
11680
+ }
11681
+ if (s !== void 0 && k !== void 0) throw invalidCursor(cursor, 'a cursor carries "s" or "k", never both');
11682
+ if (d !== void 0 && (typeof d !== "number" || !Number.isSafeInteger(d) || d < 1)) {
11683
+ throw invalidCursor(cursor, '"d" must be a positive integer');
11684
+ }
11685
+ const sortKey = k === void 0 ? null : decodeSortKey(k, cursor);
11686
+ if (o !== void 0 && typeof o !== "string") throw invalidCursor(cursor, '"o" must be a string');
11687
+ if (sortKey === null !== (o === void 0)) {
11688
+ throw invalidCursor(cursor, '"k" and "o" have to arrive together');
11689
+ }
11690
+ return {
11691
+ anchor: a,
11692
+ score: typeof s === "number" ? s : null,
11693
+ sortKey,
11694
+ sortSignature: typeof o === "string" ? o : null,
11695
+ depth: typeof d === "number" ? d : 0,
11696
+ binding: q
11697
+ };
11698
+ }
11699
+ function requireMatchingCursor(cursor, encoded, signature, scoreAnchored, binding) {
11700
+ if (cursor.binding !== binding) {
11701
+ throw invalidCursor(encoded, "the cursor belongs to a different query");
11702
+ }
11703
+ if (signature !== null) {
11704
+ if (cursor.sortSignature !== signature) {
11705
+ throw invalidCursor(encoded, "the cursor belongs to a different sort order");
11706
+ }
11707
+ return;
11708
+ }
11709
+ if (cursor.sortSignature !== null) {
11710
+ throw invalidCursor(encoded, "the cursor belongs to a sorted request");
11711
+ }
11712
+ if (scoreAnchored && cursor.score === null) {
11713
+ throw invalidCursor(encoded, "the cursor carries no score to seek from");
11714
+ }
11715
+ if (!scoreAnchored && cursor.score !== null) {
11716
+ throw invalidCursor(encoded, "the cursor belongs to a search rather than a listing");
11717
+ }
11718
+ }
11719
+
11720
+ // src/search/pagination.ts
11721
+ function clampRowCount(value, fallback) {
11722
+ if (value === void 0 || !Number.isFinite(value)) return fallback;
11723
+ return Math.max(0, Math.floor(value));
11724
+ }
11725
+ function requireWithinResultWindow(limit, offset) {
11726
+ const depth = offset + limit;
11727
+ if (depth <= RESULT_WINDOW) return;
11728
+ throw new NarsilError(
11729
+ ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
11730
+ `A request reaches the first ${RESULT_WINDOW} results, and offset + limit is ${depth}. Page past that with the cursor each result carries`,
11731
+ { limit, offset, window: RESULT_WINDOW }
11732
+ );
11733
+ }
11734
+ function lastOrganicResult(sliced, pinnedIds) {
11735
+ for (let i = sliced.length - 1; i >= 0; i--) {
11736
+ if (pinnedIds === void 0 || !pinnedIds.has(sliced[i].id)) return sliced[i];
11737
+ }
11738
+ return void 0;
11739
+ }
11740
+ function ordersAfterAnchor(result, anchor, sort) {
11741
+ if (sort !== void 0 && anchor.sortKey !== null) {
11742
+ const comparison = compareSortValues(sort.sortKeyOf(result.id), anchor.sortKey, sort.directions) || compareCodePoints(result.id, anchor.anchor);
11743
+ return comparison > 0;
11744
+ }
11745
+ if (anchor.score === null) return true;
11746
+ if (result.score === void 0) return true;
11747
+ if (result.score < anchor.score) return true;
11748
+ return result.score === anchor.score && compareCodePoints(result.id, anchor.anchor) > 0;
11749
+ }
11750
+ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedIds) {
11751
+ const decoded = cursor ? decodePageCursor(cursor) : null;
11752
+ if (decoded !== null && cursor !== void 0) {
11753
+ requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding);
11754
+ }
11755
+ if (limit <= 0) {
11756
+ return { paginated: [] };
11757
+ }
11758
+ let startIndex = 0;
11759
+ if (decoded !== null) {
11760
+ startIndex = results.length;
11761
+ for (let i = 0; i < results.length; i++) {
11762
+ if (ordersAfterAnchor(results[i], decoded, sort)) {
11763
+ startIndex = i;
11764
+ break;
11902
11765
  }
11903
- },
11904
- attachFrozenSegment(partitionId, segment) {
11905
- validatePartitionId(partitionId);
11906
- requireNewSegmentDocuments(segment, docPartitionMap);
11907
- asCompositePartition(partitionId).attachFrozenSegment(segment);
11908
- registerSegmentDocuments(partitionId, segment);
11909
- writeLog.segmentWritten(segment);
11910
- },
11911
- swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
11912
- validatePartitionId(partitionId);
11913
- const applied = asCompositePartition(partitionId).swapFrozenSegments(
11914
- dropSegmentIds,
11915
- replacement,
11916
- removedSpellings
11766
+ }
11767
+ }
11768
+ const afterOffset = startIndex + offset;
11769
+ const sliced = results.slice(afterOffset, afterOffset + limit);
11770
+ let nextCursor;
11771
+ const hasMore = afterOffset + limit < results.length;
11772
+ const lastResult = lastOrganicResult(sliced, pinnedIds);
11773
+ const depth = afterOffset + sliced.length;
11774
+ if (hasMore && lastResult !== void 0) {
11775
+ nextCursor = sort !== void 0 ? encodePageCursor({
11776
+ anchor: lastResult.id,
11777
+ score: null,
11778
+ sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
11779
+ sortSignature: sort.signature,
11780
+ depth,
11781
+ binding
11782
+ }) : encodePageCursor({
11783
+ anchor: lastResult.id,
11784
+ score: lastResult.score ?? null,
11785
+ sortKey: null,
11786
+ sortSignature: null,
11787
+ depth,
11788
+ binding
11789
+ });
11790
+ }
11791
+ return { paginated: sliced, nextCursor };
11792
+ }
11793
+
11794
+ // src/search/fulltext.ts
11795
+ function prepareFulltextQuery(partition, params, language, schema, options) {
11796
+ if (!params.term || params.term.trim().length === 0) return null;
11797
+ if (params.fields && params.fields.length === 0) return null;
11798
+ const flatSchema = flattenSchema(schema);
11799
+ if (params.fields) {
11800
+ validateSearchFields(params.fields, flatSchema);
11801
+ }
11802
+ const queryTokenResult = tokenize(params.term, language, {
11803
+ stem: true,
11804
+ removeStopWords: true,
11805
+ customTokenizer: options?.customTokenizer,
11806
+ stopWordOverride: options?.stopWords
11807
+ });
11808
+ if (queryTokenResult.tokens.length === 0) return null;
11809
+ const queryTokens = deduplicateTokens(queryTokenResult.tokens);
11810
+ let filterBitset;
11811
+ if (params.filters) {
11812
+ filterBitset = partition.applyFiltersBitset(
11813
+ params.filters,
11814
+ schema,
11815
+ options?.patternWork ?? createPatternWorkMeter()
11816
+ );
11817
+ if (bitsetIsEmpty(filterBitset)) return null;
11818
+ }
11819
+ const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
11820
+ const synonyms = params.exact === true ? void 0 : options?.synonyms?.expand(queryTokens.map((qt) => qt.token));
11821
+ const groups = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, options, synonyms);
11822
+ return { queryTokens, ...groups, filterBitset };
11823
+ }
11824
+ function fulltextMatches(partition, params, language, schema, options) {
11825
+ const prepared = prepareFulltextQuery(partition, params, language, schema, options);
11826
+ if (prepared === null) return null;
11827
+ return partition.searchFulltextMatches({
11828
+ queryTokens: prepared.queryTokens,
11829
+ termGroups: prepared.termGroups,
11830
+ partialFields: prepared.partialFields,
11831
+ fields: params.fields,
11832
+ tolerance: params.tolerance ?? 0,
11833
+ prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
11834
+ exact: params.exact ?? false,
11835
+ filterBitset: prepared.filterBitset
11836
+ });
11837
+ }
11838
+ function fulltextSearch(partition, params, language, schema, options) {
11839
+ const prepared = prepareFulltextQuery(partition, params, language, schema, options);
11840
+ if (prepared === null) {
11841
+ return { scored: [], totalMatched: 0 };
11842
+ }
11843
+ const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
11844
+ const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
11845
+ const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
11846
+ const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
11847
+ const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
11848
+ const rawResult = partition.searchFulltext({
11849
+ queryTokens,
11850
+ termGroups,
11851
+ partialFields,
11852
+ fields: params.fields,
11853
+ boost: params.boost,
11854
+ tolerance: params.tolerance ?? 0,
11855
+ prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
11856
+ exact: params.exact ?? false,
11857
+ bm25Params: options?.bm25Params,
11858
+ globalStats: options?.globalStats,
11859
+ maxResults,
11860
+ termMatch: params.termMatch,
11861
+ filterBitset,
11862
+ collectComponents,
11863
+ collectMatchedSet
11864
+ });
11865
+ let scored = rawResult.scored;
11866
+ const termMatch = params.termMatch ?? "any";
11867
+ if (termMatch !== "any") {
11868
+ scored = filterByTermCoverage(
11869
+ scored,
11870
+ queryTokens,
11871
+ termMatch,
11872
+ params.tolerance ?? 0,
11873
+ params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
11874
+ params.exact ?? false,
11875
+ termGroups,
11876
+ partialFields ?? []
11877
+ );
11878
+ }
11879
+ if (params.minScore !== void 0 && params.minScore > 0) {
11880
+ const threshold = params.minScore;
11881
+ scored = scored.filter((doc) => doc.score >= threshold);
11882
+ }
11883
+ const totalMatched = needsAllResults ? scored.length : rawResult.totalMatched;
11884
+ if (params.facets === void 0) {
11885
+ return { scored, totalMatched };
11886
+ }
11887
+ if (needsAllResults) {
11888
+ return { scored, totalMatched, matchedIds: scored.map((doc) => doc.docId) };
11889
+ }
11890
+ return { scored, totalMatched, matchedOrdinalBitset: rawResult.matchedOrdinalBitset };
11891
+ }
11892
+ function validateSearchFields(fields, flatSchema) {
11893
+ for (const field of fields) {
11894
+ const fieldType = flatSchema[field];
11895
+ if (!fieldType) {
11896
+ throw new NarsilError(ErrorCodes.SEARCH_INVALID_FIELD, `Field "${field}" does not exist in the schema`, { field });
11897
+ }
11898
+ if (!isWordIndexedFieldType(fieldType)) {
11899
+ throw new NarsilError(
11900
+ ErrorCodes.SEARCH_INVALID_FIELD,
11901
+ `Field "${field}" has type "${fieldType}" which cannot be used for full-text search`,
11902
+ { field, fieldType }
11917
11903
  );
11918
- registerSegmentDocuments(partitionId, replacement);
11919
- return applied;
11920
- },
11921
- freezeLiveTail(partitionId, freeze) {
11922
- validatePartitionId(partitionId);
11923
- return asCompositePartition(partitionId).freezeLiveTail(freeze);
11924
- },
11925
- replaceLiveTail(partitionId, segment) {
11926
- validatePartitionId(partitionId);
11927
- asCompositePartition(partitionId).replaceLiveTail(segment);
11928
- registerSegmentDocuments(partitionId, segment);
11929
- },
11930
- deserializePartition(partitionId, data) {
11931
- validatePartitionId(partitionId);
11932
- writeLog.partitionsReplaced();
11933
- partitions[partitionId].deserialize(data, config.schema);
11934
- preparePartialWords(partitionId);
11935
- rebuildDocPartitionMapForPartition(partitionId);
11936
- },
11937
- getAggregateStats() {
11938
- return aggregateStatsOf(partitions);
11939
- },
11940
- estimateMemoryBytes() {
11941
- return estimateBytesOf(partitions, vecIndexes);
11942
- },
11943
- getPartitionStats() {
11944
- return partitionStatsOf(partitions);
11945
- },
11946
- getVectorIndexes() {
11947
- return vecIndexes;
11948
- },
11949
- resetVectorIndexes(newIndexes) {
11950
- vecIndexes.clear();
11951
- for (const [key, value] of newIndexes) {
11952
- vecIndexes.set(key, value);
11904
+ }
11905
+ }
11906
+ }
11907
+ function deduplicateTokens(tokens) {
11908
+ const seen = /* @__PURE__ */ new Set();
11909
+ const result = [];
11910
+ for (const t of tokens) {
11911
+ if (!seen.has(t.token)) {
11912
+ seen.add(t.token);
11913
+ result.push(t);
11914
+ }
11915
+ }
11916
+ return result;
11917
+ }
11918
+ function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
11919
+ const requiredCount = policy === "all" ? queryTokens.length : policy;
11920
+ if (requiredCount <= 0) return scored;
11921
+ if (requiredCount > queryTokens.length) return [];
11922
+ const groups = /* @__PURE__ */ new Map();
11923
+ for (const group of termGroups ?? []) {
11924
+ const partialOnly = /* @__PURE__ */ new Set();
11925
+ for (const member of group.members) {
11926
+ if (member.factor === 0) partialOnly.add(member.token);
11927
+ }
11928
+ groups.set(group.token, {
11929
+ members: /* @__PURE__ */ new Set([...group.members.map((member) => member.token), ...group.synonyms?.tokens ?? []]),
11930
+ partialOnly,
11931
+ typoMatches: group.typoMatches
11932
+ });
11933
+ }
11934
+ return scored.filter((doc) => {
11935
+ const matched = countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields);
11936
+ return matched >= requiredCount;
11937
+ });
11938
+ }
11939
+ function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
11940
+ const indexTokens = Object.keys(doc.idf);
11941
+ if (indexTokens.length === 0) return 0;
11942
+ let count = 0;
11943
+ for (const qt of queryTokens) {
11944
+ const group = groups.get(qt.token);
11945
+ const satisfied = group === void 0 ? queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact) : groupSatisfied(group, doc, indexTokens, partialFields) || group.typoMatches && queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact);
11946
+ if (satisfied) count++;
11947
+ }
11948
+ return count;
11949
+ }
11950
+ function matchedInPartialField(doc, token, partialFields) {
11951
+ for (const field of partialFields) {
11952
+ if (doc.termFrequencies[`${field}:${token}`] !== void 0) return true;
11953
+ }
11954
+ return false;
11955
+ }
11956
+ function groupSatisfied(group, doc, indexTokens, partialFields) {
11957
+ for (const indexToken of indexTokens) {
11958
+ if (!group.members.has(indexToken)) continue;
11959
+ if (!group.partialOnly.has(indexToken) || matchedInPartialField(doc, indexToken, partialFields)) return true;
11960
+ }
11961
+ return false;
11962
+ }
11963
+ function queryTermSatisfied(queryToken, indexTokens, tolerance, prefixLength, exact) {
11964
+ if (exact || tolerance === 0) {
11965
+ return indexTokens.includes(queryToken);
11966
+ }
11967
+ for (const indexToken of indexTokens) {
11968
+ if (fuzzyTermMatches(queryToken, indexToken, tolerance, prefixLength)) {
11969
+ return true;
11970
+ }
11971
+ }
11972
+ return false;
11973
+ }
11974
+
11975
+ // src/partitioning/fan-out.ts
11976
+ async function fanOutQuery(manager, params, language, schema, config, searchOptions) {
11977
+ const partitions = partitionsIn(manager, config.partitionIds);
11978
+ if (partitions.length === 0) {
11979
+ return { scored: [], totalMatched: 0 };
11980
+ }
11981
+ let globalStats;
11982
+ const effectiveMode = resolveEffectiveMode(config);
11983
+ if (effectiveMode === "dfs") {
11984
+ if (params.term !== void 0 && params.term.trim().length > 0) {
11985
+ const analysis = {
11986
+ stopWords: searchOptions?.stopWords,
11987
+ customTokenizer: searchOptions?.customTokenizer,
11988
+ synonyms: searchOptions?.synonyms
11989
+ };
11990
+ globalStats = collectQueryTermStats(manager, params.term, language, analysis, config.partitionIds, {
11991
+ expansion: { options: params, schema }
11992
+ });
11993
+ }
11994
+ } else if (effectiveMode === "broadcast") {
11995
+ globalStats = config.globalStats;
11996
+ }
11997
+ const options = buildSearchOptions(searchOptions, globalStats);
11998
+ if (partitions.length === 1 && !config.dispatcher) {
11999
+ const result = dispatchSinglePartition(partitions[0], params, language, schema, options);
12000
+ let facets2;
12001
+ if (params.facets) {
12002
+ facets2 = partitions[0].computeFacets(facetMatchSetOf(result), params.facets, schema);
12003
+ }
12004
+ return { scored: result.scored, totalMatched: result.totalMatched, facets: facets2 };
12005
+ }
12006
+ let outcomes;
12007
+ let facets;
12008
+ if (config.dispatcher) {
12009
+ outcomes = await dispatchWithDispatcher(partitions, params, language, schema, options, config.dispatcher);
12010
+ if (params.facets) {
12011
+ facets = collectAndMergeFacets(outcomes, params.facets, schema);
12012
+ }
12013
+ } else {
12014
+ outcomes = [];
12015
+ const partitionFacets = [];
12016
+ const everyValue = params.facets ? everyValueFacetConfig(params.facets) : void 0;
12017
+ for (const partition of partitions) {
12018
+ const result = dispatchSinglePartition(partition, params, language, schema, options);
12019
+ if (everyValue) {
12020
+ partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), everyValue, schema));
11953
12021
  }
12022
+ outcomes.push({ result, partition });
11954
12023
  }
12024
+ if (params.facets) {
12025
+ facets = mergeFacets(partitionFacets, params.facets);
12026
+ }
12027
+ }
12028
+ const allScoredArrays = outcomes.map((o) => o.result.scored);
12029
+ const merged = kWayMerge(allScoredArrays);
12030
+ let totalMatched = 0;
12031
+ for (const outcome of outcomes) {
12032
+ totalMatched += outcome.result.totalMatched;
12033
+ }
12034
+ return { scored: merged, totalMatched, facets };
12035
+ }
12036
+ function facetMatchSetOf(result) {
12037
+ if (result.matchedOrdinalBitset !== void 0) {
12038
+ return { ordinalBitset: result.matchedOrdinalBitset };
12039
+ }
12040
+ return new Set(result.matchedIds ?? []);
12041
+ }
12042
+ function resolveEffectiveMode(config) {
12043
+ if (config.scoringMode === "broadcast" && !config.globalStats) {
12044
+ return "dfs";
12045
+ }
12046
+ return config.scoringMode;
12047
+ }
12048
+ function buildSearchOptions(base, globalStats) {
12049
+ return {
12050
+ ...base,
12051
+ globalStats: globalStats ?? base?.globalStats,
12052
+ patternWork: base?.patternWork ?? createPatternWorkMeter()
11955
12053
  };
11956
- return manager;
12054
+ }
12055
+ function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
12056
+ const promises = partitions.map(
12057
+ (partition) => dispatcher.dispatch(partition, params, language, schema, options).then((result) => ({ result, partition }))
12058
+ );
12059
+ return Promise.all(promises);
12060
+ }
12061
+ function dispatchSinglePartition(partition, params, language, schema, options) {
12062
+ return fulltextSearch(partition, params, language, schema, options);
12063
+ }
12064
+ function collectAndMergeFacets(outcomes, facetConfig, schema) {
12065
+ const everyValue = everyValueFacetConfig(facetConfig);
12066
+ const partitionFacets = [];
12067
+ for (const outcome of outcomes) {
12068
+ partitionFacets.push(outcome.partition.computeFacets(facetMatchSetOf(outcome.result), everyValue, schema));
12069
+ }
12070
+ return mergeFacets(partitionFacets, facetConfig);
11957
12071
  }
11958
12072
 
11959
12073
  // src/partitioning/match-count.ts
@@ -11973,6 +12087,43 @@ function fanOutMatchCount(manager, params, language, schema, options) {
11973
12087
  return count;
11974
12088
  }
11975
12089
 
12090
+ // src/workers/query-actions.ts
12091
+ async function runQueryAction(entry, action) {
12092
+ const patternWork = joinPatternWorkMeter(action.patternWork);
12093
+ const searchOptions = { ...entry.searchOptions, patternWork };
12094
+ if (action.type === "query") {
12095
+ const result2 = await fanOutQuery(
12096
+ entry.manager,
12097
+ action.params,
12098
+ entry.language,
12099
+ entry.config.schema,
12100
+ {
12101
+ scoringMode: action.params.scoring ?? entry.config.defaultScoring ?? "local",
12102
+ partitionIds: action.partitionIds,
12103
+ ...action.globalStats !== void 0 ? { globalStats: sanitizeGlobalStats(action.globalStats) } : {}
12104
+ },
12105
+ searchOptions
12106
+ );
12107
+ result2.patternWorkSpent = patternWork.spent();
12108
+ return result2;
12109
+ }
12110
+ if (countsWithoutScores(action.params)) {
12111
+ const count = fanOutMatchCount(entry.manager, action.params, entry.language, entry.config.schema, {
12112
+ searchOptions
12113
+ });
12114
+ return { count };
12115
+ }
12116
+ const result = await fanOutQuery(
12117
+ entry.manager,
12118
+ action.params,
12119
+ entry.language,
12120
+ entry.config.schema,
12121
+ { scoringMode: entry.config.defaultScoring ?? "local" },
12122
+ searchOptions
12123
+ );
12124
+ return { count: result.totalMatched };
12125
+ }
12126
+
11976
12127
  // src/core/partition/frozen/doc-table.ts
11977
12128
  function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
11978
12129
  const capacity = idTable.count;
@@ -14835,8 +14986,11 @@ function createDirectExecutor(options) {
14835
14986
  }
14836
14987
  return vectorIndexes;
14837
14988
  }
14838
- function createIndex(indexName, config, language, analysisStale = false) {
14839
- addIndex(indexName, config, language, analysisStale, null);
14989
+ function createIndex(indexName, config, language, analysisStale = false, synonyms = null) {
14990
+ addIndex(indexName, config, language, analysisStale, null).synonyms = synonyms;
14991
+ }
14992
+ function lookupOf(config, language, encoded) {
14993
+ return encoded === void 0 ? null : synonymModule().storedLookup(encoded, config, language);
14840
14994
  }
14841
14995
  function addIndex(indexName, config, language, analysisStale, awaitedCopyLoad) {
14842
14996
  if (indexes.has(indexName)) {
@@ -14855,13 +15009,17 @@ function createDirectExecutor(options) {
14855
15009
  searchOptions: {
14856
15010
  bm25Params: config.bm25,
14857
15011
  stopWords: manager.analysis.stopWords,
14858
- customTokenizer: manager.analysis.customTokenizer
15012
+ customTokenizer: manager.analysis.customTokenizer,
15013
+ get synonyms() {
15014
+ return manager.synonyms;
15015
+ }
14859
15016
  },
14860
15017
  vectorIndexes,
14861
15018
  vectorCopies: createHeldVectorCopies(),
14862
15019
  analysisStale,
14863
15020
  awaitedCopyLoad
14864
15021
  });
15022
+ return manager;
14865
15023
  }
14866
15024
  function dropIndex(indexName) {
14867
15025
  const entry = requireIndex(indexName);
@@ -14914,7 +15072,19 @@ function createDirectExecutor(options) {
14914
15072
  const loading2 = loadVectorCodeFor(action.config.schema);
14915
15073
  if (loading2 !== null) await loading2;
14916
15074
  const language = getLanguage(action.config.language ?? "english");
14917
- addIndex(action.indexName, action.config, language, action.analysisStale ?? false, action.copyLoad ?? null);
15075
+ const manager = addIndex(
15076
+ action.indexName,
15077
+ action.config,
15078
+ language,
15079
+ action.analysisStale ?? false,
15080
+ action.copyLoad ?? null
15081
+ );
15082
+ try {
15083
+ manager.synonyms = lookupOf(action.config, language, action.synonyms);
15084
+ } catch (err) {
15085
+ dropIndex(action.indexName);
15086
+ throw err;
15087
+ }
14918
15088
  return void 0;
14919
15089
  }
14920
15090
  case "copyLoaded": {
@@ -14963,42 +15133,13 @@ function createDirectExecutor(options) {
14963
15133
  entry.manager.update(action.docId, action.document, void 0, action.removedSpellings);
14964
15134
  return void 0;
14965
15135
  }
14966
- case "query": {
14967
- const entry = requireIndex(action.indexName);
14968
- const patternWork = joinPatternWorkMeter(action.patternWork);
14969
- const result = await fanOutQuery(
14970
- entry.manager,
14971
- action.params,
14972
- entry.language,
14973
- entry.config.schema,
14974
- {
14975
- scoringMode: action.params.scoring ?? entry.config.defaultScoring ?? "local",
14976
- partitionIds: action.partitionIds,
14977
- ...action.globalStats !== void 0 ? { globalStats: sanitizeGlobalStats(action.globalStats) } : {}
14978
- },
14979
- { ...entry.searchOptions, patternWork }
14980
- );
14981
- result.patternWorkSpent = patternWork.spent();
14982
- return result;
14983
- }
14984
- case "preflight": {
14985
- const entry = requireIndex(action.indexName);
14986
- const searchOptions = { ...entry.searchOptions, patternWork: joinPatternWorkMeter(action.patternWork) };
14987
- if (countsWithoutScores(action.params)) {
14988
- const count = fanOutMatchCount(entry.manager, action.params, entry.language, entry.config.schema, {
14989
- searchOptions
14990
- });
14991
- return { count };
14992
- }
14993
- const result = await fanOutQuery(
14994
- entry.manager,
14995
- action.params,
14996
- entry.language,
14997
- entry.config.schema,
14998
- { scoringMode: entry.config.defaultScoring ?? "local" },
14999
- searchOptions
15000
- );
15001
- return { count: result.totalMatched };
15136
+ case "query":
15137
+ case "preflight":
15138
+ return await runQueryAction(requireIndex(action.indexName), action);
15139
+ case "replaceSynonyms": {
15140
+ const entry = indexes.get(action.indexName);
15141
+ if (entry !== void 0) entry.manager.synonyms = lookupOf(entry.config, entry.language, action.synonyms);
15142
+ return void 0;
15002
15143
  }
15003
15144
  case "get": {
15004
15145
  const entry = requireIndex(action.indexName);
@@ -15061,7 +15202,7 @@ function createDirectExecutor(options) {
15061
15202
  }
15062
15203
  }
15063
15204
  async function shutdown() {
15064
- await execute({ type: "shutdown"});
15205
+ await execute({ type: "shutdown", requestId: "internal-shutdown" });
15065
15206
  }
15066
15207
  return {
15067
15208
  execute,
@@ -15483,6 +15624,25 @@ function reconstructSchemaFromMetadata(metadata) {
15483
15624
  return config;
15484
15625
  }
15485
15626
 
15627
+ // src/engine/synonym-sets.ts
15628
+ function withoutSynonyms(config) {
15629
+ if (!Object.hasOwn(config, "synonyms")) return config;
15630
+ const { synonyms: _synonyms, ...rest } = config;
15631
+ return rest;
15632
+ }
15633
+ function prepareStoredSynonyms(value, config, language, raise, strictWords = false) {
15634
+ return isEmptySynonymSet(value) ? null : synonymModule().prepare(value, config, language, strictWords, raise);
15635
+ }
15636
+ function prepareConfiguredSynonyms(config, language, strictWords = true) {
15637
+ return prepareStoredSynonyms(config.synonyms, config, language, void 0, strictWords);
15638
+ }
15639
+ function lookupOfStoredSynonyms(encoded, config, language) {
15640
+ return encoded === null || encoded === void 0 ? null : synonymModule().storedLookup(encoded, config, language);
15641
+ }
15642
+ function rulesOfStoredSynonyms(encoded) {
15643
+ return encoded === null ? [] : synonymModule().decodeRules(encoded);
15644
+ }
15645
+
15486
15646
  // src/engine/validation.ts
15487
15647
  function now() {
15488
15648
  if (typeof performance !== "undefined" && typeof performance.now === "function") {
@@ -15680,7 +15840,13 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
15680
15840
  const existing = deps.indexRegistry.get(metadata.indexName);
15681
15841
  if (existing !== void 0) {
15682
15842
  if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
15683
- deps.executor.createIndex(metadata.indexName, existing.config, existing.language);
15843
+ deps.executor.createIndex(
15844
+ metadata.indexName,
15845
+ existing.config,
15846
+ existing.language,
15847
+ false,
15848
+ lookupOfStoredSynonyms(existing.synonyms, existing.config, existing.language)
15849
+ );
15684
15850
  }
15685
15851
  return;
15686
15852
  }
@@ -15694,9 +15860,18 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
15694
15860
  throw recoveryFailure(metadata, err);
15695
15861
  }
15696
15862
  const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
15863
+ const prepared = prepareStoredSynonyms(
15864
+ metadata.synonyms,
15865
+ indexConfig,
15866
+ language,
15867
+ "The synonym set in the index metadata is invalid: "
15868
+ );
15869
+ const synonyms = prepared?.encoded ?? null;
15697
15870
  try {
15698
15871
  validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
15699
- if (loadData) deps.executor.createIndex(metadata.indexName, indexConfig, language);
15872
+ if (loadData) {
15873
+ deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
15874
+ }
15700
15875
  } catch (err) {
15701
15876
  throw recoveryFailure(metadata, err);
15702
15877
  }
@@ -15710,7 +15885,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
15710
15885
  heldPartitions: metadata.heldPartitions ?? null,
15711
15886
  documentCount: metadata.documentCount ?? 0,
15712
15887
  partitionCount: metadata.partitionCount,
15713
- analysisRevision: metadata.analysisRevision ?? null
15888
+ analysisRevision: metadata.analysisRevision ?? null,
15889
+ synonyms
15714
15890
  });
15715
15891
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
15716
15892
  else deps.indexState().registerClosed(metadata.indexName);
@@ -16536,17 +16712,17 @@ async function sendPartition(manager, executors, captured) {
16536
16712
  )
16537
16713
  );
16538
16714
  }
16539
- function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
16715
+ function transferIndexToPool(indexName, pool, source, manager, analysisStale = false) {
16540
16716
  return loadIndexOntoWorkers(
16541
16717
  indexName,
16542
16718
  pool.getAllExecutors(),
16543
- config,
16719
+ source,
16544
16720
  manager,
16545
16721
  () => pool.addIndexToAll(indexName),
16546
16722
  analysisStale
16547
16723
  );
16548
16724
  }
16549
- async function loadIndexOntoWorkers(indexName, executors, config, manager, onDropped, analysisStale = false) {
16725
+ async function loadIndexOntoWorkers(indexName, executors, source, manager, onDropped, analysisStale = false) {
16550
16726
  copyLoadSequence += 1;
16551
16727
  const copyLoad = `${indexName}#${copyLoadSequence}`;
16552
16728
  const captured = capturePartitions(manager);
@@ -16561,10 +16737,11 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
16561
16737
  (workerExecutor) => workerExecutor.execute({
16562
16738
  type: "createIndex",
16563
16739
  indexName,
16564
- config,
16740
+ config: source.config,
16565
16741
  requestId: `resync-create-${indexName}`,
16566
16742
  analysisStale,
16567
- copyLoad
16743
+ copyLoad,
16744
+ ...source.synonyms !== null ? { synonyms: source.synonyms } : {}
16568
16745
  })
16569
16746
  )
16570
16747
  );
@@ -17951,7 +18128,7 @@ async function loadOntoReplacement(state, replacement, indexName) {
17951
18128
  copyLoad = await loadIndexOntoWorkers(
17952
18129
  indexName,
17953
18130
  [replacement.executor],
17954
- entry.config,
18131
+ entry,
17955
18132
  manager,
17956
18133
  void 0,
17957
18134
  state.callbacks?.isAnalysisStale?.(indexName)
@@ -18141,7 +18318,7 @@ async function loadCopies(state, indexName, reason) {
18141
18318
  const copyLoad = await transferIndexToPool(
18142
18319
  indexName,
18143
18320
  pool,
18144
- entry.config,
18321
+ entry,
18145
18322
  manager,
18146
18323
  state.callbacks?.isAnalysisStale?.(indexName)
18147
18324
  );
@@ -18564,7 +18741,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
18564
18741
  const copyLoad = await transferIndexToPool(
18565
18742
  indexName,
18566
18743
  pool,
18567
- entry.config,
18744
+ entry,
18568
18745
  manager,
18569
18746
  callbacks?.isAnalysisStale?.(indexName)
18570
18747
  );
@@ -18708,7 +18885,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
18708
18885
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
18709
18886
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
18710
18887
  }
18711
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-VSUBLUDK.mjs');
18888
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-F4OGQUSU.mjs');
18712
18889
  const filesystemDurability = durabilityTier?.kind === "wal";
18713
18890
  const executor = createDirectExecutor({
18714
18891
  vectorWorkerCopies: vectorCopyPolicy,
@@ -19315,7 +19492,14 @@ function refuseCode(entry) {
19315
19492
  }
19316
19493
  function indexConfigurationOf(entry) {
19317
19494
  refuseCode(entry);
19318
- return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
19495
+ const synonyms = entry.synonyms === null ? void 0 : rulesOfStoredSynonyms(entry.synonyms);
19496
+ return configurationPayloadOf(
19497
+ entry.config,
19498
+ entry.language,
19499
+ entry.embeddingAdapterName,
19500
+ entry.analysisRevision,
19501
+ synonyms
19502
+ );
19319
19503
  }
19320
19504
  function requireConfigurationWithinSizeLimit(config, language) {
19321
19505
  const adapter = config.embedding?.adapter;
@@ -19334,7 +19518,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
19334
19518
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
19335
19519
  );
19336
19520
  }
19337
- function configurationPayloadOf(config, language, adapterName, analysisRevision) {
19521
+ function configurationPayloadOf(config, language, adapterName, analysisRevision, synonyms) {
19338
19522
  const bm25 = config.bm25;
19339
19523
  return {
19340
19524
  schema: config.schema,
@@ -19357,6 +19541,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision)
19357
19541
  ...config.required !== void 0 ? { required: config.required } : {},
19358
19542
  ...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
19359
19543
  ...config.patternValueLimit !== void 0 ? { patternValueLimit: config.patternValueLimit } : {},
19544
+ ...synonyms !== void 0 ? { synonyms } : {},
19360
19545
  ...config.embedding !== void 0 ? {
19361
19546
  embedding: {
19362
19547
  fields: config.embedding.fields,
@@ -19429,7 +19614,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
19429
19614
  }
19430
19615
  const schema = validateSchema(requestedConfig.schema);
19431
19616
  if (appliesCreationLimits) requireCreatableSchema(schema);
19432
- const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
19617
+ const configured = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
19618
+ const indexConfig = withoutSynonyms(configured);
19433
19619
  requirePatternSearchFor(schema);
19434
19620
  requirePartialWordSearchFor(schema);
19435
19621
  validatePatternValueLimit(indexConfig.patternValueLimit);
@@ -19463,10 +19649,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
19463
19649
  const fields = indexConfig.embedding.fields;
19464
19650
  if (typeof configuredAdapter === "string") {
19465
19651
  embeddingAdapterName = configuredAdapter;
19466
- const registered3 = core.embeddingAdapters.get(configuredAdapter);
19467
- if (registered3 !== void 0) {
19652
+ const registered4 = core.embeddingAdapters.get(configuredAdapter);
19653
+ if (registered4 !== void 0) {
19468
19654
  resolvedEmbeddingAdapter = validateEmbeddingConfig(
19469
- { fields, adapter: registered3 },
19655
+ { fields, adapter: registered4 },
19470
19656
  indexConfig.schema,
19471
19657
  config?.embedding
19472
19658
  );
@@ -19490,7 +19676,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
19490
19676
  }
19491
19677
  const language = getLanguage(indexConfig.language ?? "english");
19492
19678
  if (appliesCreationLimits) requireConfigurationWithinSizeLimit(indexConfig, language);
19493
- core.executor.createIndex(name, indexConfig, language);
19679
+ const synonyms = prepareConfiguredSynonyms(configured, language, appliesCreationLimits);
19680
+ core.executor.createIndex(name, indexConfig, language, false, synonyms?.lookup ?? null);
19494
19681
  core.indexRegistry.set(name, {
19495
19682
  config: indexConfig,
19496
19683
  language,
@@ -19501,7 +19688,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
19501
19688
  heldPartitions: null,
19502
19689
  documentCount: 0,
19503
19690
  partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
19504
- analysisRevision: language.revision
19691
+ analysisRevision: language.revision,
19692
+ synonyms: synonyms?.encoded ?? null
19505
19693
  });
19506
19694
  await core.indexState.registerOpen(name);
19507
19695
  if (core.durability) {
@@ -22103,7 +22291,7 @@ function portabilityOperations(core, config, engine) {
22103
22291
  };
22104
22292
  }
22105
22293
  async function restoreIndex(indexName, data) {
22106
- const { restoreFromSnapshot } = await import('./snapshot-5MXBEDXF.mjs');
22294
+ const { restoreFromSnapshot } = await import('./snapshot-5EJ7I7QY.mjs');
22107
22295
  await restoreFromSnapshot(indexName, data, {
22108
22296
  executor: core.executor,
22109
22297
  indexRegistry: core.indexRegistry,
@@ -22123,7 +22311,7 @@ function portabilityOperations(core, config, engine) {
22123
22311
  return {
22124
22312
  snapshot(indexName) {
22125
22313
  return withOpenIndex(indexName, async () => {
22126
- const { createSnapshot } = await import('./snapshot-5MXBEDXF.mjs');
22314
+ const { createSnapshot } = await import('./snapshot-5EJ7I7QY.mjs');
22127
22315
  return createSnapshot(snapshotDepsOf(indexName));
22128
22316
  });
22129
22317
  },
@@ -22135,8 +22323,8 @@ function portabilityOperations(core, config, engine) {
22135
22323
  exportDocuments(indexName) {
22136
22324
  return withOpenIndex(indexName, async () => {
22137
22325
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
22138
- import('./snapshot-5MXBEDXF.mjs'),
22139
- import('./write-6SBYTS62.mjs')
22326
+ import('./snapshot-5EJ7I7QY.mjs'),
22327
+ import('./write-K5V4XQ72.mjs')
22140
22328
  ]);
22141
22329
  const header = indexConfigurationOf(core.requireIndex(indexName));
22142
22330
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -22443,13 +22631,19 @@ function broadcastStatsForWorker(params, context, scoring) {
22443
22631
  if (term === void 0 || term.trim().length === 0) {
22444
22632
  return void 0;
22445
22633
  }
22446
- return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
22634
+ const { manager } = context;
22635
+ const analysis = { ...manager.analysis, synonyms: manager.synonyms };
22636
+ return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, analysis, {
22637
+ partitions: partitionsFor(manager, context.partitionIds),
22638
+ expansion: { options: params, schema: context.config.schema }
22639
+ });
22447
22640
  }
22448
22641
  function searchOptionsFor(manager, patternWork) {
22449
22642
  return {
22450
22643
  bm25Params: manager.config.bm25,
22451
22644
  stopWords: manager.analysis.stopWords,
22452
22645
  customTokenizer: manager.analysis.customTokenizer,
22646
+ synonyms: manager.synonyms,
22453
22647
  patternWork
22454
22648
  };
22455
22649
  }
@@ -22864,12 +23058,13 @@ function highlightField(text, queryTokens, language, options) {
22864
23058
  const prefixLength = options?.prefixLength ?? 0;
22865
23059
  const completedToken = options?.completedToken;
22866
23060
  const queryTolerances = queryTokens.map((qt) => qt.token === completedToken ? 0 : tolerance);
23061
+ const synonyms = options?.synonyms;
22867
23062
  const matchByFieldToken = /* @__PURE__ */ new Map();
22868
23063
  const matchesQuery = (fieldToken) => {
22869
23064
  const known = matchByFieldToken.get(fieldToken);
22870
23065
  if (known !== void 0) return known;
22871
23066
  const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
22872
- const matches = stemmedQueryTokens.some(
23067
+ const matches = synonyms?.includes(stemmedField) === true || stemmedQueryTokens.some(
22873
23068
  (stemmedQuery, index) => fuzzyTermMatches(stemmedQuery, stemmedField, queryTolerances[index], prefixLength)
22874
23069
  );
22875
23070
  matchByFieldToken.set(fieldToken, matches);
@@ -22932,7 +23127,7 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
22932
23127
  highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
22933
23128
  };
22934
23129
  }
22935
- function applyHighlights(hits, params, language, analysis, schema, readStoredDocument) {
23130
+ function applyHighlights(hits, params, language, analysis, schema, readStoredDocument, synonymLookup) {
22936
23131
  if (!params.highlight) return;
22937
23132
  const queryTokenResult = tokenize(params.term ?? "", language, {
22938
23133
  stem: true,
@@ -22957,6 +23152,7 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
22957
23152
  }
22958
23153
  }
22959
23154
  const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
23155
+ const synonyms = params.exact === true ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
22960
23156
  for (const hit of hits) {
22961
23157
  const highlights = {};
22962
23158
  const projected = hit.document;
@@ -22976,7 +23172,8 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
22976
23172
  completedToken,
22977
23173
  tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
22978
23174
  prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
22979
- partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0
23175
+ partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0,
23176
+ synonyms
22980
23177
  });
22981
23178
  }
22982
23179
  }
@@ -23489,7 +23686,7 @@ async function executeQuery(request, callerContext) {
23489
23686
  if (params.highlight) {
23490
23687
  const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
23491
23688
  const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
23492
- applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument);
23689
+ applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument, manager.synonyms);
23493
23690
  }
23494
23691
  const elapsed = now() - startTime;
23495
23692
  return {
@@ -23806,6 +24003,22 @@ async function runEngineListDocuments(core, indexName, params, partitionIds) {
23806
24003
  }
23807
24004
  }
23808
24005
 
24006
+ // src/narsil/synonyms.ts
24007
+ function synonymOperations(core) {
24008
+ return {
24009
+ async updateSynonyms(indexName, synonyms) {
24010
+ core.guardShutdown();
24011
+ const absent = core.requireIndex(indexName).synonyms === null && isEmptySynonymSet(synonyms);
24012
+ if (absent && availableSynonymModule() === null) return;
24013
+ await synonymModule().replace(core, indexName, synonyms, true);
24014
+ },
24015
+ async getSynonyms(indexName) {
24016
+ core.guardShutdown();
24017
+ return rulesOfStoredSynonyms(core.requireIndex(indexName).synonyms);
24018
+ }
24019
+ };
24020
+ }
24021
+
23809
24022
  // src/narsil/operations.ts
23810
24023
  function createNarsilFromCore(core, config) {
23811
24024
  const {
@@ -23960,6 +24173,7 @@ function createNarsilFromCore(core, config) {
23960
24173
  await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
23961
24174
  },
23962
24175
  ...portabilityOperations(core, config, () => narsil),
24176
+ ...synonymOperations(core),
23963
24177
  async checkpoint(indexName) {
23964
24178
  guardShutdown();
23965
24179
  requireIndex(indexName);
@@ -24100,4 +24314,4 @@ async function createNarsil(config) {
24100
24314
  // src/index.ts
24101
24315
  var VERSION = "0.1.0";
24102
24316
 
24103
- export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, raiseEarlierReleaseGraphSetting, readHeader, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerTokenizer, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, segmentArraysOf, selectionKeeps, setNestedValue, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
24317
+ export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, setNestedValue, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };