@delali/narsil 0.3.1-next.162 → 0.3.1-next.169

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +24 -8
  2. package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
  3. package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
  4. package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
  5. package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
  6. package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
  7. package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
  8. package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
  9. package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
  10. package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
  11. package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
  12. package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
  13. package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
  14. package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
  15. package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
  16. package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
  17. package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
  18. package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
  19. package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
  20. package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
  21. package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
  22. package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
  23. package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
  24. package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
  25. package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
  26. package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
  27. package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
  28. package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
  29. package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
  30. package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
  31. package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
  32. package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
  33. package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
  34. package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
  35. package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
  36. package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
  37. package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
  38. package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
  39. package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
  40. package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
  41. package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
  42. package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
  43. package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
  44. package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
  45. package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
  46. package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
  47. package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
  48. package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
  49. package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
  50. package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
  51. package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
  52. package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
  53. package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
  54. package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
  55. package/dist/client.d.ts +8 -8
  56. package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
  57. package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
  58. package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
  59. package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
  60. package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
  61. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  62. package/dist/distribution/coordinator/etcd.mjs +1 -1
  63. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  64. package/dist/distribution/transport/grpc.mjs +1 -1
  65. package/dist/distribution/transport/tcp.mjs +1 -1
  66. package/dist/distribution.d.ts +7 -7
  67. package/dist/distribution.mjs +39 -39
  68. package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
  69. package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
  70. package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
  71. package/dist/index.browser.mjs +3 -3
  72. package/dist/index.d.ts +6 -6
  73. package/dist/index.mjs +7 -7
  74. package/dist/index.react-native.mjs +6 -6
  75. package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
  76. package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
  77. package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
  78. package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
  79. package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
  80. package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
  81. package/dist/partial-words.browser.mjs +3 -3
  82. package/dist/partial-words.mjs +7 -7
  83. package/dist/pattern.browser.mjs +2 -2
  84. package/dist/pattern.mjs +2 -2
  85. package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
  86. package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
  87. package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
  88. package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
  89. package/dist/query-suggestions/merge-worker.mjs +10 -10
  90. package/dist/query-suggestions.browser.mjs +3 -3
  91. package/dist/query-suggestions.d.ts +2 -2
  92. package/dist/query-suggestions.mjs +8 -8
  93. package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
  94. package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
  95. package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
  96. package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
  97. package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
  98. package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
  99. package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
  100. package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
  101. package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
  102. package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
  103. package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
  104. package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
  105. package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
  106. package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
  107. package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
  108. package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
  109. package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
  110. package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
  111. package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
  112. package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
  113. package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
  114. package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
  115. package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
  116. package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
  117. package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
  118. package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
  119. package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
  120. package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
  121. package/dist/react-native/partial-words.mjs +6 -6
  122. package/dist/react-native/pattern.mjs +3 -3
  123. package/dist/react-native/query-suggestions.mjs +6 -6
  124. package/dist/react-native/sync.mjs +6 -6
  125. package/dist/react-native/synonyms.mjs +6 -6
  126. package/dist/react.d.ts +9 -9
  127. package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
  128. package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
  129. package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
  130. package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
  131. package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
  132. package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
  133. package/dist/server.d.ts +10 -10
  134. package/dist/server.mjs +26 -26
  135. package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
  136. package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
  137. package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
  138. package/dist/sync/filesystem.d.ts +3 -3
  139. package/dist/sync/filesystem.mjs +15 -15
  140. package/dist/sync/indexeddb.browser.mjs +1 -1
  141. package/dist/sync/indexeddb.d.ts +3 -3
  142. package/dist/sync/indexeddb.mjs +2 -2
  143. package/dist/sync.browser.mjs +3 -3
  144. package/dist/sync.d.ts +4 -4
  145. package/dist/sync.mjs +7 -7
  146. package/dist/synonyms.browser.mjs +3 -3
  147. package/dist/synonyms.mjs +7 -7
  148. package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
  149. package/dist/thread-OWXVP2ZA.mjs +30 -0
  150. package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
  151. package/dist/vector/search-worker.d.ts +1 -1
  152. package/dist/vector/search-worker.mjs +5 -5
  153. package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
  154. package/dist/workers/entry.mjs +13 -13
  155. package/package.json +9 -8
  156. package/dist/thread-XWDK6TAR.mjs +0 -30
  157. /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
@@ -1,5 +1,5 @@
1
- import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-AUXXPSFB.mjs';
2
- import { flattenSchema } from './chunk-FTQ7JNMU.mjs';
1
+ import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-F4MIGBBP.mjs';
2
+ import { flattenSchema } from './chunk-ZDDJVR7X.mjs';
3
3
  import { MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-VUJCSQG5.mjs';
4
4
  import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
5
5
  import { encode } from '@msgpack/msgpack';
@@ -1,7 +1,7 @@
1
- import { createSurfaceRegistry, SegmentRefusal, encodeSurfaceTable, encodePartialSpellingsOf, encodeStringBlob, mergeTextAt, generateId, encodeExternalIdTableData, buildPartialWordTableFrom, wrapFrozenTokenTable, encodeDocumentTableData, topLevelMapKeys, mergeStoredFields, prepareDocumentText, recordValueSize, patternIndexOf, encodeFieldIndexes, mergeFieldRegions, sameFields, partialSpellingsOfDocuments, storedDocumentsOf, encodeFrozenTokenTableData, mergeKeywordSources, GrowingSegment, LivePostings, createSpellingTable, createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex } from './chunk-AUXXPSFB.mjs';
2
- import { storedText, bitsetIterator } from './chunk-4EXKQ2IB.mjs';
1
+ import { createSurfaceRegistry, SegmentRefusal, encodeSurfaceTable, encodePartialSpellingsOf, encodeStringBlob, mergeTextAt, generateId, encodeExternalIdTableData, buildPartialWordTableFrom, wrapFrozenTokenTable, encodeDocumentTableData, topLevelMapKeys, mergeStoredFields, prepareDocumentText, recordValueSize, patternIndexOf, encodeFieldIndexes, mergeFieldRegions, sameFields, partialSpellingsOfDocuments, storedDocumentsOf, encodeFrozenTokenTableData, mergeKeywordSources, GrowingSegment, LivePostings, createSpellingTable, createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex } from './chunk-F4MIGBBP.mjs';
2
+ import { storedText, bitsetIterator } from './chunk-SK3DPAI4.mjs';
3
3
  import { encodeDocument } from './chunk-YOTP2FPE.mjs';
4
- import { schemaFieldsOf, getNestedValue, isWordIndexedBase, validateDocument, validateDocumentStrict } from './chunk-FTQ7JNMU.mjs';
4
+ import { schemaFieldsOf, getNestedValue, isWordIndexedBase, validateDocument, validateDocumentStrict } from './chunk-ZDDJVR7X.mjs';
5
5
  import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
6
6
 
7
7
  // src/engine/orchestration/eligibility.ts
@@ -1,5 +1,5 @@
1
- import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-3RUXKZJG.mjs';
2
- import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-AUXXPSFB.mjs';
1
+ import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-F6BJYCF2.mjs';
2
+ import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-F4MIGBBP.mjs';
3
3
  import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
4
4
 
5
5
  // src/query-suggestions/engine-hooks.ts
@@ -1,5 +1,5 @@
1
- import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-AUXXPSFB.mjs';
2
- import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-FTQ7JNMU.mjs';
1
+ import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-F4MIGBBP.mjs';
2
+ import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-ZDDJVR7X.mjs';
3
3
 
4
4
  // src/core/partition/pattern-snapshot.ts
5
5
  function serializePatternIndexes(state, schema) {
@@ -1,6 +1,6 @@
1
- import { stopUsingTheNativeSearchCore, nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_OK, NATIVE_NEEDS_ROOM, NATIVE_GRAPH_NOT_HELD_ALONE } from './chunk-XBTFK3AZ.mjs';
1
+ import { stopUsingTheNativeSearchCore, nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_OK, NATIVE_NEEDS_ROOM, NATIVE_GRAPH_NOT_HELD_ALONE } from './chunk-WFQ6YQZA.mjs';
2
2
  import { reactNativeHost } from './chunk-PVMEHBQO.mjs';
3
- import { DEFAULT_EF_SEARCH, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, MAX_M, compareCodePoints, INITIAL_UPPER_SLOTS, CONNECT_RESTARTS_BEFORE_ONE_PASS, FARTHEST_REJECTED_LINKS_KEPT, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, NEAREST_LINK_FALLBACK_HOSTS, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-HRDMIQ63.mjs';
3
+ import { DEFAULT_EF_SEARCH, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, MAX_M, MIN_M, compareCodePoints, INITIAL_UPPER_SLOTS, CONNECT_RESTARTS_BEFORE_ONE_PASS, FARTHEST_REJECTED_LINKS_KEPT, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, NEAREST_LINK_FALLBACK_HOSTS } from './chunk-HRDMIQ63.mjs';
4
4
  import { MAX_FILE_BYTES } from './chunk-VUJCSQG5.mjs';
5
5
  import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
6
6
 
@@ -487,6 +487,8 @@ var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
487
487
  var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
488
488
  var MAX_INDEX_NAME_LENGTH = 255;
489
489
  var MAX_DOC_ID_LENGTH = 512;
490
+ var DEFAULT_MAX_PARTITIONS = 1;
491
+ var MAX_PARTITIONS = 65536;
490
492
  var EXPORT_CHUNK_BYTES = 1048576;
491
493
  var MAX_EXPORT_LINE_BYTES = 117443592;
492
494
  var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
@@ -581,14 +583,16 @@ function growBufferToExactly(buffer, bytes) {
581
583
  if (buffer.byteLength >= bytes) return;
582
584
  resizeBuffer(buffer, bytes, bytes);
583
585
  }
586
+ function requireBytesWithinLimit(bytes, maxBytes) {
587
+ if (bytes <= maxBytes) return;
588
+ throw new NarsilError(
589
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
590
+ `A vector structure needs ${bytes} bytes where it may hold at most ${maxBytes}`,
591
+ { bytes, maxBytes }
592
+ );
593
+ }
584
594
  function resizeBuffer(buffer, bytes, target) {
585
- if (bytes > buffer.maxByteLength) {
586
- throw new NarsilError(
587
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
588
- `A vector structure needs ${bytes} bytes where it may hold at most ${buffer.maxByteLength}`,
589
- { bytes, maxBytes: buffer.maxByteLength }
590
- );
591
- }
595
+ requireBytesWithinLimit(bytes, buffer.maxByteLength);
592
596
  if (buffer instanceof ArrayBuffer) {
593
597
  buffer.resize(target);
594
598
  return;
@@ -3491,6 +3495,16 @@ function markTombstone(state, ord) {
3491
3495
  Atomics.add(state.header, GRAPH_TOMBSTONE_COUNT, 1);
3492
3496
  }
3493
3497
  }
3498
+ function reviveTombstone(state, ord) {
3499
+ if (!nodeExists(state, ord) || !isTombstoned(state, ord)) return;
3500
+ lockGraphShared(state.locks);
3501
+ try {
3502
+ clearTombstone(state, ord);
3503
+ } finally {
3504
+ unlockGraphShared(state.locks);
3505
+ }
3506
+ state.linkWrites += 1;
3507
+ }
3494
3508
  function compactTombstones(state, mayFree2) {
3495
3509
  if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
3496
3510
  countIncomingBeforeWriting(state, true);
@@ -4733,6 +4747,7 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
4733
4747
  if (ord !== void 0) markTombstone(state, ord);
4734
4748
  },
4735
4749
  markTombstoneOrdinal: (ordinal) => markTombstone(state, ordinal),
4750
+ reviveOrdinal: (ordinal) => reviveTombstone(state, ordinal),
4736
4751
  holdsNode: (ordinal) => hasNode(state.adjacency, ordinal),
4737
4752
  holdsTombstone: (ordinal) => hasNode(state.adjacency, ordinal) && isTombstoned(state, ordinal),
4738
4753
  has: (docId) => {
@@ -5612,6 +5627,30 @@ function decodeVectorIndexParts(raw) {
5612
5627
  return raw.map(decodeVectorIndexPart);
5613
5628
  }
5614
5629
 
5630
+ // src/vector/vector-index/upper-list-bound.ts
5631
+ var UPPER_LISTS_PER_NODE_FACTOR = 4;
5632
+ var UPPER_LISTS_ALLOWANCE = 64;
5633
+ function requireBoundedUpperLists(levelsOfParts, m, what) {
5634
+ let nodes = 0;
5635
+ let upperLists = 0;
5636
+ for (const levels of levelsOfParts) {
5637
+ for (let at = 0; at < levels.length; at++) {
5638
+ const level2 = levels[at];
5639
+ if (level2 === 0) continue;
5640
+ nodes++;
5641
+ upperLists += level2 - 1;
5642
+ }
5643
+ }
5644
+ const spread = m - 1;
5645
+ if (upperLists * spread <= UPPER_LISTS_PER_NODE_FACTOR * nodes + UPPER_LISTS_ALLOWANCE * spread) return;
5646
+ const limit = Math.floor(UPPER_LISTS_PER_NODE_FACTOR * nodes / spread) + UPPER_LISTS_ALLOWANCE;
5647
+ throw new NarsilError(
5648
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5649
+ `${what} holds ${upperLists} upper-layer lists over ${nodes} nodes, while a graph of ${nodes} nodes at m ${m} holds at most ${limit}`,
5650
+ { upperLists, nodes, m, limit }
5651
+ );
5652
+ }
5653
+
5615
5654
  // src/vector/vector-index/checkpoint-payload.ts
5616
5655
  var VECTOR_FILE_PAYLOAD_VERSION = 1;
5617
5656
  var VECTOR_GRAPH_PAYLOAD_VERSION = 1;
@@ -5674,15 +5713,15 @@ function calibratedMeanLengthOf(name, value) {
5674
5713
  }
5675
5714
  return value;
5676
5715
  }
5677
- function decodeNumberedGraph(raw, minimumM) {
5716
+ function decodeNumberedGraph(raw) {
5678
5717
  const name = "vector graph";
5679
5718
  if (!isRecord2(raw)) invalid2(name, "each graph must be a map");
5680
5719
  const entryPoint = raw.entryPoint ?? null;
5681
5720
  if (entryPoint !== null && !isIntegerUpTo(entryPoint, 4294967295))
5682
5721
  invalid2(name, "entryPoint must be a uint32 or nil");
5683
5722
  if (!isIntegerUpTo(raw.maxLayer, 255)) invalid2(name, "maxLayer must be a uint8");
5684
- if (!isIntegerUpTo(raw.m, MAX_M) || raw.m < minimumM) {
5685
- invalid2(name, `m must be a whole number from ${minimumM} to ${MAX_M}`);
5723
+ if (!isIntegerUpTo(raw.m, MAX_M) || raw.m < MIN_M) {
5724
+ invalid2(name, `m must be a whole number from ${MIN_M} to ${MAX_M}`);
5686
5725
  }
5687
5726
  if (!isIntegerUpTo(raw.efConstruction, 65535)) invalid2(name, "efConstruction must be a uint16");
5688
5727
  if (!isMetric(raw.metric)) invalid2(name, "metric must name a supported metric", { metric: raw.metric });
@@ -5713,7 +5752,7 @@ function decodeVectorGraphPayload(raw) {
5713
5752
  return {
5714
5753
  v: VECTOR_GRAPH_PAYLOAD_VERSION,
5715
5754
  fieldName: raw.fieldName,
5716
- graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph, EARLIER_RELEASE_MIN_M))
5755
+ graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph))
5717
5756
  };
5718
5757
  }
5719
5758
  function decodeVectorGraphPartPayload(raw) {
@@ -5725,7 +5764,7 @@ function decodeVectorGraphPartPayload(raw) {
5725
5764
  return {
5726
5765
  v: VECTOR_GRAPH_PART_PAYLOAD_VERSION,
5727
5766
  fieldName: raw.fieldName,
5728
- graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph, MIN_M))
5767
+ graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph))
5729
5768
  };
5730
5769
  }
5731
5770
  function sameGraphHeader(a, b) {
@@ -5767,10 +5806,21 @@ function assembleGraphParts(parts) {
5767
5806
  }
5768
5807
  requireOneGraphPerVector(part, p);
5769
5808
  }
5809
+ for (let g = 0; g < partLists.length; g++) {
5810
+ const list = partLists[g];
5811
+ requireBoundedUpperLists(
5812
+ list.map((graph) => graph.levels),
5813
+ first.graphs[g].m,
5814
+ `Graph ${g} of the vector graph parts of field "${first.fieldName}"`
5815
+ );
5816
+ }
5770
5817
  const graphs = partLists.map((list) => ({ partSize: VECTOR_GRAPH_PART_VECTORS, parts: list }));
5771
5818
  return { fieldName: first.fieldName, graphs };
5772
5819
  }
5773
5820
  function graphPartsOfEarlierPayload(payload) {
5821
+ payload.graphs.forEach((graph, g) => {
5822
+ requireBoundedUpperLists([graph.levels], graph.m, `Graph ${g} of the vector graph of field "${payload.fieldName}"`);
5823
+ });
5774
5824
  return {
5775
5825
  fieldName: payload.fieldName,
5776
5826
  graphs: payload.graphs.map((graph) => ({ partSize: Math.max(graph.levels.length, 1), parts: [graph] }))
@@ -6724,4 +6774,4 @@ function holdWritesAndBackgroundWork(read) {
6724
6774
  function adoptBackgroundScheduler(_scheduler) {
6725
6775
  }
6726
6776
 
6727
- export { ABSOLUTE_WINDOWS_PATH_PATTERN, ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, CURSOR_BINDING_BUFFER_BYTES, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_FILTER_THRESHOLD, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_RANK_CONSTANT, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_SUGGEST_LIMIT, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DISK_STORAGE_BLOCK_BYTES, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, EXPORT_CHUNK_BYTES, FNV_OFFSET_BASIS, FNV_PRIME, FOREIGN_STATS_TTL_MS, FUSION_TIE_INSERTION_LIMIT, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_RETIRED, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HYBRID_RESCORE_WINDOW, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, IN_MEMORY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_EXPORT_LINE_BYTES, MAX_HYBRID_DEPTH, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LINEAR_ALPHA, MAX_PHRASE_WORDS, MAX_PREFIX_LENGTH, MAX_QUERY_PHRASES, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_OVERSAMPLE, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, RESULT_WINDOW, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, SHUTDOWN_TIMEOUT_MS, STATISTICS_BROADCAST_INTERVAL_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_MEMORY_LIMITS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_WORKER_COPIES_ALLOWED, WASM_PAGE_BYTES, WORKER_COPY_MIN_VECTORS, WORKER_YOUNG_GENERATION_MB, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, adoptSharedTurns, allLiveDocIds, answerFromOneState, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, backgroundTurnIsSpent, bindSearchSignal, bindVectorCoreHost, bindVectorHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, changeIndexOutsideExternalWaits, collectAndReturnPages, collectYoungGeneration, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, countWrite, coverOrdinal, createCentroidSums, createDocIdMap, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, detectRuntime, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUtf16, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, holdWritesAndBackgroundWork, indexChangeOutsideExternalWaits, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, nodeThreadId, noteThreadSlotGone, noteThreadSlotLive, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requirePinnedState, requireSearchActive, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedStopOfSignal, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signalOfSearch, signatureOf, simdModuleLoaded, slotByteOffset, spawnNodeWorker, splitWorkerBudget, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, stopOfSignal, storePinsOf, threadSlotGone, threadSlotOfWorker, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, turnIsSpent, turnsAreBudgeted, unplacedChunkCount, vectorsPerFile, vectorsToBytes, waitOutsideCountedWrite, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBackgroundTurn, yieldBuildTurn, yieldTurn };
6777
+ export { ABSOLUTE_WINDOWS_PATH_PATTERN, ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, CURSOR_BINDING_BUFFER_BYTES, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_FILTER_THRESHOLD, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PARTITIONS, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_RANK_CONSTANT, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_SUGGEST_LIMIT, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DISK_STORAGE_BLOCK_BYTES, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, EXPORT_CHUNK_BYTES, FNV_OFFSET_BASIS, FNV_PRIME, FOREIGN_STATS_TTL_MS, FUSION_TIE_INSERTION_LIMIT, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_RETIRED, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HYBRID_RESCORE_WINDOW, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, IN_MEMORY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_EXPORT_LINE_BYTES, MAX_HYBRID_DEPTH, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LINEAR_ALPHA, MAX_PARTITIONS, MAX_PHRASE_WORDS, MAX_PREFIX_LENGTH, MAX_QUERY_PHRASES, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_OVERSAMPLE, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, RESULT_WINDOW, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, SHUTDOWN_TIMEOUT_MS, STATISTICS_BROADCAST_INTERVAL_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_MEMORY_LIMITS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_WORKER_COPIES_ALLOWED, WASM_PAGE_BYTES, WORKER_COPY_MIN_VECTORS, WORKER_YOUNG_GENERATION_MB, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, adoptSharedTurns, allLiveDocIds, answerFromOneState, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, backgroundTurnIsSpent, bindSearchSignal, bindVectorCoreHost, bindVectorHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, changeIndexOutsideExternalWaits, collectAndReturnPages, collectYoungGeneration, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, countWrite, coverOrdinal, createCentroidSums, createDocIdMap, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, detectRuntime, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUtf16, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, holdWritesAndBackgroundWork, indexChangeOutsideExternalWaits, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, nodeThreadId, noteThreadSlotGone, noteThreadSlotLive, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requireBoundedUpperLists, requireBytesWithinLimit, requirePinnedState, requireSearchActive, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedStopOfSignal, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signalOfSearch, signatureOf, simdModuleLoaded, slotByteOffset, spawnNodeWorker, splitWorkerBudget, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, stopOfSignal, storePinsOf, threadSlotGone, threadSlotOfWorker, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, turnIsSpent, turnsAreBudgeted, unplacedChunkCount, vectorsPerFile, vectorsToBytes, waitOutsideCountedWrite, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBackgroundTurn, yieldBuildTurn, yieldTurn };
@@ -1,4 +1,4 @@
1
- import { REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, collectYoungGeneration, collectAndReturnPages, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE } from './chunk-VIPNDN4B.mjs';
1
+ import { REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, collectYoungGeneration, collectAndReturnPages, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE } from './chunk-I7DTUOK2.mjs';
2
2
 
3
3
  // src/runtime/thread-heap.ts
4
4
  var reservedBytes = 0;
@@ -1,4 +1,4 @@
1
- import { getNestedValue } from './chunk-R4CE4IVU.mjs';
1
+ import { getNestedValue } from './chunk-ZDDJVR7X.mjs';
2
2
 
3
3
  // src/serialization/vector-field-bytes.ts
4
4
  var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
@@ -1,4 +1,4 @@
1
- import { requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-FTQ7JNMU.mjs';
1
+ import { requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-ZDDJVR7X.mjs';
2
2
  import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
3
3
 
4
4
  // src/core/bitset.ts
@@ -1,7 +1,7 @@
1
1
  import { reactNativeHost } from './chunk-PVMEHBQO.mjs';
2
2
 
3
3
  // src/vector/native/types/index.ts
4
- var NATIVE_CORE_ABI_VERSION = 50;
4
+ var NATIVE_CORE_ABI_VERSION = 51;
5
5
  var NATIVE_OK = 0;
6
6
  var NATIVE_NEEDS_ROOM = 3;
7
7
  var NATIVE_GRAPH_NOT_HELD_ALONE = 4;
@@ -16,6 +16,11 @@ var PARSED_FIELD_TYPE_CACHE_ENTRIES = 1024;
16
16
 
17
17
  // src/schema/validator/shared.ts
18
18
  var VECTOR_PATTERN = /^vector\[(\d+)]$/;
19
+ var VECTOR_TYPE_PATTERN = /^vector\[(.*)]$/;
20
+ function vectorDimensionOf(fieldType) {
21
+ const match = VECTOR_PATTERN.exec(fieldType);
22
+ return match === null ? null : Number.parseInt(match[1], 10);
23
+ }
19
24
  function isGeopointOrVectorType(fieldType) {
20
25
  return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
21
26
  }
@@ -57,15 +62,15 @@ function invalidType(path, type, message) {
57
62
  }
58
63
  function parseBase(baseName, path, type) {
59
64
  if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
60
- const vectorMatch = VECTOR_PATTERN.exec(baseName);
65
+ const vectorMatch = VECTOR_TYPE_PATTERN.exec(baseName);
61
66
  if (vectorMatch === null) {
62
67
  invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
63
68
  }
64
- const dimension = Number.parseInt(vectorMatch[1], 10);
65
- if (dimension <= 0 || dimension > MAX_STORED_VECTOR_DIMENSION) {
69
+ const dimension = vectorDimensionOf(baseName) ?? Number.NaN;
70
+ if (!(dimension > 0 && dimension <= MAX_STORED_VECTOR_DIMENSION)) {
66
71
  throw new NarsilError(
67
72
  ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
68
- `Vector field "${path}" has ${dimension} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
73
+ `Vector field "${path}" has ${vectorMatch[1]} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
69
74
  { field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
70
75
  );
71
76
  }
@@ -218,7 +223,7 @@ function validateVector(path, value, expectedDimension) {
218
223
  );
219
224
  }
220
225
  for (let i = 0; i < arr.length; i++) {
221
- if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
226
+ if (typeof arr[i] !== "number" || !Number.isFinite(Math.fround(arr[i]))) {
222
227
  throw new NarsilError(
223
228
  ErrorCodes.DOC_VALIDATION_FAILED,
224
229
  `Field "${path}" vector element at index ${i} is not a valid number`,
@@ -802,7 +807,6 @@ function getOrCreateFieldNameIndex(table, fieldName) {
802
807
  table.indexMap.set(fieldName, idx);
803
808
  return idx;
804
809
  }
805
- var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
806
810
  function getNestedValue(obj, path) {
807
811
  if (obj === null || typeof obj !== "object") return void 0;
808
812
  if (!path.includes(".")) return Object.hasOwn(obj, path) ? obj[path] : void 0;
@@ -838,8 +842,7 @@ function textFieldsChanged(oldDoc, newDoc, fields) {
838
842
  return false;
839
843
  }
840
844
  function parseVectorDimension(fieldType) {
841
- const match = VECTOR_PATTERN2.exec(fieldType);
842
- return match ? Number.parseInt(match[1], 10) : null;
845
+ return vectorDimensionOf(fieldType);
843
846
  }
844
847
  function tokenizeOptions(options) {
845
848
  return {
@@ -847,7 +850,8 @@ function tokenizeOptions(options) {
847
850
  removeStopWords: true,
848
851
  collectSurfaces: options?.collectSurfaces === true,
849
852
  stopWordOverride: options?.stopWordOverride,
850
- customTokenizer: options?.customTokenizer
853
+ customTokenizer: options?.customTokenizer,
854
+ textSource: "document"
851
855
  };
852
856
  }
853
857
  function getFlatSchema(state, schema) {
@@ -862,7 +866,7 @@ var loaded = null;
862
866
  var loading = null;
863
867
  function loadMessagePack() {
864
868
  if (loaded !== null) return Promise.resolve(loaded);
865
- loading ??= import('./msgpack-leaf-TR66ZVAX.mjs').then(
869
+ loading ??= import('./msgpack-leaf-GTKLF6GJ.mjs').then(
866
870
  (code) => {
867
871
  code.bindMsgpackHost({
868
872
  BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER,
@@ -895,4 +899,4 @@ function requireLoadedMessagePack() {
895
899
  return loaded;
896
900
  }
897
901
 
898
- export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_NESTING_DEPTH, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, PROTOTYPE_POLLUTION_KEYS, VECTOR_PATTERN, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isPlainObject, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, textFieldsChanged, tokenizeOptions, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, withSortableOption };
902
+ export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_NESTING_DEPTH, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, PROTOTYPE_POLLUTION_KEYS, VECTOR_PATTERN, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isPlainObject, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, textFieldsChanged, tokenizeOptions, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, vectorDimensionOf, withSortableOption };
@@ -1,5 +1,5 @@
1
- export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_RETIRED, IN_MEMORY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_OVERSAMPLE, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SEARCH_SLOTS, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_MEMORY_LIMITS, VECTOR_SCRATCH_SLOTS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_WORKER_COPIES_ALLOWED, WASM_PAGE_BYTES, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, backgroundTurnIsSpent, bindVectorCoreHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, collectAndReturnPages, collectYoungGeneration, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, coverOrdinal, createCentroidSums, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, holdWritesAndBackgroundWork, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signatureOf, simdModuleLoaded, slotByteOffset, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, storePinsOf, threadSlotGone, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, turnsAreBudgeted, unplacedChunkCount, vectorsPerFile, vectorsToBytes, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBackgroundTurn, yieldBuildTurn } from './chunk-VIPNDN4B.mjs';
2
- export { NATIVE_OK, loadNativeCore, nativeSearchCoreIsEnabled, noteFallback } from './chunk-XBTFK3AZ.mjs';
1
+ export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_RETIRED, IN_MEMORY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_OVERSAMPLE, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SEARCH_SLOTS, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_MEMORY_LIMITS, VECTOR_SCRATCH_SLOTS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_WORKER_COPIES_ALLOWED, WASM_PAGE_BYTES, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, backgroundTurnIsSpent, bindVectorCoreHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, collectAndReturnPages, collectYoungGeneration, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, coverOrdinal, createCentroidSums, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, holdWritesAndBackgroundWork, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requireBoundedUpperLists, requireBytesWithinLimit, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signatureOf, simdModuleLoaded, slotByteOffset, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, storePinsOf, threadSlotGone, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, turnsAreBudgeted, unplacedChunkCount, vectorsPerFile, vectorsToBytes, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBackgroundTurn, yieldBuildTurn } from './chunk-I7DTUOK2.mjs';
2
+ export { NATIVE_OK, loadNativeCore, nativeSearchCoreIsEnabled, noteFallback } from './chunk-WFQ6YQZA.mjs';
3
3
  import './chunk-PVMEHBQO.mjs';
4
4
  export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, CONNECT_RESTARTS_BEFORE_ONE_PASS, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, DEFAULT_EF_SEARCH, EARLIER_RELEASE_MIN_M, FARTHEST_REJECTED_LINKS_KEPT, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, MIN_M, NEAREST_LINK_FALLBACK_HOSTS, PLACEMENT_ROOM_REQUESTS, compareCodePoints } from './chunk-HRDMIQ63.mjs';
5
5
  export { MAX_FILE_BYTES } from './chunk-VUJCSQG5.mjs';
@@ -0,0 +1,59 @@
1
+ // src/analysis/constants.ts
2
+ var MAX_TOKEN_POSITION = 4294967295;
3
+
4
+ // src/analysis/custom-token-host.ts
5
+ var ErrorCodes;
6
+ var NarsilError;
7
+ var withUnpairedSurrogatesReplaced;
8
+ function bindCustomTokenHost(host) {
9
+ ({ ErrorCodes, NarsilError, withUnpairedSurrogatesReplaced } = host);
10
+ }
11
+
12
+ // src/analysis/custom-token-results.ts
13
+ function refuseTokenizerResult(source, message, details) {
14
+ const code = source === "document" ? ErrorCodes.DOC_VALIDATION_FAILED : ErrorCodes.CONFIG_INVALID;
15
+ throw new NarsilError(code, `The custom tokenizer ${message}`, details);
16
+ }
17
+ function kindOf(value) {
18
+ if (value === null) return "null";
19
+ return Array.isArray(value) ? "array" : typeof value;
20
+ }
21
+ function checkedTokenizerEntry(entry, at, source) {
22
+ const { token, position } = entry !== null && typeof entry === "object" ? entry : {};
23
+ if (typeof token !== "string") {
24
+ refuseTokenizerResult(
25
+ source,
26
+ `returned a token of type ${kindOf(token)} at entry ${at}, where a token is a string`,
27
+ { entry: at, tokenType: kindOf(token) }
28
+ );
29
+ }
30
+ if (typeof position !== "number" || !Number.isInteger(position) || position < 0 || position > MAX_TOKEN_POSITION) {
31
+ const shown = typeof position === "number" ? String(position) : kindOf(position);
32
+ refuseTokenizerResult(
33
+ source,
34
+ `returned the position ${shown} at entry ${at}, where a position is a whole number from 0 to ${MAX_TOKEN_POSITION}`,
35
+ { entry: at, position: shown }
36
+ );
37
+ }
38
+ return { token: withUnpairedSurrogatesReplaced(token), position };
39
+ }
40
+ function holdsPositionsOutOfOrder(entries) {
41
+ for (let index = 1; index < entries.length; index++) {
42
+ if (entries[index].position < entries[index - 1].position) return true;
43
+ }
44
+ return false;
45
+ }
46
+ function customTokensOf(tokenizer, text, source) {
47
+ const entries = tokenizer.tokenize(text);
48
+ if (!Array.isArray(entries)) {
49
+ refuseTokenizerResult(source, `returned ${kindOf(entries)}, where it returns a list of tokens`, {
50
+ received: kindOf(entries)
51
+ });
52
+ }
53
+ const checked = new Array(entries.length);
54
+ for (let at = 0; at < checked.length; at++) checked[at] = checkedTokenizerEntry(entries[at], at, source);
55
+ if (!holdsPositionsOutOfOrder(checked)) return checked;
56
+ return checked.sort((first, second) => first.position - second.position);
57
+ }
58
+
59
+ export { bindCustomTokenHost, customTokensOf };
@@ -33,10 +33,10 @@ function validateVector(vector, targetField, expectedDimensions) {
33
33
  );
34
34
  }
35
35
  for (let i = 0; i < vector.length; i++) {
36
- if (Number.isNaN(vector[i])) {
36
+ if (!Number.isFinite(vector[i])) {
37
37
  throw new NarsilError(
38
38
  ErrorCodes.EMBEDDING_FAILED,
39
- `Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
39
+ `Adapter returned vector with ${vector[i]} at index ${i} for field "${targetField}"`,
40
40
  { field: targetField, index: i }
41
41
  );
42
42
  }