@delali/narsil 0.3.1-next.162 → 0.3.1-next.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
- package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
- package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
- package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
- package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
- package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
- package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
- package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
- package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
- package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
- package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
- package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
- package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
- package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
- package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
- package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
- package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
- package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
- package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
- package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
- package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
- package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
- package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
- package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
- package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
- package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
- package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
- package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
- package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
- package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
- package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
- package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
- package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
- package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
- package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
- package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
- package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
- package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
- package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
- package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
- package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
- package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
- package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
- package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
- package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
- package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
- package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
- package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
- package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
- package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
- package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
- package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +39 -39
- package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
- package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
- package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +7 -7
- package/dist/index.react-native.mjs +6 -6
- package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
- package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
- package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
- package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
- package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
- package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +7 -7
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
- package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
- package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
- package/dist/query-suggestions/merge-worker.mjs +10 -10
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +8 -8
- package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
- package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
- package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
- package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
- package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
- package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
- package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
- package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
- package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
- package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
- package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
- package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
- package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
- package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
- package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
- package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
- package/dist/react-native/partial-words.mjs +6 -6
- package/dist/react-native/pattern.mjs +3 -3
- package/dist/react-native/query-suggestions.mjs +6 -6
- package/dist/react-native/sync.mjs +6 -6
- package/dist/react-native/synonyms.mjs +6 -6
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
- package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
- package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
- package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
- package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
- package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +26 -26
- package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
- package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
- package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +15 -15
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +3 -3
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +7 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +7 -7
- package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
- package/dist/thread-OWXVP2ZA.mjs +30 -0
- package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
- package/dist/workers/entry.mjs +13 -13
- package/package.json +9 -8
- package/dist/thread-XWDK6TAR.mjs +0 -30
- /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { reservedHeapBytes, reserveHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, admittedHeapBytes } from './chunk-
|
|
2
|
-
import { MIN_DOCUMENTS_FOR_SEGMENTS, fnv1aUtf16, MAX_INDEX_CONFIGURATION_BYTES, changeIndexOutsideExternalWaits, BATCH_CHUNK_SIZE, turnsAreBudgeted, turnIsSpent, yieldTurn, createDocIdMap, detectRuntime, fnv1a, indexChangeOutsideExternalWaits, BM25_DISPLAYED_NAME_LENGTH, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, splitWorkerBudget, resolveWorkerCount, yieldBackgroundTurn, backgroundTurnIsSpent, FNV_OFFSET_BASIS, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, DEFAULT_REQUEST_TIMEOUT_MS, answerFromOneState, countWrite, HEAP_PRESSURE_WARNING_FRACTION, MIN_SWEEP_INTERVAL_MS, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, signalOfSearch, requirePinnedState, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, stopOfSignal, DEFAULT_PAGE_SIZE, CURSOR_BINDING_BUFFER_BYTES, fnv1aContinue, sharedStopOfSignal, requireSearchActive, fnv1aContinueTaggedNumbers, PREFIX_MAX_EXPANSIONS, FNV_PRIME, collectAndReturnPages, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, threadSlotOfWorker, resolveRequestThreadCount, SHUTDOWN_TIMEOUT_MS, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, threadSlotGone, spawnNodeWorker, resolveWorkerEntry, noteThreadSlotLive, noteThreadSlotGone, bindSearchSignal, adoptSharedTurns, WORKER_YOUNG_GENERATION_MB, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_BLOCK_MAX_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_ATTACH_RETRY_MS, MIN_RANK_CONSTANT, MIN_LINEAR_ALPHA, MAX_WASM_PAGES, MAX_VECTOR_ORDINALS, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_LINEAR_ALPHA, MAX_DOC_ID_TABLE_BYTES, LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, FUSION_TIE_INSERTION_LIMIT, DISK_STORAGE_BLOCK_BYTES, DEFAULT_RANK_CONSTANT, DEFAULT_LINEAR_ALPHA, waitOutsideCountedWrite, nodeThreadId, STATISTICS_BROADCAST_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, EXPORT_CHUNK_BYTES, ANALYSIS_HOLD_APPLY_INTERVAL_MS, ANALYSIS_CHANGE_MEMBERS, MIN_HYBRID_DEPTH, MAX_SUGGEST_SCATTER_LIMIT, MAX_SUGGEST_LIMIT, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_SUGGEST_LIMIT, DEFAULT_MAX_PER_GROUP } from './chunk-
|
|
3
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, bitsetSet, discardStoredValues, createBitSet, installStoredFields, storedValueBytes, DEFAULT_PATTERN_WORK_CAP, bitsetIsEmpty, prepareStoredValues, fuzzyPrefixOf, boundedLevenshtein, checkedSize, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, buildStoredValues, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyTermMatches, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIterator, GrowableUint32, bitsetFromSet } from './chunk-
|
|
4
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_NEEDS_ROOM, NATIVE_OK } from './chunk-
|
|
1
|
+
import { reservedHeapBytes, reserveHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, admittedHeapBytes } from './chunk-J2BQTJJE.mjs';
|
|
2
|
+
import { MIN_DOCUMENTS_FOR_SEGMENTS, MAX_PARTITIONS, fnv1aUtf16, MAX_INDEX_CONFIGURATION_BYTES, changeIndexOutsideExternalWaits, BATCH_CHUNK_SIZE, turnsAreBudgeted, turnIsSpent, yieldTurn, createDocIdMap, detectRuntime, fnv1a, indexChangeOutsideExternalWaits, BM25_DISPLAYED_NAME_LENGTH, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, splitWorkerBudget, resolveWorkerCount, yieldBackgroundTurn, backgroundTurnIsSpent, FNV_OFFSET_BASIS, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, DEFAULT_REQUEST_TIMEOUT_MS, answerFromOneState, countWrite, HEAP_PRESSURE_WARNING_FRACTION, MIN_SWEEP_INTERVAL_MS, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, signalOfSearch, requirePinnedState, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, stopOfSignal, DEFAULT_PAGE_SIZE, CURSOR_BINDING_BUFFER_BYTES, fnv1aContinue, sharedStopOfSignal, requireSearchActive, fnv1aContinueTaggedNumbers, PREFIX_MAX_EXPANSIONS, FNV_PRIME, collectAndReturnPages, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, threadSlotOfWorker, resolveRequestThreadCount, SHUTDOWN_TIMEOUT_MS, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, threadSlotGone, spawnNodeWorker, resolveWorkerEntry, noteThreadSlotLive, noteThreadSlotGone, bindSearchSignal, adoptSharedTurns, WORKER_YOUNG_GENERATION_MB, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_BLOCK_MAX_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_ATTACH_RETRY_MS, MIN_RANK_CONSTANT, MIN_LINEAR_ALPHA, MAX_WASM_PAGES, MAX_VECTOR_ORDINALS, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_LINEAR_ALPHA, MAX_DOC_ID_TABLE_BYTES, LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, FUSION_TIE_INSERTION_LIMIT, DISK_STORAGE_BLOCK_BYTES, DEFAULT_RANK_CONSTANT, DEFAULT_LINEAR_ALPHA, waitOutsideCountedWrite, nodeThreadId, STATISTICS_BROADCAST_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, EXPORT_CHUNK_BYTES, DEFAULT_MAX_PARTITIONS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, ANALYSIS_CHANGE_MEMBERS, MIN_HYBRID_DEPTH, MAX_SUGGEST_SCATTER_LIMIT, MAX_SUGGEST_LIMIT, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_SUGGEST_LIMIT, DEFAULT_MAX_PER_GROUP } from './chunk-I7DTUOK2.mjs';
|
|
3
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, bitsetSet, discardStoredValues, createBitSet, installStoredFields, storedValueBytes, DEFAULT_PATTERN_WORK_CAP, bitsetIsEmpty, prepareStoredValues, fuzzyPrefixOf, boundedLevenshtein, checkedSize, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, buildStoredValues, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyTermMatches, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIterator, GrowableUint32, bitsetFromSet } from './chunk-SK3DPAI4.mjs';
|
|
4
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_NEEDS_ROOM, NATIVE_OK } from './chunk-WFQ6YQZA.mjs';
|
|
5
5
|
import { reactNativeHost } from './chunk-PVMEHBQO.mjs';
|
|
6
6
|
import { headerAt, MAP_TAGS, decode, encodeDocument, Decoder } from './chunk-YOTP2FPE.mjs';
|
|
7
|
-
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, schemaFieldsOf, getNestedValue, tokenizeOptions, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, flattenSchema, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, PROTOTYPE_POLLUTION_KEYS, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, MAX_NESTING_DEPTH, getFlatSchema, getOrCreateFieldNameIndex, isWordIndexedBase, isStringValuedBase, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-
|
|
7
|
+
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, schemaFieldsOf, getNestedValue, tokenizeOptions, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, flattenSchema, vectorDimensionOf, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, PROTOTYPE_POLLUTION_KEYS, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, MAX_NESTING_DEPTH, getFlatSchema, getOrCreateFieldNameIndex, isWordIndexedBase, isStringValuedBase, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-ZDDJVR7X.mjs';
|
|
8
8
|
import { withUnpairedSurrogatesReplaced, compareCodePoints, holdsUnpairedSurrogate, defaultSortMode, toComparableSortValue, exceedsCodePoints, SORT_VALUE_MAX_CODE_POINTS, sortModeOf, PLACEMENT_ROOM_REQUESTS, NEAREST_LINK_FALLBACK_HOSTS, MIN_M, MAX_M, MAX_LAYER_CAP, INITIAL_UPPER_SLOTS, INITIAL_LIST_CAPACITY, FARTHEST_REJECTED_LINKS_KEPT, EARLIER_RELEASE_MIN_M, DEFAULT_EF_SEARCH, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, CONNECT_RESTARTS_BEFORE_ONE_PASS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, truncateSortString, SORT_MODES, isSortMode, codePointCount } from './chunk-HRDMIQ63.mjs';
|
|
9
9
|
import { MAX_DOCUMENT_ENCODING_DEPTH, MAX_FILE_BYTES, VALID_HNSW_METRICS, NATIVE_CRC32_MIN_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, DOCUMENT_REPLY_ENVELOPE_DEPTH, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER } from './chunk-VUJCSQG5.mjs';
|
|
10
10
|
import { NarsilError, ErrorCodes, createNarsilError, isNarsilError } from './chunk-TQ26BKBH.mjs';
|
|
@@ -61,26 +61,11 @@ function getStopWords(name) {
|
|
|
61
61
|
}
|
|
62
62
|
return stopWords2;
|
|
63
63
|
}
|
|
64
|
-
function holdsPositionsOutOfOrder(entries) {
|
|
65
|
-
for (let index = 1; index < entries.length; index++) {
|
|
66
|
-
if (entries[index].position < entries[index - 1].position) return true;
|
|
67
|
-
}
|
|
68
|
-
return false;
|
|
69
|
-
}
|
|
70
|
-
function inPositionOrder(tokenizer) {
|
|
71
|
-
return {
|
|
72
|
-
tokenize(text) {
|
|
73
|
-
const entries = tokenizer.tokenize(text);
|
|
74
|
-
if (!holdsPositionsOutOfOrder(entries)) return entries;
|
|
75
|
-
return [...entries].sort((first, second) => first.position - second.position);
|
|
76
|
-
}
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
64
|
function resolveIndexAnalysis(config) {
|
|
80
65
|
const tokenizer = typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer;
|
|
81
66
|
return {
|
|
82
67
|
stopWords: typeof config.stopWords === "string" ? getStopWords(config.stopWords) : config.stopWords,
|
|
83
|
-
customTokenizer: tokenizer
|
|
68
|
+
customTokenizer: tokenizer
|
|
84
69
|
};
|
|
85
70
|
}
|
|
86
71
|
|
|
@@ -2977,6 +2962,21 @@ function findEntry(table2, text, stride = 1, tokens, token = 0) {
|
|
|
2977
2962
|
return -1;
|
|
2978
2963
|
}
|
|
2979
2964
|
|
|
2965
|
+
// src/analysis/custom-token-code.ts
|
|
2966
|
+
var customTokenCode = codeLoader(
|
|
2967
|
+
() => import('./custom-token-results-3OZEMRE7.mjs').then((code7) => {
|
|
2968
|
+
code7.bindCustomTokenHost(optionalCodeHost());
|
|
2969
|
+
return code7;
|
|
2970
|
+
}),
|
|
2971
|
+
"custom tokenizer"
|
|
2972
|
+
);
|
|
2973
|
+
function loadCustomTokenCodeFor(tokenizers2) {
|
|
2974
|
+
if (customTokenCode.loaded() !== null) return null;
|
|
2975
|
+
for (const tokenizer of tokenizers2) if (tokenizer !== void 0 && tokenizer !== null) return customTokenCode.load();
|
|
2976
|
+
return null;
|
|
2977
|
+
}
|
|
2978
|
+
var customTokensOf = (...parameters) => customTokenCode.require().customTokensOf(...parameters);
|
|
2979
|
+
|
|
2980
2980
|
// src/core/tokenizer/constants.ts
|
|
2981
2981
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
2982
2982
|
var TOKEN_CACHE_SIZE_CEILING = 2e6;
|
|
@@ -3353,18 +3353,6 @@ function stripPossessive(token) {
|
|
|
3353
3353
|
}
|
|
3354
3354
|
return token;
|
|
3355
3355
|
}
|
|
3356
|
-
function customTokensOf(tokenizer, text) {
|
|
3357
|
-
const entries = tokenizer.tokenize(text);
|
|
3358
|
-
let repaired = null;
|
|
3359
|
-
for (let at = 0; at < entries.length; at++) {
|
|
3360
|
-
const entry = entries[at];
|
|
3361
|
-
const token = withUnpairedSurrogatesReplaced(entry.token);
|
|
3362
|
-
if (token === entry.token) continue;
|
|
3363
|
-
repaired ??= entries.slice();
|
|
3364
|
-
repaired[at] = { token, position: entry.position };
|
|
3365
|
-
}
|
|
3366
|
-
return repaired ?? entries;
|
|
3367
|
-
}
|
|
3368
3356
|
function visitTokens(text, language, options, visit, packed) {
|
|
3369
3357
|
const {
|
|
3370
3358
|
stem: stem2 = true,
|
|
@@ -3375,7 +3363,7 @@ function visitTokens(text, language, options, visit, packed) {
|
|
|
3375
3363
|
stopWordOverride
|
|
3376
3364
|
} = options ?? {};
|
|
3377
3365
|
if (customTokenizer) {
|
|
3378
|
-
for (const entry of customTokensOf(customTokenizer, text)) {
|
|
3366
|
+
for (const entry of customTokensOf(customTokenizer, text, options?.textSource ?? "query")) {
|
|
3379
3367
|
visit(entry.token, entry.position, void 0, entry.token);
|
|
3380
3368
|
}
|
|
3381
3369
|
return;
|
|
@@ -3422,7 +3410,7 @@ function visitSurfaceSpellings(text, language, visit) {
|
|
|
3422
3410
|
}
|
|
3423
3411
|
function tokenize(text, language, options) {
|
|
3424
3412
|
if (options?.customTokenizer) {
|
|
3425
|
-
const customResult = customTokensOf(options.customTokenizer, text);
|
|
3413
|
+
const customResult = customTokensOf(options.customTokenizer, text, options.textSource ?? "query");
|
|
3426
3414
|
return { tokens: customResult, originalTokens: customResult.map((entry) => entry.token) };
|
|
3427
3415
|
}
|
|
3428
3416
|
const tokens = [];
|
|
@@ -4546,7 +4534,7 @@ function requireLoadedPhraseCode() {
|
|
|
4546
4534
|
}
|
|
4547
4535
|
function loadPhraseCode() {
|
|
4548
4536
|
if (loading === null) {
|
|
4549
|
-
loading = import('./phrase-module-
|
|
4537
|
+
loading = import('./phrase-module-UC4YGEBB.mjs').then(
|
|
4550
4538
|
(code7) => {
|
|
4551
4539
|
code7.bindPhraseHost(optionalCodeHost());
|
|
4552
4540
|
loaded = code7;
|
|
@@ -5399,10 +5387,14 @@ function validatePartitionConfig(partitions) {
|
|
|
5399
5387
|
maxDocsPerPartition
|
|
5400
5388
|
});
|
|
5401
5389
|
}
|
|
5402
|
-
if (maxPartitions !== void 0 && (!Number.isInteger(maxPartitions) || maxPartitions < 1)) {
|
|
5403
|
-
throw new NarsilError(
|
|
5404
|
-
|
|
5405
|
-
|
|
5390
|
+
if (maxPartitions !== void 0 && (!Number.isInteger(maxPartitions) || maxPartitions < 1 || maxPartitions > MAX_PARTITIONS)) {
|
|
5391
|
+
throw new NarsilError(
|
|
5392
|
+
ErrorCodes.CONFIG_INVALID,
|
|
5393
|
+
`partitions.maxPartitions must be an integer from 1 to ${MAX_PARTITIONS}`,
|
|
5394
|
+
{
|
|
5395
|
+
maxPartitions
|
|
5396
|
+
}
|
|
5397
|
+
);
|
|
5406
5398
|
}
|
|
5407
5399
|
if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
|
|
5408
5400
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.watermark must be above 0 and at most 1", {
|
|
@@ -8685,7 +8677,7 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
8685
8677
|
|
|
8686
8678
|
// src/core/segment-codec-code.ts
|
|
8687
8679
|
var segmentCodec = codeLoader(async () => {
|
|
8688
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-
|
|
8680
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-WMUZ5P4F.mjs')]);
|
|
8689
8681
|
code7.bindSegmentCodecHost(optionalCodeHost(), msgpack);
|
|
8690
8682
|
return code7;
|
|
8691
8683
|
}, "segment codec");
|
|
@@ -8696,7 +8688,7 @@ var segmentMerge = codeLoader(async () => {
|
|
|
8696
8688
|
const [msgpack, codec, code7] = await Promise.all([
|
|
8697
8689
|
loadMessagePack(),
|
|
8698
8690
|
loadSegmentCodec(),
|
|
8699
|
-
import('./segment-merge-leaf-
|
|
8691
|
+
import('./segment-merge-leaf-Q2OEJIQ3.mjs')
|
|
8700
8692
|
]);
|
|
8701
8693
|
code7.bindSegmentMergeHost(optionalCodeHost(), msgpack, codec);
|
|
8702
8694
|
return code7;
|
|
@@ -8705,7 +8697,7 @@ var loadSegmentMerge = segmentMerge.load;
|
|
|
8705
8697
|
|
|
8706
8698
|
// src/query-suggestions/store-code.ts
|
|
8707
8699
|
var suggestionStore = codeLoader(async () => {
|
|
8708
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-
|
|
8700
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-ZMDXYK4E.mjs')]);
|
|
8709
8701
|
code7.bindSuggestionStoreHost(optionalCodeHost(), msgpack);
|
|
8710
8702
|
return code7;
|
|
8711
8703
|
}, "query suggestions store");
|
|
@@ -8764,7 +8756,7 @@ function markSimdUnavailable() {
|
|
|
8764
8756
|
|
|
8765
8757
|
// src/workers/vector-core-code.ts
|
|
8766
8758
|
var vectorCore = codeLoader(
|
|
8767
|
-
() => import('./core-leaf-
|
|
8759
|
+
() => import('./core-leaf-UDUMVC3M.mjs').then((code7) => {
|
|
8768
8760
|
code7.bindVectorCoreHost(optionalCodeHost());
|
|
8769
8761
|
if (!code7.simdModuleLoaded()) markSimdUnavailable();
|
|
8770
8762
|
return code7;
|
|
@@ -8786,7 +8778,7 @@ var storageCode = codeLoader(async () => {
|
|
|
8786
8778
|
loadSegmentMerge(),
|
|
8787
8779
|
loadSuggestionStore(),
|
|
8788
8780
|
loadVectorCore(),
|
|
8789
|
-
import('./storage-leaf-
|
|
8781
|
+
import('./storage-leaf-OXYV6O6P.mjs')
|
|
8790
8782
|
]);
|
|
8791
8783
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
8792
8784
|
return code7;
|
|
@@ -19950,14 +19942,13 @@ function hasLanguage(name) {
|
|
|
19950
19942
|
|
|
19951
19943
|
// src/narsil/restore-queue-code.ts
|
|
19952
19944
|
var restoreQueue = codeLoader(async () => {
|
|
19953
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
19945
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-SZLWIL3B.mjs')]);
|
|
19954
19946
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
19955
19947
|
return code7;
|
|
19956
19948
|
}, "restore");
|
|
19957
19949
|
var loadRestoreQueue = restoreQueue.load;
|
|
19958
19950
|
|
|
19959
19951
|
// src/schema/embedding-validator.ts
|
|
19960
|
-
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
19961
19952
|
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
19962
19953
|
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
19963
19954
|
if (fieldEntries.length === 0) {
|
|
@@ -19996,15 +19987,14 @@ function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
|
19996
19987
|
{ field: targetField }
|
|
19997
19988
|
);
|
|
19998
19989
|
}
|
|
19999
|
-
const
|
|
20000
|
-
if (
|
|
19990
|
+
const schemaDimension = vectorDimensionOf(targetType);
|
|
19991
|
+
if (schemaDimension === null) {
|
|
20001
19992
|
throw new NarsilError(
|
|
20002
19993
|
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
20003
19994
|
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
20004
19995
|
{ field: targetField, type: targetType }
|
|
20005
19996
|
);
|
|
20006
19997
|
}
|
|
20007
|
-
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
20008
19998
|
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
20009
19999
|
throw new NarsilError(
|
|
20010
20000
|
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
@@ -20174,7 +20164,7 @@ function storedBM25Params(params) {
|
|
|
20174
20164
|
|
|
20175
20165
|
// src/workers/vector-code.ts
|
|
20176
20166
|
var vectorCode = codeLoader(async () => {
|
|
20177
|
-
const [vectorCore2, code7] = await Promise.all([loadVectorCore(), import('./module-
|
|
20167
|
+
const [vectorCore2, code7] = await Promise.all([loadVectorCore(), import('./module-7BJ3VOWQ.mjs')]);
|
|
20178
20168
|
code7.bindVectorHost(optionalCodeHost(), vectorCore2);
|
|
20179
20169
|
return code7;
|
|
20180
20170
|
}, "vector");
|
|
@@ -20184,9 +20174,14 @@ var loadVectorCode = vectorCode.load;
|
|
|
20184
20174
|
var loadVectorCodeFor = vectorCode.loadFor;
|
|
20185
20175
|
|
|
20186
20176
|
// src/workers/schema-code.ts
|
|
20187
|
-
function loadSchemaCodeFor(schema) {
|
|
20177
|
+
function loadSchemaCodeFor(schema, ...tokenizers2) {
|
|
20188
20178
|
const loading4 = [];
|
|
20189
|
-
for (const code7 of [
|
|
20179
|
+
for (const code7 of [
|
|
20180
|
+
loadVectorCodeFor(schema),
|
|
20181
|
+
loadGeoCodeFor(schema),
|
|
20182
|
+
loadOptionalReadsFor(schema),
|
|
20183
|
+
loadCustomTokenCodeFor(tokenizers2)
|
|
20184
|
+
]) {
|
|
20190
20185
|
if (code7 !== null) loading4.push(code7);
|
|
20191
20186
|
}
|
|
20192
20187
|
if (loading4.length === 0) return null;
|
|
@@ -20226,7 +20221,7 @@ function configurationWithinSizeLimit(config, language) {
|
|
|
20226
20221
|
function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms, analysisGeneration = 0) {
|
|
20227
20222
|
const bm25 = config.bm25;
|
|
20228
20223
|
return {
|
|
20229
|
-
schema: config.schema,
|
|
20224
|
+
schema: { ...flattenSchema(config.schema) },
|
|
20230
20225
|
language: language.name,
|
|
20231
20226
|
...analysisRevision !== null ? { analysisRevision } : {},
|
|
20232
20227
|
...arrayPositionGap !== null ? { arrayPositionGap } : {},
|
|
@@ -20308,7 +20303,7 @@ async function announceIndexDropped(core2, name, config) {
|
|
|
20308
20303
|
}
|
|
20309
20304
|
async function createEngineIndex(core2, config, name, requestedConfig, creation = {}) {
|
|
20310
20305
|
const { indexUuid, adapterBinding = "required", appliesCreationLimits = false } = creation;
|
|
20311
|
-
const loadingSchemaCode = loadSchemaCodeFor(requestedConfig.schema);
|
|
20306
|
+
const loadingSchemaCode = loadSchemaCodeFor(requestedConfig.schema, requestedConfig.tokenizer);
|
|
20312
20307
|
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
20313
20308
|
core2.guardShutdown();
|
|
20314
20309
|
validateIndexName(name);
|
|
@@ -20537,7 +20532,7 @@ function storedArrayPositionGap(value) {
|
|
|
20537
20532
|
|
|
20538
20533
|
// src/engine/embed-code.ts
|
|
20539
20534
|
var embedCode = codeLoader(
|
|
20540
|
-
() => import('./embed-
|
|
20535
|
+
() => import('./embed-U7CIKEY4.mjs').then((code7) => {
|
|
20541
20536
|
code7.bindEmbedHost(optionalCodeHost());
|
|
20542
20537
|
return code7;
|
|
20543
20538
|
}),
|
|
@@ -20547,7 +20542,7 @@ var loadEmbedCode = embedCode.load;
|
|
|
20547
20542
|
|
|
20548
20543
|
// src/engine/optional-writes-code.ts
|
|
20549
20544
|
var optionalWrites = codeLoader(async () => {
|
|
20550
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
20545
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-TXTJ6HUZ.mjs')]);
|
|
20551
20546
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
20552
20547
|
return code7;
|
|
20553
20548
|
}, "write");
|
|
@@ -20557,15 +20552,20 @@ var loadOptionalWrites = optionalWrites.load;
|
|
|
20557
20552
|
function extractVectorFromDoc(document, fieldPath) {
|
|
20558
20553
|
const value = getNestedValue(document, fieldPath);
|
|
20559
20554
|
if (value === void 0 || value === null) return null;
|
|
20560
|
-
if (value instanceof Float32Array) return value;
|
|
20555
|
+
if (value instanceof Float32Array) return holdsOnlyFiniteFloats(value) ? value : null;
|
|
20561
20556
|
if (!Array.isArray(value)) return null;
|
|
20562
|
-
const arr = new Float32Array(value.length);
|
|
20563
20557
|
for (let i = 0; i < value.length; i++) {
|
|
20564
|
-
|
|
20565
|
-
|
|
20566
|
-
|
|
20558
|
+
if (typeof value[i] !== "number") return null;
|
|
20559
|
+
}
|
|
20560
|
+
const vector = new Float32Array(value);
|
|
20561
|
+
return holdsOnlyFiniteFloats(vector) ? vector : null;
|
|
20562
|
+
}
|
|
20563
|
+
function holdsOnlyFiniteFloats(vector) {
|
|
20564
|
+
const length = vector.length;
|
|
20565
|
+
for (let i = 0; i < length; i++) {
|
|
20566
|
+
if (vector[i] - vector[i] !== 0) return false;
|
|
20567
20567
|
}
|
|
20568
|
-
return
|
|
20568
|
+
return true;
|
|
20569
20569
|
}
|
|
20570
20570
|
function deleteNestedValue(obj, path) {
|
|
20571
20571
|
if (!path.includes(".")) {
|
|
@@ -21091,6 +21091,20 @@ function bindOptionalWritesHost(_host, _segmentCodec) {
|
|
|
21091
21091
|
}
|
|
21092
21092
|
|
|
21093
21093
|
// src/engine/mutations/durable-rollback.ts
|
|
21094
|
+
function holdVectorsForRollback(indexes, docId) {
|
|
21095
|
+
const releases = [];
|
|
21096
|
+
for (const index of indexes.values()) releases.push(index.holdForRollback(docId));
|
|
21097
|
+
return () => {
|
|
21098
|
+
for (const release of releases) release();
|
|
21099
|
+
};
|
|
21100
|
+
}
|
|
21101
|
+
function restoreDocumentVectors(docId, vectors, indexes, partitionId) {
|
|
21102
|
+
for (const [fieldPath, index] of indexes) {
|
|
21103
|
+
const vector = vectors.get(fieldPath);
|
|
21104
|
+
if (vector !== void 0) index.restore(docId, vector, partitionId);
|
|
21105
|
+
else if (index.has(docId)) index.remove(docId);
|
|
21106
|
+
}
|
|
21107
|
+
}
|
|
21094
21108
|
function rollbackFailed(docId, originalError, rollbackError) {
|
|
21095
21109
|
throw new NarsilError(
|
|
21096
21110
|
ErrorCodes.REPLICATION_ROLLBACK_FAILED,
|
|
@@ -21139,7 +21153,7 @@ async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
21139
21153
|
manager.insert(docId, previous.partitionDoc, {
|
|
21140
21154
|
preparedText: { after: preparedText?.before }
|
|
21141
21155
|
});
|
|
21142
|
-
|
|
21156
|
+
restoreDocumentVectors(
|
|
21143
21157
|
docId,
|
|
21144
21158
|
previous.extractedVectors,
|
|
21145
21159
|
manager.getVectorIndexes(),
|
|
@@ -21173,8 +21187,12 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
21173
21187
|
skipClone: rollbackRef !== void 0,
|
|
21174
21188
|
preparedText: { before: preparedText?.after, after: preparedText?.before }
|
|
21175
21189
|
});
|
|
21176
|
-
|
|
21177
|
-
|
|
21190
|
+
restoreDocumentVectors(
|
|
21191
|
+
docId,
|
|
21192
|
+
previous.extractedVectors,
|
|
21193
|
+
manager.getVectorIndexes(),
|
|
21194
|
+
manager.partitionIdOf(docId)
|
|
21195
|
+
);
|
|
21178
21196
|
ctx.orchestrator.publishToWorkers({
|
|
21179
21197
|
type: "update",
|
|
21180
21198
|
indexName,
|
|
@@ -21873,6 +21891,7 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
21873
21891
|
return queued;
|
|
21874
21892
|
}
|
|
21875
21893
|
prepared.restoreDoc = ctx.durability ? manager.get(docId) : void 0;
|
|
21894
|
+
if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
|
|
21876
21895
|
manager.remove(docId, void 0, {
|
|
21877
21896
|
onRemovedSpellings: (spellings) => {
|
|
21878
21897
|
prepared.removedSpellings = spellings;
|
|
@@ -21913,6 +21932,8 @@ async function removeDocument3(ctx, indexName, docId, options) {
|
|
|
21913
21932
|
} catch (err) {
|
|
21914
21933
|
await rollbackRemovedDocument(ctx, indexName, docId, prepared.restoreDoc, err, prepared.preparedText);
|
|
21915
21934
|
throw err;
|
|
21935
|
+
} finally {
|
|
21936
|
+
prepared.releaseVectorHolds?.();
|
|
21916
21937
|
}
|
|
21917
21938
|
} else {
|
|
21918
21939
|
await apply();
|
|
@@ -21941,29 +21962,35 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
21941
21962
|
}
|
|
21942
21963
|
}
|
|
21943
21964
|
const applies = prepared.map((removal) => applyOfRemoval(ctx, indexName, removal));
|
|
21944
|
-
|
|
21945
|
-
|
|
21946
|
-
|
|
21947
|
-
|
|
21948
|
-
|
|
21949
|
-
|
|
21950
|
-
|
|
21951
|
-
|
|
21952
|
-
|
|
21953
|
-
|
|
21954
|
-
|
|
21955
|
-
|
|
21956
|
-
|
|
21957
|
-
|
|
21958
|
-
|
|
21959
|
-
|
|
21960
|
-
|
|
21965
|
+
try {
|
|
21966
|
+
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
21967
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
21968
|
+
const removal = prepared[i];
|
|
21969
|
+
const failure = failures[i];
|
|
21970
|
+
if (failure !== null) {
|
|
21971
|
+
let error = failure.error;
|
|
21972
|
+
try {
|
|
21973
|
+
await rollbackRemovedDocument(
|
|
21974
|
+
ctx,
|
|
21975
|
+
indexName,
|
|
21976
|
+
removal.docId,
|
|
21977
|
+
removal.restoreDoc,
|
|
21978
|
+
error,
|
|
21979
|
+
removal.preparedText
|
|
21980
|
+
);
|
|
21981
|
+
} catch (rollbackError) {
|
|
21982
|
+
error = rollbackError;
|
|
21983
|
+
}
|
|
21984
|
+
removal.releaseVectorHolds?.();
|
|
21985
|
+
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
21986
|
+
continue;
|
|
21961
21987
|
}
|
|
21962
|
-
|
|
21963
|
-
|
|
21988
|
+
removal.releaseVectorHolds?.();
|
|
21989
|
+
if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
|
|
21990
|
+
succeeded.push(removal.docId);
|
|
21964
21991
|
}
|
|
21965
|
-
|
|
21966
|
-
|
|
21992
|
+
} finally {
|
|
21993
|
+
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
21967
21994
|
}
|
|
21968
21995
|
if (chunkEnd < docIds.length) {
|
|
21969
21996
|
await new Promise((r) => setTimeout(r, 0));
|
|
@@ -21978,9 +22005,6 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
21978
22005
|
}
|
|
21979
22006
|
|
|
21980
22007
|
// src/engine/mutations/update.ts
|
|
21981
|
-
function extractVectorFromDocForUpdate(document, fieldPath) {
|
|
21982
|
-
return extractVectorFromDoc(document, fieldPath);
|
|
21983
|
-
}
|
|
21984
22008
|
function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
21985
22009
|
if (extractedVectors.size === 0) {
|
|
21986
22010
|
return { partitionDoc: document };
|
|
@@ -22004,8 +22028,8 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
22004
22028
|
for (const fieldPath of extractedVectors.keys()) delete source[fieldPath];
|
|
22005
22029
|
}
|
|
22006
22030
|
const partitionDoc = structuredClone(source);
|
|
22007
|
-
for (const fieldPath of extractedVectors
|
|
22008
|
-
deleteNestedValue(partitionDoc, fieldPath);
|
|
22031
|
+
for (const [fieldPath, vector] of extractedVectors) {
|
|
22032
|
+
if (vector !== null || getNestedValue(partitionDoc, fieldPath) == null) deleteNestedValue(partitionDoc, fieldPath);
|
|
22009
22033
|
}
|
|
22010
22034
|
return { partitionDoc };
|
|
22011
22035
|
}
|
|
@@ -22070,7 +22094,7 @@ async function prepareUpdate(ctx, indexName, entry, docId, document) {
|
|
|
22070
22094
|
if (vecIndexes.size > 0) {
|
|
22071
22095
|
const dimensionCheckVectors = /* @__PURE__ */ new Map();
|
|
22072
22096
|
for (const fieldPath of entry.vectorFieldPaths) {
|
|
22073
|
-
const newVec =
|
|
22097
|
+
const newVec = extractVectorFromDoc(document, fieldPath);
|
|
22074
22098
|
extractedVectors.set(fieldPath, newVec);
|
|
22075
22099
|
if (newVec) {
|
|
22076
22100
|
dimensionCheckVectors.set(fieldPath, newVec);
|
|
@@ -22108,9 +22132,15 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
22108
22132
|
prepared.buffered = true;
|
|
22109
22133
|
return queued;
|
|
22110
22134
|
}
|
|
22135
|
+
if (manager.has(docId)) {
|
|
22136
|
+
for (const [fieldPath, vector] of extractedVectors) {
|
|
22137
|
+
if (vector !== null) manager.getVectorIndexes().get(fieldPath)?.requireRoomFor(docId, vector);
|
|
22138
|
+
}
|
|
22139
|
+
}
|
|
22111
22140
|
prepared.oldDocument = extractedVectors.size > 0 || ctx.durability !== null || ctx.pluginRegistry.hasHooks("afterUpdate") ? manager.get(docId) : void 0;
|
|
22112
22141
|
prepared.rollbackDoc = prepared.oldDocument;
|
|
22113
22142
|
prepared.rollbackRef = ctx.durability === null ? void 0 : manager.getRef(docId);
|
|
22143
|
+
if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
|
|
22114
22144
|
const stored = manager.update(docId, partitionDoc, {
|
|
22115
22145
|
onRemovedSpellings: (spellings) => {
|
|
22116
22146
|
prepared.removedSpellings = spellings;
|
|
@@ -22172,6 +22202,8 @@ async function updateDocument(ctx, indexName, docId, document, options) {
|
|
|
22172
22202
|
prepared.rollbackRef
|
|
22173
22203
|
);
|
|
22174
22204
|
throw err;
|
|
22205
|
+
} finally {
|
|
22206
|
+
prepared.releaseVectorHolds?.();
|
|
22175
22207
|
}
|
|
22176
22208
|
} else {
|
|
22177
22209
|
await apply();
|
|
@@ -22220,25 +22252,30 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
22220
22252
|
}
|
|
22221
22253
|
}
|
|
22222
22254
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
22223
|
-
|
|
22224
|
-
|
|
22225
|
-
|
|
22226
|
-
|
|
22227
|
-
|
|
22228
|
-
|
|
22229
|
-
|
|
22230
|
-
|
|
22231
|
-
|
|
22232
|
-
|
|
22233
|
-
|
|
22234
|
-
|
|
22235
|
-
|
|
22236
|
-
|
|
22237
|
-
|
|
22238
|
-
|
|
22239
|
-
|
|
22240
|
-
|
|
22255
|
+
let failures;
|
|
22256
|
+
try {
|
|
22257
|
+
failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
22258
|
+
for (let i = prepared.length - 1; i >= 0; i--) {
|
|
22259
|
+
const failure = failures[i];
|
|
22260
|
+
if (failure === null) continue;
|
|
22261
|
+
const update = prepared[i];
|
|
22262
|
+
try {
|
|
22263
|
+
await rollbackUpdatedDocument(
|
|
22264
|
+
ctx,
|
|
22265
|
+
indexName,
|
|
22266
|
+
update.docId,
|
|
22267
|
+
update.rollbackDoc,
|
|
22268
|
+
failure.error,
|
|
22269
|
+
update.preparedText,
|
|
22270
|
+
update.appliedRef,
|
|
22271
|
+
update.rollbackRef
|
|
22272
|
+
);
|
|
22273
|
+
} catch (error) {
|
|
22274
|
+
failure.error = error;
|
|
22275
|
+
}
|
|
22241
22276
|
}
|
|
22277
|
+
} finally {
|
|
22278
|
+
for (const update of prepared) update.releaseVectorHolds?.();
|
|
22242
22279
|
}
|
|
22243
22280
|
for (let i = 0; i < prepared.length; i++) {
|
|
22244
22281
|
const update = prepared[i];
|
|
@@ -22402,7 +22439,7 @@ var suggestionsCode = codeLoader(async () => {
|
|
|
22402
22439
|
const [msgpack, suggestionStore2, code7] = await Promise.all([
|
|
22403
22440
|
loadMessagePack(),
|
|
22404
22441
|
loadSuggestionStore(),
|
|
22405
|
-
import('./suggestions-leaf-
|
|
22442
|
+
import('./suggestions-leaf-FIMGQZ2T.mjs')
|
|
22406
22443
|
]);
|
|
22407
22444
|
code7.bindSuggestionsHost(optionalCodeHost(), msgpack, suggestionStore2);
|
|
22408
22445
|
return code7;
|
|
@@ -23534,6 +23571,12 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
23534
23571
|
partitions.push(partition);
|
|
23535
23572
|
return partition;
|
|
23536
23573
|
},
|
|
23574
|
+
growToSavedPartitionCount(partitionCount) {
|
|
23575
|
+
if (partitions.length >= partitionCount) return;
|
|
23576
|
+
writeLogs.partitionsReplaced();
|
|
23577
|
+
while (partitions.length < partitionCount)
|
|
23578
|
+
partitions.push(createPartitionIndex(partitions.length, trackPositions));
|
|
23579
|
+
},
|
|
23537
23580
|
removePartition(partitionId) {
|
|
23538
23581
|
validatePartitionId(partitionId);
|
|
23539
23582
|
writeLogs.partitionsReplaced();
|
|
@@ -23741,6 +23784,7 @@ function storageHostPart() {
|
|
|
23741
23784
|
ARRAY_POSITION_GAP,
|
|
23742
23785
|
DEFAULT_BM25_B,
|
|
23743
23786
|
DEFAULT_BM25_K1,
|
|
23787
|
+
DEFAULT_MAX_PARTITIONS,
|
|
23744
23788
|
EXPORT_CHUNK_BYTES,
|
|
23745
23789
|
FOREIGN_STATS_TTL_MS,
|
|
23746
23790
|
IMPORT_BATCH_DOCUMENTS,
|
|
@@ -23749,6 +23793,7 @@ function storageHostPart() {
|
|
|
23749
23793
|
MAX_EXPORT_LINE_BYTES,
|
|
23750
23794
|
MAX_IMPORT_REFUSALS_REPORTED,
|
|
23751
23795
|
MAX_INDEX_NAME_LENGTH,
|
|
23796
|
+
MAX_PARTITIONS,
|
|
23752
23797
|
STATISTICS_BROADCAST_INTERVAL_MS,
|
|
23753
23798
|
TermLookup,
|
|
23754
23799
|
VALID_HNSW_METRICS,
|
|
@@ -23813,7 +23858,8 @@ function textHostPart() {
|
|
|
23813
23858
|
SegmentRefusal,
|
|
23814
23859
|
termCount,
|
|
23815
23860
|
termText,
|
|
23816
|
-
tokenize
|
|
23861
|
+
tokenize,
|
|
23862
|
+
withUnpairedSurrogatesReplaced
|
|
23817
23863
|
};
|
|
23818
23864
|
}
|
|
23819
23865
|
|
|
@@ -24007,7 +24053,7 @@ var workerCopies = codeLoader(async () => {
|
|
|
24007
24053
|
const [segmentCodec2, segmentMerge2, code7] = await Promise.all([
|
|
24008
24054
|
loadSegmentCodec(),
|
|
24009
24055
|
loadSegmentMerge(),
|
|
24010
|
-
import('./worker-copies-
|
|
24056
|
+
import('./worker-copies-72TGFYQB.mjs')
|
|
24011
24057
|
]);
|
|
24012
24058
|
code7.bindWorkerCopiesHost(optionalCodeHost(), segmentCodec2, segmentMerge2);
|
|
24013
24059
|
return code7;
|
|
@@ -24019,9 +24065,7 @@ function isSegmentAction(action) {
|
|
|
24019
24065
|
return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail" || action.type === "adoptDiskSegments";
|
|
24020
24066
|
}
|
|
24021
24067
|
function growPartitionsTo(manager, partitionId) {
|
|
24022
|
-
|
|
24023
|
-
manager.addPartition();
|
|
24024
|
-
}
|
|
24068
|
+
manager.growToSavedPartitionCount(partitionId + 1);
|
|
24025
24069
|
}
|
|
24026
24070
|
var runSegmentAction = async (...args) => {
|
|
24027
24071
|
const code7 = await loadWorkerCopies();
|
|
@@ -24239,7 +24283,7 @@ var loaded2 = null;
|
|
|
24239
24283
|
var loading3 = null;
|
|
24240
24284
|
function importOptionalReads() {
|
|
24241
24285
|
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
24242
|
-
loading3 ??= import('./optional-reads-
|
|
24286
|
+
loading3 ??= import('./optional-reads-74HYI4DO.mjs').then(
|
|
24243
24287
|
(code7) => {
|
|
24244
24288
|
code7.bindOptionalReadsHost(optionalCodeHost());
|
|
24245
24289
|
loaded2 = code7;
|
|
@@ -24806,7 +24850,7 @@ function vectorBytesOf(document, fields) {
|
|
|
24806
24850
|
function customTermBytes(text, tokenizer, partial) {
|
|
24807
24851
|
let bytes = 0;
|
|
24808
24852
|
const termBytes = DISTINCT_TERM_HEAP_BYTES + (partial ? PARTIAL_SPELLING_HEAP_BYTES : 0);
|
|
24809
|
-
for (const { token } of tokenizer
|
|
24853
|
+
for (const { token } of customTokensOf(tokenizer, text, "document")) {
|
|
24810
24854
|
bytes += termBytes + token.length * TERM_UNIT_HEAP_BYTES;
|
|
24811
24855
|
}
|
|
24812
24856
|
return bytes;
|
|
@@ -24823,7 +24867,7 @@ function countedTermBytes(text, analysis, partial) {
|
|
|
24823
24867
|
visitTokens(
|
|
24824
24868
|
text,
|
|
24825
24869
|
analysis.language,
|
|
24826
|
-
{ stem: false, removeStopWords: false, customTokenizer: analysis.customTokenizer },
|
|
24870
|
+
{ stem: false, removeStopWords: false, customTokenizer: analysis.customTokenizer, textSource: "document" },
|
|
24827
24871
|
(token) => {
|
|
24828
24872
|
distinct.add(token);
|
|
24829
24873
|
units += token.length;
|
|
@@ -25441,6 +25485,20 @@ function queryContextOfEntry(entry) {
|
|
|
25441
25485
|
analysisStale: entry.analysisStale
|
|
25442
25486
|
};
|
|
25443
25487
|
}
|
|
25488
|
+
function vectorIndexesFor(indexName, config, copies, vectorStorage, onVectorGraphRepaired) {
|
|
25489
|
+
const vectorIndexes = /* @__PURE__ */ new Map();
|
|
25490
|
+
const vectorFields = extractVectorFieldsFromSchema(config.schema);
|
|
25491
|
+
if (vectorFields.size === 0) return vectorIndexes;
|
|
25492
|
+
const { createVectorIndex } = requireLoadedVectorCode();
|
|
25493
|
+
const promotion = config.vectorPromotion;
|
|
25494
|
+
const storage = promotion?.storage ?? vectorStorage;
|
|
25495
|
+
const repaired = () => onVectorGraphRepaired?.(indexName);
|
|
25496
|
+
for (const [fieldPath, dim] of vectorFields) {
|
|
25497
|
+
const index = createVectorIndex(fieldPath, dim, promotion, copies, indexName, storage, repaired);
|
|
25498
|
+
vectorIndexes.set(fieldPath, index);
|
|
25499
|
+
}
|
|
25500
|
+
return vectorIndexes;
|
|
25501
|
+
}
|
|
25444
25502
|
function disposeEntry(entry) {
|
|
25445
25503
|
closeCopyPublicationOrder(entry.copyPublications);
|
|
25446
25504
|
entry.manager.afterSearchesRelease(() => {
|
|
@@ -25493,40 +25551,34 @@ function createDirectExecutor(options) {
|
|
|
25493
25551
|
}
|
|
25494
25552
|
return entry;
|
|
25495
25553
|
}
|
|
25496
|
-
function
|
|
25497
|
-
|
|
25498
|
-
const vectorFields = extractVectorFieldsFromSchema(config.schema);
|
|
25499
|
-
if (vectorFields.size === 0) return vectorIndexes;
|
|
25500
|
-
const { createVectorIndex } = requireLoadedVectorCode();
|
|
25501
|
-
const promotion = config.vectorPromotion;
|
|
25502
|
-
const storage = promotion?.storage ?? vectorStorage;
|
|
25503
|
-
const repaired = () => options?.onVectorGraphRepaired?.(indexName);
|
|
25504
|
-
for (const [fieldPath, dim] of vectorFields) {
|
|
25505
|
-
const index = createVectorIndex(fieldPath, dim, promotion, copies, indexName, storage, repaired);
|
|
25506
|
-
vectorIndexes.set(fieldPath, index);
|
|
25507
|
-
}
|
|
25508
|
-
return vectorIndexes;
|
|
25509
|
-
}
|
|
25510
|
-
function createIndex(indexName, config, language, analysisStale = false, synonyms = null) {
|
|
25511
|
-
addIndex(indexName, config, language, analysisStale, null).synonyms = synonyms;
|
|
25554
|
+
function createIndex(indexName, config, language, analysisStale = false, synonyms = null, partitionCount) {
|
|
25555
|
+
addIndex(indexName, config, language, analysisStale, null, partitionCount).synonyms = synonyms;
|
|
25512
25556
|
}
|
|
25513
25557
|
function lookupOf(config, language, encoded) {
|
|
25514
25558
|
return encoded === void 0 ? null : synonymModule().storedLookup(encoded, config, language);
|
|
25515
25559
|
}
|
|
25516
|
-
function addIndex(indexName, config, language, analysisStale, awaitedCopyLoad) {
|
|
25560
|
+
function addIndex(indexName, config, language, analysisStale, awaitedCopyLoad, savedPartitionCount) {
|
|
25517
25561
|
if (indexes.has(indexName)) {
|
|
25518
25562
|
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${indexName}" already exists`, {
|
|
25519
25563
|
indexName
|
|
25520
25564
|
});
|
|
25521
25565
|
}
|
|
25522
|
-
const entry = buildEntry(
|
|
25566
|
+
const entry = buildEntry(
|
|
25567
|
+
indexName,
|
|
25568
|
+
config,
|
|
25569
|
+
language,
|
|
25570
|
+
analysisStale,
|
|
25571
|
+
awaitedCopyLoad,
|
|
25572
|
+
vectorWorkerCopies,
|
|
25573
|
+
savedPartitionCount
|
|
25574
|
+
);
|
|
25523
25575
|
indexes.set(indexName, entry);
|
|
25524
25576
|
return entry.manager;
|
|
25525
25577
|
}
|
|
25526
|
-
function buildEntry(indexName, config, language, analysisStale, awaitedCopyLoad, copies) {
|
|
25578
|
+
function buildEntry(indexName, config, language, analysisStale, awaitedCopyLoad, copies, savedPartitionCount) {
|
|
25527
25579
|
const router = createPartitionRouter();
|
|
25528
|
-
const partitionCount = config.partitions?.maxPartitions ?? 1;
|
|
25529
|
-
const vectorIndexes = vectorIndexesFor(indexName, config, copies);
|
|
25580
|
+
const partitionCount = savedPartitionCount ?? config.partitions?.maxPartitions ?? 1;
|
|
25581
|
+
const vectorIndexes = vectorIndexesFor(indexName, config, copies, vectorStorage, options?.onVectorGraphRepaired);
|
|
25530
25582
|
const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
|
|
25531
25583
|
return indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad);
|
|
25532
25584
|
}
|
|
@@ -25581,7 +25633,7 @@ function createDirectExecutor(options) {
|
|
|
25581
25633
|
);
|
|
25582
25634
|
}
|
|
25583
25635
|
case "createIndex": {
|
|
25584
|
-
const loading4 = loadSchemaCodeFor(action.config.schema);
|
|
25636
|
+
const loading4 = loadSchemaCodeFor(action.config.schema, action.config.tokenizer);
|
|
25585
25637
|
if (loading4 !== null) await loading4;
|
|
25586
25638
|
const language = getLanguage(action.config.language ?? "english");
|
|
25587
25639
|
const manager = addIndex(
|
|
@@ -25589,7 +25641,8 @@ function createDirectExecutor(options) {
|
|
|
25589
25641
|
action.config,
|
|
25590
25642
|
language,
|
|
25591
25643
|
action.analysisStale ?? false,
|
|
25592
|
-
action.copyLoad ?? null
|
|
25644
|
+
action.copyLoad ?? null,
|
|
25645
|
+
action.partitionCount
|
|
25593
25646
|
);
|
|
25594
25647
|
try {
|
|
25595
25648
|
manager.synonyms = lookupOf(action.config, language, action.synonyms);
|
|
@@ -25697,7 +25750,15 @@ function createDirectExecutor(options) {
|
|
|
25697
25750
|
case "clear": {
|
|
25698
25751
|
const entry = requireIndex(action.indexName);
|
|
25699
25752
|
entry.manager.clearPartitions();
|
|
25700
|
-
entry.manager.resetVectorIndexes(
|
|
25753
|
+
entry.manager.resetVectorIndexes(
|
|
25754
|
+
vectorIndexesFor(
|
|
25755
|
+
action.indexName,
|
|
25756
|
+
entry.config,
|
|
25757
|
+
vectorWorkerCopies,
|
|
25758
|
+
vectorStorage,
|
|
25759
|
+
options?.onVectorGraphRepaired
|
|
25760
|
+
)
|
|
25761
|
+
);
|
|
25701
25762
|
entry.vectorIndexes = entry.manager.getVectorIndexes();
|
|
25702
25763
|
if (action.publication === void 0) {
|
|
25703
25764
|
closeHeldVectorCopies(entry.vectorCopies);
|
|
@@ -25755,7 +25816,7 @@ function createDirectExecutor(options) {
|
|
|
25755
25816
|
dropIndex,
|
|
25756
25817
|
listIndexes,
|
|
25757
25818
|
vectorWorkerCopies,
|
|
25758
|
-
createDetachedEntry: (indexName, config, language, copies) => buildEntry(indexName, config, language, false, null, copies),
|
|
25819
|
+
createDetachedEntry: (indexName, config, language, copies, partitionCount) => buildEntry(indexName, config, language, false, null, copies, partitionCount),
|
|
25759
25820
|
installDetachedIndex,
|
|
25760
25821
|
discardDetachedIndex: (detached) => disposeEntry(detached.entry)
|
|
25761
25822
|
};
|
|
@@ -28141,7 +28202,12 @@ function requireVectorSearchable(params, context, needsVector) {
|
|
|
28141
28202
|
{ mode: params.mode }
|
|
28142
28203
|
);
|
|
28143
28204
|
}
|
|
28144
|
-
if (vectorSearchersOf(context).has(field))
|
|
28205
|
+
if (vectorSearchersOf(context).has(field)) {
|
|
28206
|
+
const vectorCode2 = requireLoadedVectorCode();
|
|
28207
|
+
vectorCode2.requireUsableQueryVector(params.vector?.value);
|
|
28208
|
+
vectorCode2.requireUsableVectorOptions(params.vector);
|
|
28209
|
+
return;
|
|
28210
|
+
}
|
|
28145
28211
|
throw new NarsilError(
|
|
28146
28212
|
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
28147
28213
|
`Field "${field}" has no vector index, so a vector search cannot use it`,
|
|
@@ -28380,7 +28446,6 @@ async function runQuery(request, callerContext, pin) {
|
|
|
28380
28446
|
}
|
|
28381
28447
|
const params = validQueryParams(request, config);
|
|
28382
28448
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
28383
|
-
if (isVectorOnly || isHybridMode) requireLoadedVectorCode().requireUsableVectorOptions(params.vector);
|
|
28384
28449
|
if (!isVectorOnly) requireUsablePhrases(params, context);
|
|
28385
28450
|
const shape = {
|
|
28386
28451
|
params,
|
|
@@ -28465,7 +28530,6 @@ async function executePreflight(request, callerContext) {
|
|
|
28465
28530
|
const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
|
|
28466
28531
|
const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
|
|
28467
28532
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
28468
|
-
if (isVectorOnly || isHybridMode) requireLoadedVectorCode().requireUsableVectorOptions(params.vector);
|
|
28469
28533
|
const loadingPhraseCode = loadPhraseCodeFor(params.term);
|
|
28470
28534
|
const loadingFacetCode = loadFacetMergesFor(params.facets);
|
|
28471
28535
|
if (loadingPhraseCode !== null || loadingFacetCode !== null) await Promise.all([loadingPhraseCode, loadingFacetCode]);
|
|
@@ -28525,7 +28589,7 @@ async function executePreflight(request, callerContext) {
|
|
|
28525
28589
|
|
|
28526
28590
|
// src/engine/resolve-vector-text-code.ts
|
|
28527
28591
|
var resolveVectorText = codeLoader(
|
|
28528
|
-
() => import('./resolve-vector-text-
|
|
28592
|
+
() => import('./resolve-vector-text-FTPD5EY6.mjs').then((code7) => {
|
|
28529
28593
|
code7.bindResolveVectorTextHost(optionalCodeHost());
|
|
28530
28594
|
return code7;
|
|
28531
28595
|
}),
|
|
@@ -29046,4 +29110,4 @@ async function createNarsil(config) {
|
|
|
29046
29110
|
// src/index.ts
|
|
29047
29111
|
var VERSION = "0.1.0";
|
|
29048
29112
|
|
|
29049
|
-
export { ARRAY_POSITION_GAP, COMPACTION_SEGMENT_TRIGGER, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_COPY_IDLE_TIMEOUT_MS, DEFAULT_COPY_THRESHOLD, GROUP_FORMAT, GROUP_PATTERN_SOURCE, GROWING_PARTIAL_ENTRY_BYTES, GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY, GROWING_PARTIAL_PENDING_ENTRY_BYTES, GROWING_PARTIAL_RUN_BYTES, GROWING_PARTIAL_RUN_MEMBER_BYTES, GrowingSegment, IDLE_MERGE_DELAY_MS, LIVE_TAIL_FLUSH_DOCUMENTS, LIVE_TAIL_FREEZE_FLOOR, LOAD_BUFFER_YIELD_INTERVAL, LONG_QUERY_THREADS, LivePostings2 as LivePostings, MAIN_COPY_LONE_QUERY_DOCUMENTS, MAX_PENDING_REPLICATION_DOCUMENTS, NO_REMOVED_SPELLINGS, PARTIAL_WORD_RUN_CODE_POINTS, POOL_RESTART_DELAY_MAX_MS, POOL_RESTART_DELAY_MS, PoolPlan, REPLICATION_WINDOW, RETIRED_THREADS_EXIT_WAIT_MS, RUN_MEMBER_START_CAPACITY, RUN_TABLE_START_CAPACITY, SEGMENTS_PER_MERGE, SLICE_CAPACITIES, SUGGESTIONS_INDEX_INFIX, SURFACE_TABLE_START_CAPACITY, SegmentRefusal, StringPlan, SurfaceTable, TERM_REGION_GROWTH_FACTOR, TERM_SLOT_LOAD_DENOMINATOR, TERM_SLOT_LOAD_NUMERATOR, TermLookup, VERSION, afterCurrentTurn, arrayPositionsAreStale, assemblePagedFrozenSegment, availablePatternSearch, bindOptionalWritesHost, bindSegmentCodecHost, bindSegmentMergeHost, buildPartialWordTableFrom, clearNormalizationCache, codePointCountOf, compareScores, configurationPayloadOf, configureNormalizationCache, countDocumentSpellings, countSpelling, createBooleanIndex, createCompositePartition, createEncodedDocumentSource, createEngineIndex, createEnumIndex, createFrozenRemovals, createFrozenSegment, createLoadedGeoIndex, createNarsil, createNumericIndex, createPartitionIndex, createPartitionManager, createPartitionRouter, createPublicationGate, createSharedFrozenSegment, createSpellingTable, createSurfaceRegistry, createWorkerFactory, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, dropEngineIndex, emitEngineEvent, emitSyncChange, emitWriteEventAfter, emptyPendingRegions, encodeDocumentTableData, encodeExternalIdTableData, encodeFieldIndexes, encodeFrozenTokenTableData, encodeLookupKey, encodePartialSpellings, encodePartialSpellingsOf, encodeSegmentState, encodeStringBlob, encodeSurfaceTable, endOfValue, engineCoreOf, expansionSource, generateId, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, getVectorFieldPaths, growPartitionsTo, grownCapacity, hasLanguage, hasStopWords, hasTokenizer, holdSyncEpochChange, insertDocument, insertDocumentBatch, insertDocumentVectors, isCompositePartition, isSimdAvailable, isSyncEpoch, kWayMerge, liveFrequency, loadDigest, loadGzip, loadOptionalWrites, loadSchemaCodeFor, loadStorageCode, loadSuggestionsCode, lookupOfStoredSynonyms, mergeDocFrequencies, mergeFacets, mergeFieldRegions, mergeKeywordSources, mergePartitionStats, mergePostings, mergePostings2, mergeStoredFields, mergeText, mergeTextAt, newSyncEpoch, oneAtATime, oversampledFacetConfig, oversizedConfigurationPayloadOf, partialSpellingsOfDocuments, partitionDocumentFrequency, partitionStateOf, patternHost, patternIndexOf, prepareConfiguredSynonyms, prepareDocumentText, prepareDocumentVectors, prepareRemovalText, prepareStoredSynonyms, readHeapStatistics, readResidentMemory, rebuildSizeIndexes, reconstructSchemaFromMetadata, recordValueSize, regionsOverlap, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, releaseRange, removeDocument3 as removeDocument, removeDocumentBatch, removeDocumentBatch2, removeDocumentVectors, removedSurfaceSpellings, requireEncodedConfigurationWithinSizeLimit, requireLoadedVectorCode, requirePatternSearch, requireRoom, requireSegment, requireSurfaceFormsForPartialFields, reserveStrings, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, resolveStopWords, returnsEveryMatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, rulesOfStoredSynonyms, sameFields, scanPostingBlocks, spellingCountsOf, storedArrayPositionGap, storedBM25Params, storedDocumentsOf, survivorDocumentsOf, termText, tokenize, topLevelMapKeys, transformToken, typedRegion, updateDocument, updateDocumentBatch, updateDocumentVectors, validateBM25Params, validateEmbeddingConfig, validatePartitionConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, visitDocumentSpellings, visitDocumentSurfaceSpellings, withOpenIndex, workersEnabledByDefault, wrapFrozenTokenTable, writePartitionDocument, yieldToEventLoop };
|
|
29113
|
+
export { ARRAY_POSITION_GAP, COMPACTION_SEGMENT_TRIGGER, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_COPY_IDLE_TIMEOUT_MS, DEFAULT_COPY_THRESHOLD, GROUP_FORMAT, GROUP_PATTERN_SOURCE, GROWING_PARTIAL_ENTRY_BYTES, GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY, GROWING_PARTIAL_PENDING_ENTRY_BYTES, GROWING_PARTIAL_RUN_BYTES, GROWING_PARTIAL_RUN_MEMBER_BYTES, GrowingSegment, IDLE_MERGE_DELAY_MS, LIVE_TAIL_FLUSH_DOCUMENTS, LIVE_TAIL_FREEZE_FLOOR, LOAD_BUFFER_YIELD_INTERVAL, LONG_QUERY_THREADS, LivePostings2 as LivePostings, MAIN_COPY_LONE_QUERY_DOCUMENTS, MAX_PENDING_REPLICATION_DOCUMENTS, NO_REMOVED_SPELLINGS, PARTIAL_WORD_RUN_CODE_POINTS, POOL_RESTART_DELAY_MAX_MS, POOL_RESTART_DELAY_MS, PoolPlan, REPLICATION_WINDOW, RETIRED_THREADS_EXIT_WAIT_MS, RUN_MEMBER_START_CAPACITY, RUN_TABLE_START_CAPACITY, SEGMENTS_PER_MERGE, SLICE_CAPACITIES, SUGGESTIONS_INDEX_INFIX, SURFACE_TABLE_START_CAPACITY, SegmentRefusal, StringPlan, SurfaceTable, TERM_REGION_GROWTH_FACTOR, TERM_SLOT_LOAD_DENOMINATOR, TERM_SLOT_LOAD_NUMERATOR, TermLookup, VERSION, afterCurrentTurn, arrayPositionsAreStale, assemblePagedFrozenSegment, availablePatternSearch, bindOptionalWritesHost, bindSegmentCodecHost, bindSegmentMergeHost, buildPartialWordTableFrom, clearNormalizationCache, codePointCountOf, compareScores, configurationPayloadOf, configureNormalizationCache, countDocumentSpellings, countSpelling, createBooleanIndex, createCompositePartition, createEncodedDocumentSource, createEngineIndex, createEnumIndex, createFrozenRemovals, createFrozenSegment, createLoadedGeoIndex, createNarsil, createNumericIndex, createPartitionIndex, createPartitionManager, createPartitionRouter, createPublicationGate, createSharedFrozenSegment, createSpellingTable, createSurfaceRegistry, createWorkerFactory, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, dropEngineIndex, emitEngineEvent, emitSyncChange, emitWriteEventAfter, emptyPendingRegions, encodeDocumentTableData, encodeExternalIdTableData, encodeFieldIndexes, encodeFrozenTokenTableData, encodeLookupKey, encodePartialSpellings, encodePartialSpellingsOf, encodeSegmentState, encodeStringBlob, encodeSurfaceTable, endOfValue, engineCoreOf, expansionSource, generateId, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, getVectorFieldPaths, growPartitionsTo, grownCapacity, hasLanguage, hasStopWords, hasTokenizer, holdSyncEpochChange, holdVectorsForRollback, insertDocument, insertDocumentBatch, insertDocumentVectors, isCompositePartition, isSimdAvailable, isSyncEpoch, kWayMerge, liveFrequency, loadDigest, loadGzip, loadOptionalWrites, loadSchemaCodeFor, loadStorageCode, loadSuggestionsCode, lookupOfStoredSynonyms, mergeDocFrequencies, mergeFacets, mergeFieldRegions, mergeKeywordSources, mergePartitionStats, mergePostings, mergePostings2, mergeStoredFields, mergeText, mergeTextAt, newSyncEpoch, oneAtATime, oversampledFacetConfig, oversizedConfigurationPayloadOf, partialSpellingsOfDocuments, partitionDocumentFrequency, partitionStateOf, patternHost, patternIndexOf, prepareConfiguredSynonyms, prepareDocumentText, prepareDocumentVectors, prepareRemovalText, prepareStoredSynonyms, readHeapStatistics, readResidentMemory, rebuildSizeIndexes, reconstructSchemaFromMetadata, recordValueSize, regionsOverlap, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, releaseRange, removeDocument3 as removeDocument, removeDocumentBatch, removeDocumentBatch2, removeDocumentVectors, removedSurfaceSpellings, requireEncodedConfigurationWithinSizeLimit, requireLoadedVectorCode, requirePatternSearch, requireRoom, requireSegment, requireSurfaceFormsForPartialFields, reserveStrings, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, resolveStopWords, returnsEveryMatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, rulesOfStoredSynonyms, sameFields, scanPostingBlocks, spellingCountsOf, storedArrayPositionGap, storedBM25Params, storedDocumentsOf, survivorDocumentsOf, termText, tokenize, topLevelMapKeys, transformToken, typedRegion, updateDocument, updateDocumentBatch, updateDocumentVectors, validateBM25Params, validateEmbeddingConfig, validatePartitionConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, visitDocumentSpellings, visitDocumentSurfaceSpellings, withOpenIndex, workersEnabledByDefault, wrapFrozenTokenTable, writePartitionDocument, yieldToEventLoop };
|