@delali/narsil 0.4.1 → 0.4.2-next.273
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-56UNHLAF.mjs} +1 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-EKISQBV2.mjs} +283 -244
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-VAPCFSMX.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-VIVG3OXM.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-76Q5SZ5S.mjs} +23 -10
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-3HKIVZFU.mjs} +295 -106
- package/dist/browser/{segment-codec-leaf-A427QJUX.mjs → segment-codec-leaf-GXSRX5VT.mjs} +47 -30
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-35RWLUVY.mjs} +63 -144
- package/dist/{chunk-6GHX35ZM.mjs → chunk-43T3P3BE.mjs} +1 -1
- package/dist/{chunk-IL5DRCNU.mjs → chunk-4JLHHEBN.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-4YIVY3PB.mjs} +2 -2
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-5SHWA6PE.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-6LVQ3GLT.mjs} +3 -3
- package/dist/{chunk-DDUXEJXW.mjs → chunk-6SIEVWMG.mjs} +45 -22
- package/dist/{chunk-WM5KJSQV.mjs → chunk-ACNHJOGC.mjs} +1 -1
- package/dist/{chunk-755NM4PL.mjs → chunk-BWPVDIRN.mjs} +183 -21
- package/dist/{chunk-SMQPZHPS.mjs → chunk-C57GWAXN.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-DRZP5MSG.mjs} +5 -5
- package/dist/{chunk-MYN4K5IS.mjs → chunk-DZMD4OMD.mjs} +1 -1
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-EY5U4P5V.mjs} +1 -1
- package/dist/{chunk-X3EQCQLP.mjs → chunk-EZJ6HBG7.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-FE2XILWN.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-G3KVEHIW.mjs} +2 -2
- package/dist/{chunk-B3MO37YF.mjs → chunk-GTGQLCEI.mjs} +1 -1
- package/dist/{chunk-MSJD74GN.mjs → chunk-H23FB267.mjs} +5 -5
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-WTMCGZ74.mjs → chunk-KXA77DPZ.mjs} +2 -2
- package/dist/{chunk-6I6OO6B2.mjs → chunk-LRIJ7U3T.mjs} +4 -4
- package/dist/{chunk-3IE3A74H.mjs → chunk-PNFP5THV.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-PQWK7KXS.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-OGYSPNL2.mjs → chunk-QNZJYMCF.mjs} +7 -7
- package/dist/{chunk-DBC4EJWB.mjs → chunk-R3PFPFME.mjs} +1 -1
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-MI6UZGUR.mjs → chunk-SPA7GXST.mjs} +3 -3
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-WYH3QHBK.mjs} +698 -451
- package/dist/{chunk-D456ZFPQ.mjs → chunk-X5V25BPN.mjs} +2 -2
- package/dist/{chunk-GXCDWLQI.mjs → chunk-Y7GXGJFZ.mjs} +23 -10
- package/dist/{chunk-NCIGNQJE.mjs → chunk-YGRJLI2Z.mjs} +930 -387
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-ZO7VLAIT.mjs} +2 -2
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-ZTB7AWTW.mjs} +4 -4
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-7G57AYXH.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-KPN7NSOC.mjs} +14 -14
- package/dist/msgpack-leaf-LIT7TKZW.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-CSULLISE.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-BKQADXOT.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-HTSJKZRU.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AOLLV37A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-DN4FT62B.mjs} +23 -10
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-EF4Y5C3Y.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-ESVBC3AT.mjs} +1 -1
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-H6LK5WCF.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-JEYZQA4Z.mjs} +699 -451
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-KJQ3BUVM.mjs} +4 -4
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-QYVRD6C5.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-X3VY2YT4.mjs} +2 -2
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-ZF3VSGEI.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-FOUPTT7J.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-MGFFGKYQ.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-MVJTP3E4.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PSWHL55A.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-WOJWSDLW.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-ZUN6Y44X.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-K5XG46FV.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-KDPM3P27.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-FVFMBBH6.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-3QEJNMFH.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-Q76Q4S35.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-BSDDR7DQ.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-6TT66HC5.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-R5JQ2X2M.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-NCGETZYY.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-TX3GTD6S.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-VLVPAUWC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-NZSBQZKG.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -12
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { reservedHeapBytes, reserveHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, admittedHeapBytes } from './chunk-HQEVQEDJ.mjs';
|
|
2
2
|
import { fnv1aUtf16, changeIndexOutsideExternalWaits, turnsAreBudgeted, turnIsSpent, yieldTurn, createDocIdMap, detectRuntime, fnv1a, indexChangeOutsideExternalWaits, stopOfSignal, splitWorkerBudget, resolveWorkerCount, sharedStopOfSignal, requireSearchActive, yieldBackgroundTurn, backgroundTurnIsSpent, FNV_OFFSET_BASIS, DEFAULT_REQUEST_TIMEOUT_MS, countWrite, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, answerFromOneState, signalOfSearch, requirePinnedState, fnv1aContinue, fnv1aContinueTaggedNumbers, FNV_PRIME, collectAndReturnPages, shardEntryRangeOf, threadSlotOfWorker, resolveRequestThreadCount, SHUTDOWN_TIMEOUT_MS, DEFAULT_BACKPRESSURE_LIMIT, threadSlotGone, spawnNodeWorker, resolveWorkerEntry, noteThreadSlotLive, noteThreadSlotGone, bindSearchSignal, adoptSharedTurns, WORKER_YOUNG_GENERATION_MB, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_BLOCK_MAX_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_ATTACH_RETRY_MS, MAX_WASM_PAGES, MAX_VECTOR_ORDINALS, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_DOC_ID_TABLE_BYTES, LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, DISK_STORAGE_BLOCK_BYTES, waitOutsideCountedWrite, nodeThreadId } from './chunk-AKRYHAVG.mjs';
|
|
3
3
|
import { MIN_DOCUMENTS_FOR_SEGMENTS, MAX_PARTITIONS, MAX_INDEX_CONFIGURATION_BYTES, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_CALLS_ACROSS_INDEX_CHANGES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MIN_SWEEP_INTERVAL_MS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, DEFAULT_PAGE_SIZE, CURSOR_BINDING_BUFFER_BYTES, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, MIN_RANK_CONSTANT, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, FUSION_TIE_INSERTION_LIMIT, DEFAULT_RANK_CONSTANT, DEFAULT_LINEAR_ALPHA, STATISTICS_BROADCAST_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, EXPORT_CHUNK_BYTES, DEFAULT_MAX_PARTITIONS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, ANALYSIS_CHANGE_MEMBERS, MIN_HYBRID_DEPTH, MAX_SUGGEST_SCATTER_LIMIT, MAX_SUGGEST_LIMIT, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_SUGGEST_LIMIT, DEFAULT_MAX_PER_GROUP } from './chunk-LX62Y4B5.mjs';
|
|
4
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, DEFAULT_PATTERN_WORK_CAP, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, bitsetSet, discardStoredValues, createBitSet, installStoredFields, storedValueBytes, bitsetIsEmpty, prepareStoredValues, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyPrefixOf, boundedLevenshtein, checkedSize, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, buildStoredValues, FILTER_OPERAND_COPY_DEPTH, fuzzyTermMatches, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIterator, GrowableUint32, bitsetFromSet } from './chunk-
|
|
4
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, DEFAULT_PATTERN_WORK_CAP, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, bitsetSet, discardStoredValues, createBitSet, installStoredFields, storedValueBytes, bitsetIsEmpty, prepareStoredValues, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyPrefixOf, boundedLevenshtein, checkedSize, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, buildStoredValues, FILTER_OPERAND_COPY_DEPTH, fuzzyTermMatches, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIterator, GrowableUint32, bitsetFromSet } from './chunk-ESVBC3AT.mjs';
|
|
5
5
|
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_NEEDS_ROOM, NATIVE_OK } from './chunk-UYKIV2KT.mjs';
|
|
6
6
|
import { reactNativeHost } from './chunk-CW2ZFXSE.mjs';
|
|
7
|
-
import { headerAt, MAP_TAGS, decode, encodeDocument, Decoder } from './chunk-
|
|
8
|
-
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, schemaFieldsOf, getNestedValue, tokenizeOptions, flattenSchema, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, vectorDimensionOf, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, PROTOTYPE_POLLUTION_KEYS, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, MAX_NESTING_DEPTH, getFlatSchema, getOrCreateFieldNameIndex, isWordIndexedBase, isStringValuedBase, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-
|
|
7
|
+
import { headerAt, MAP_TAGS, decode, encodeDocument, Decoder } from './chunk-DN4FT62B.mjs';
|
|
8
|
+
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, schemaFieldsOf, getNestedValue, tokenizeOptions, flattenSchema, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, vectorDimensionOf, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, PROTOTYPE_POLLUTION_KEYS, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, MAX_NESTING_DEPTH, getFlatSchema, getOrCreateFieldNameIndex, isWordIndexedBase, isStringValuedBase, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-AOLLV37A.mjs';
|
|
9
9
|
import { withUnpairedSurrogatesReplaced, compareCodePoints, holdsUnpairedSurrogate, defaultSortMode, toComparableSortValue, exceedsCodePoints, SORT_VALUE_MAX_CODE_POINTS, sortModeOf, PLACEMENT_ROOM_REQUESTS, NEAREST_LINK_FALLBACK_HOSTS, MIN_M, MAX_M, MAX_LAYER_CAP, INITIAL_UPPER_SLOTS, INITIAL_LIST_CAPACITY, FARTHEST_REJECTED_LINKS_KEPT, EARLIER_RELEASE_MIN_M, DEFAULT_EF_SEARCH, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, CONNECT_RESTARTS_BEFORE_ONE_PASS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, truncateSortString, SORT_MODES, isSortMode, codePointCount } from './chunk-HRDMIQ63.mjs';
|
|
10
10
|
import { MAX_DOCUMENT_ENCODING_DEPTH, MAX_FILE_BYTES, VALID_HNSW_METRICS, NATIVE_CRC32_MIN_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, DOCUMENT_REPLY_ENVELOPE_DEPTH, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER } from './chunk-VUJCSQG5.mjs';
|
|
11
11
|
import { NarsilError, ErrorCodes, createNarsilError, isNarsilError } from './chunk-43R4GCM7.mjs';
|
|
@@ -8682,7 +8682,7 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
8682
8682
|
|
|
8683
8683
|
// src/core/segment-codec-code.ts
|
|
8684
8684
|
var segmentCodec = codeLoader(async () => {
|
|
8685
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-
|
|
8685
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-PSWHL55A.mjs')]);
|
|
8686
8686
|
code7.bindSegmentCodecHost(optionalCodeHost(), msgpack);
|
|
8687
8687
|
return code7;
|
|
8688
8688
|
}, "segment codec");
|
|
@@ -8693,7 +8693,7 @@ var segmentMerge = codeLoader(async () => {
|
|
|
8693
8693
|
const [msgpack, codec, code7] = await Promise.all([
|
|
8694
8694
|
loadMessagePack(),
|
|
8695
8695
|
loadSegmentCodec(),
|
|
8696
|
-
import('./segment-merge-leaf-
|
|
8696
|
+
import('./segment-merge-leaf-WOJWSDLW.mjs')
|
|
8697
8697
|
]);
|
|
8698
8698
|
code7.bindSegmentMergeHost(optionalCodeHost(), msgpack, codec);
|
|
8699
8699
|
return code7;
|
|
@@ -8702,7 +8702,7 @@ var loadSegmentMerge = segmentMerge.load;
|
|
|
8702
8702
|
|
|
8703
8703
|
// src/query-suggestions/store-code.ts
|
|
8704
8704
|
var suggestionStore = codeLoader(async () => {
|
|
8705
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-
|
|
8705
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-K5XG46FV.mjs')]);
|
|
8706
8706
|
code7.bindSuggestionStoreHost(optionalCodeHost(), msgpack);
|
|
8707
8707
|
return code7;
|
|
8708
8708
|
}, "query suggestions store");
|
|
@@ -8783,7 +8783,7 @@ var storageCode = codeLoader(async () => {
|
|
|
8783
8783
|
loadSegmentMerge(),
|
|
8784
8784
|
loadSuggestionStore(),
|
|
8785
8785
|
loadVectorCore(),
|
|
8786
|
-
import('./storage-leaf-
|
|
8786
|
+
import('./storage-leaf-ZUN6Y44X.mjs')
|
|
8787
8787
|
]);
|
|
8788
8788
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
8789
8789
|
return code7;
|
|
@@ -11585,6 +11585,177 @@ function mergeGroupComponents(components, internalId, contribution) {
|
|
|
11585
11585
|
});
|
|
11586
11586
|
}
|
|
11587
11587
|
|
|
11588
|
+
// src/core/partition/top-k-heap.ts
|
|
11589
|
+
function candidateWorse(a, b) {
|
|
11590
|
+
if (a.score < b.score) return true;
|
|
11591
|
+
if (a.score > b.score) return false;
|
|
11592
|
+
return unorderedCandidateWorse(a, b);
|
|
11593
|
+
}
|
|
11594
|
+
function unorderedCandidateWorse(a, b) {
|
|
11595
|
+
return (compareScores(a.score, b.score) || compareCodePoints(a.externalId, b.externalId)) > 0;
|
|
11596
|
+
}
|
|
11597
|
+
function buildMinHeap(heap) {
|
|
11598
|
+
for (let index = (heap.length >> 1) - 1; index >= 0; index--) {
|
|
11599
|
+
siftDown(heap, index);
|
|
11600
|
+
}
|
|
11601
|
+
}
|
|
11602
|
+
function siftDown(heap, start) {
|
|
11603
|
+
const len = heap.length;
|
|
11604
|
+
let current = start;
|
|
11605
|
+
for (; ; ) {
|
|
11606
|
+
let worst = current;
|
|
11607
|
+
const left = 2 * current + 1;
|
|
11608
|
+
const right = 2 * current + 2;
|
|
11609
|
+
if (left < len && candidateWorse(heap[left], heap[worst])) worst = left;
|
|
11610
|
+
if (right < len && candidateWorse(heap[right], heap[worst])) worst = right;
|
|
11611
|
+
if (worst === current) break;
|
|
11612
|
+
const held = heap[current];
|
|
11613
|
+
heap[current] = heap[worst];
|
|
11614
|
+
heap[worst] = held;
|
|
11615
|
+
current = worst;
|
|
11616
|
+
}
|
|
11617
|
+
}
|
|
11618
|
+
function sortSelection(heap) {
|
|
11619
|
+
heap.sort((a, b) => compareScores(a.score, b.score) || compareCodePoints(a.externalId, b.externalId));
|
|
11620
|
+
}
|
|
11621
|
+
|
|
11622
|
+
// src/core/partition/score-buffer.ts
|
|
11623
|
+
var MAX_GENERATION = 2147483647;
|
|
11624
|
+
function createScoreBuffer(capacity) {
|
|
11625
|
+
return {
|
|
11626
|
+
scores: new Float64Array(capacity),
|
|
11627
|
+
stamps: new Int32Array(capacity),
|
|
11628
|
+
touched: new Int32Array(Math.max(1, Math.min(capacity, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY))),
|
|
11629
|
+
touchedCount: 0,
|
|
11630
|
+
generation: 0
|
|
11631
|
+
};
|
|
11632
|
+
}
|
|
11633
|
+
function beginScoring(buffer, capacity) {
|
|
11634
|
+
if (buffer.scores.length < capacity) {
|
|
11635
|
+
buffer.scores = new Float64Array(capacity);
|
|
11636
|
+
buffer.stamps = new Int32Array(capacity);
|
|
11637
|
+
buffer.generation = 0;
|
|
11638
|
+
}
|
|
11639
|
+
if (buffer.generation >= MAX_GENERATION) {
|
|
11640
|
+
buffer.stamps.fill(0);
|
|
11641
|
+
buffer.generation = 0;
|
|
11642
|
+
}
|
|
11643
|
+
buffer.generation++;
|
|
11644
|
+
buffer.touchedCount = 0;
|
|
11645
|
+
}
|
|
11646
|
+
function appendTouched(buffer, internalId) {
|
|
11647
|
+
if (buffer.touchedCount === buffer.touched.length) {
|
|
11648
|
+
const grown4 = new Int32Array(buffer.touched.length * 2);
|
|
11649
|
+
grown4.set(buffer.touched);
|
|
11650
|
+
buffer.touched = grown4;
|
|
11651
|
+
}
|
|
11652
|
+
buffer.touched[buffer.touchedCount] = internalId;
|
|
11653
|
+
buffer.touchedCount++;
|
|
11654
|
+
}
|
|
11655
|
+
function addScore(buffer, internalId, termScore) {
|
|
11656
|
+
if (buffer.stamps[internalId] === buffer.generation) {
|
|
11657
|
+
buffer.scores[internalId] += termScore;
|
|
11658
|
+
return;
|
|
11659
|
+
}
|
|
11660
|
+
buffer.stamps[internalId] = buffer.generation;
|
|
11661
|
+
buffer.scores[internalId] = termScore;
|
|
11662
|
+
appendTouched(buffer, internalId);
|
|
11663
|
+
}
|
|
11664
|
+
function markCounted(buffer, internalId) {
|
|
11665
|
+
const countedStamp = -buffer.generation;
|
|
11666
|
+
const stamp = buffer.stamps[internalId];
|
|
11667
|
+
if (stamp === buffer.generation || stamp === countedStamp) return false;
|
|
11668
|
+
buffer.stamps[internalId] = countedStamp;
|
|
11669
|
+
return true;
|
|
11670
|
+
}
|
|
11671
|
+
function kthBestScore(buffer, selection, resolver) {
|
|
11672
|
+
const wanted = selection.length;
|
|
11673
|
+
const { touched, touchedCount, scores } = buffer;
|
|
11674
|
+
let size = 0;
|
|
11675
|
+
let unordered = 0;
|
|
11676
|
+
for (let index = 0; index < touchedCount; index++) {
|
|
11677
|
+
const internalId = touched[index];
|
|
11678
|
+
const score = scores[internalId];
|
|
11679
|
+
if (size === wanted && !(score > selection[0])) continue;
|
|
11680
|
+
if (!resolver.isLive(internalId)) continue;
|
|
11681
|
+
let at = 0;
|
|
11682
|
+
if (size < wanted) {
|
|
11683
|
+
if (Number.isNaN(score)) {
|
|
11684
|
+
unordered++;
|
|
11685
|
+
continue;
|
|
11686
|
+
}
|
|
11687
|
+
at = size++;
|
|
11688
|
+
while (at > 0) {
|
|
11689
|
+
const parent = at - 1 >> 1;
|
|
11690
|
+
if (selection[parent] <= score) break;
|
|
11691
|
+
selection[at] = selection[parent];
|
|
11692
|
+
at = parent;
|
|
11693
|
+
}
|
|
11694
|
+
} else {
|
|
11695
|
+
for (; ; ) {
|
|
11696
|
+
const left = 2 * at + 1;
|
|
11697
|
+
if (left >= size) break;
|
|
11698
|
+
const right = left + 1;
|
|
11699
|
+
const child = right < size && selection[right] < selection[left] ? right : left;
|
|
11700
|
+
if (selection[child] >= score) break;
|
|
11701
|
+
selection[at] = selection[child];
|
|
11702
|
+
at = child;
|
|
11703
|
+
}
|
|
11704
|
+
}
|
|
11705
|
+
selection[at] = score;
|
|
11706
|
+
}
|
|
11707
|
+
if (wanted === 0) return Number.NEGATIVE_INFINITY;
|
|
11708
|
+
if (size === wanted) return selection[0];
|
|
11709
|
+
return size + unordered >= wanted ? Number.NaN : Number.NEGATIVE_INFINITY;
|
|
11710
|
+
}
|
|
11711
|
+
function hasScore(buffer, internalId) {
|
|
11712
|
+
return buffer.stamps[internalId] === buffer.generation;
|
|
11713
|
+
}
|
|
11714
|
+
function topKFromBuffer(buffer, k, resolver, components) {
|
|
11715
|
+
const wanted = Number.isFinite(k) ? Math.max(0, Math.floor(k)) : 0;
|
|
11716
|
+
if (wanted <= 0) return [];
|
|
11717
|
+
const heap = [];
|
|
11718
|
+
const { touched, touchedCount, scores } = buffer;
|
|
11719
|
+
let full = false;
|
|
11720
|
+
let threshold = 0;
|
|
11721
|
+
for (let index = 0; index < touchedCount; index++) {
|
|
11722
|
+
const internalId = touched[index];
|
|
11723
|
+
const score = scores[internalId];
|
|
11724
|
+
if (full && (score < threshold || Number.isNaN(score) && !Number.isNaN(threshold))) continue;
|
|
11725
|
+
const externalId = resolver.toExternal(internalId);
|
|
11726
|
+
if (externalId === void 0) continue;
|
|
11727
|
+
const candidate = { internalId, externalId, score };
|
|
11728
|
+
if (!full) {
|
|
11729
|
+
heap.push(candidate);
|
|
11730
|
+
if (heap.length === wanted) {
|
|
11731
|
+
buildMinHeap(heap);
|
|
11732
|
+
full = true;
|
|
11733
|
+
threshold = heap[0].score;
|
|
11734
|
+
}
|
|
11735
|
+
continue;
|
|
11736
|
+
}
|
|
11737
|
+
if (candidateWorse(heap[0], candidate)) {
|
|
11738
|
+
heap[0] = candidate;
|
|
11739
|
+
siftDown(heap, 0);
|
|
11740
|
+
threshold = heap[0].score;
|
|
11741
|
+
}
|
|
11742
|
+
}
|
|
11743
|
+
sortSelection(heap);
|
|
11744
|
+
const result = new Array(heap.length);
|
|
11745
|
+
for (let index = 0; index < heap.length; index++) {
|
|
11746
|
+
const entry = heap[index];
|
|
11747
|
+
const record2 = components?.get(entry.internalId);
|
|
11748
|
+
result[index] = {
|
|
11749
|
+
docId: entry.externalId,
|
|
11750
|
+
score: entry.score,
|
|
11751
|
+
termFrequencies: record2?.termFrequencies ?? EMPTY_COMPONENTS,
|
|
11752
|
+
fieldLengths: record2?.fieldLengths ?? EMPTY_COMPONENTS,
|
|
11753
|
+
idf: record2?.idf ?? EMPTY_COMPONENTS
|
|
11754
|
+
};
|
|
11755
|
+
}
|
|
11756
|
+
return result;
|
|
11757
|
+
}
|
|
11758
|
+
|
|
11588
11759
|
// src/core/partition/group-scoring.ts
|
|
11589
11760
|
function partialFieldMarks(fieldNames, partialFields) {
|
|
11590
11761
|
const marks = new Uint8Array(fieldNames.length);
|
|
@@ -11638,72 +11809,98 @@ function groupPostingCount(matches) {
|
|
|
11638
11809
|
for (const match of matches) total += match.postingList.length;
|
|
11639
11810
|
return total;
|
|
11640
11811
|
}
|
|
11641
|
-
function
|
|
11642
|
-
const { fields,
|
|
11812
|
+
function scoreMatchPostings(context, match, visit) {
|
|
11813
|
+
const { fields, filterBitset, partialMarks } = context;
|
|
11643
11814
|
const scoresAreZero = context.totalDocs === 0;
|
|
11644
11815
|
const onePair = fields.boundPairs.length === 2;
|
|
11816
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
11817
|
+
match.postingList
|
|
11818
|
+
);
|
|
11819
|
+
let runStart = 0;
|
|
11820
|
+
let runEnd = 0;
|
|
11821
|
+
for (let at = 0; at < count; at++) {
|
|
11822
|
+
const internalId = docIds[at];
|
|
11823
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
11824
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
11825
|
+
let pi = at;
|
|
11826
|
+
if (at < runEnd) {
|
|
11827
|
+
pi = fieldRun.entries[at - runStart];
|
|
11828
|
+
} else if (at + 2 < count && docIds[at + 2] === internalId) {
|
|
11829
|
+
pi = orderRunByFieldName(docIds, fieldNameIndices, fields.nameRanks, at, count)[0];
|
|
11830
|
+
runStart = at;
|
|
11831
|
+
runEnd = at + fieldRun.length;
|
|
11832
|
+
}
|
|
11833
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
11834
|
+
if (fields.searchable[fieldIndex] === 0) continue;
|
|
11835
|
+
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
11836
|
+
if (factor === 0) continue;
|
|
11837
|
+
const termFrequency = termFrequencies[pi];
|
|
11838
|
+
const fieldBoost = fields.boosts[fieldIndex];
|
|
11839
|
+
const avgLen = fields.averageLengths[fieldIndex];
|
|
11840
|
+
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
11841
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
11842
|
+
termFrequency,
|
|
11843
|
+
match.idf,
|
|
11844
|
+
actualFieldLength,
|
|
11845
|
+
avgLen,
|
|
11846
|
+
onePair ? fields.boundPairs[0] : fields.k1s[fieldIndex],
|
|
11847
|
+
onePair ? fields.boundPairs[1] : fields.bs[fieldIndex]
|
|
11848
|
+
)) * fieldBoost * factor;
|
|
11849
|
+
visit(internalId, fieldIndex, termFrequency, actualFieldLength, termScore);
|
|
11850
|
+
}
|
|
11851
|
+
}
|
|
11852
|
+
var termScores = null;
|
|
11853
|
+
var bestScores = null;
|
|
11854
|
+
function addGroupScores(context, matches, target, only, coverage) {
|
|
11855
|
+
const best = bestGroupScores(context, matches, target.scores.length);
|
|
11856
|
+
for (let index = 0; index < best.touchedCount; index++) {
|
|
11857
|
+
const internalId = best.touched[index];
|
|
11858
|
+
if (only !== null && !bitsetHas(only, internalId)) continue;
|
|
11859
|
+
coverage?.mark(internalId);
|
|
11860
|
+
addScore(target, internalId, best.scores[internalId]);
|
|
11861
|
+
}
|
|
11862
|
+
}
|
|
11863
|
+
function bestGroupScores(context, matches, capacity) {
|
|
11864
|
+
termScores ??= createScoreBuffer(capacity);
|
|
11865
|
+
bestScores ??= createScoreBuffer(capacity);
|
|
11866
|
+
const perTerm = termScores;
|
|
11867
|
+
const best = bestScores;
|
|
11868
|
+
const addTermScore = (internalId, _fieldIndex, _termFrequency, _fieldLength, termScore) => addScore(perTerm, internalId, termScore);
|
|
11869
|
+
beginScoring(best, capacity);
|
|
11870
|
+
for (const match of matches) {
|
|
11871
|
+
beginScoring(perTerm, capacity);
|
|
11872
|
+
scoreMatchPostings(context, match, addTermScore);
|
|
11873
|
+
for (let index = 0; index < perTerm.touchedCount; index++) {
|
|
11874
|
+
const internalId = perTerm.touched[index];
|
|
11875
|
+
const score = perTerm.scores[internalId];
|
|
11876
|
+
if (!hasScore(best, internalId)) addScore(best, internalId, score);
|
|
11877
|
+
else if (compareScores(score, best.scores[internalId]) < 0) best.scores[internalId] = score;
|
|
11878
|
+
}
|
|
11879
|
+
}
|
|
11880
|
+
return best;
|
|
11881
|
+
}
|
|
11882
|
+
function computeGroupContributions(context, matches) {
|
|
11883
|
+
const { fieldNames } = context;
|
|
11645
11884
|
const best = /* @__PURE__ */ new Map();
|
|
11646
11885
|
for (const match of matches) {
|
|
11647
11886
|
const perTerm = /* @__PURE__ */ new Map();
|
|
11648
|
-
|
|
11649
|
-
match.postingList
|
|
11650
|
-
);
|
|
11651
|
-
let runStart = 0;
|
|
11652
|
-
let runEnd = 0;
|
|
11653
|
-
for (let at = 0; at < count; at++) {
|
|
11654
|
-
const internalId = docIds[at];
|
|
11655
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
11656
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
11657
|
-
let pi = at;
|
|
11658
|
-
if (at < runEnd) {
|
|
11659
|
-
pi = fieldRun.entries[at - runStart];
|
|
11660
|
-
} else if (at + 2 < count && docIds[at + 2] === internalId) {
|
|
11661
|
-
pi = orderRunByFieldName(docIds, fieldNameIndices, fields.nameRanks, at, count)[0];
|
|
11662
|
-
runStart = at;
|
|
11663
|
-
runEnd = at + fieldRun.length;
|
|
11664
|
-
}
|
|
11665
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
11666
|
-
if (fields.searchable[fieldIndex] === 0) continue;
|
|
11667
|
-
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
11668
|
-
if (factor === 0) continue;
|
|
11887
|
+
scoreMatchPostings(context, match, (internalId, fieldIndex, termFrequency, actualFieldLength, termScore) => {
|
|
11669
11888
|
const fieldName = fieldNames[fieldIndex];
|
|
11670
|
-
const termFrequency = termFrequencies[pi];
|
|
11671
|
-
const fieldBoost = fields.boosts[fieldIndex];
|
|
11672
|
-
const avgLen = fields.averageLengths[fieldIndex];
|
|
11673
|
-
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
11674
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
11675
|
-
termFrequency,
|
|
11676
|
-
match.idf,
|
|
11677
|
-
actualFieldLength,
|
|
11678
|
-
avgLen,
|
|
11679
|
-
onePair ? fields.boundPairs[0] : fields.k1s[fieldIndex],
|
|
11680
|
-
onePair ? fields.boundPairs[1] : fields.bs[fieldIndex]
|
|
11681
|
-
)) * fieldBoost * factor;
|
|
11682
11889
|
const existing = perTerm.get(internalId);
|
|
11683
11890
|
if (existing) {
|
|
11684
11891
|
existing.score += termScore;
|
|
11685
|
-
|
|
11686
|
-
|
|
11687
|
-
|
|
11688
|
-
}
|
|
11689
|
-
} else if (collect2) {
|
|
11690
|
-
perTerm.set(internalId, {
|
|
11691
|
-
score: termScore,
|
|
11692
|
-
token: match.token,
|
|
11693
|
-
idf: match.idf,
|
|
11694
|
-
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
11695
|
-
fieldLengths: { [fieldName]: actualFieldLength }
|
|
11696
|
-
});
|
|
11697
|
-
} else {
|
|
11698
|
-
perTerm.set(internalId, {
|
|
11699
|
-
score: termScore,
|
|
11700
|
-
token: match.token,
|
|
11701
|
-
idf: match.idf,
|
|
11702
|
-
termFrequencies: EMPTY_COMPONENTS,
|
|
11703
|
-
fieldLengths: EMPTY_COMPONENTS
|
|
11704
|
-
});
|
|
11892
|
+
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
11893
|
+
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
11894
|
+
return;
|
|
11705
11895
|
}
|
|
11706
|
-
|
|
11896
|
+
perTerm.set(internalId, {
|
|
11897
|
+
score: termScore,
|
|
11898
|
+
token: match.token,
|
|
11899
|
+
idf: match.idf,
|
|
11900
|
+
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
11901
|
+
fieldLengths: { [fieldName]: actualFieldLength }
|
|
11902
|
+
});
|
|
11903
|
+
});
|
|
11707
11904
|
for (const [internalId, contribution] of perTerm) {
|
|
11708
11905
|
const current = best.get(internalId);
|
|
11709
11906
|
if (!current || compareScores(contribution.score, current.score) < 0) {
|
|
@@ -12184,221 +12381,50 @@ function scanPostingBlocks(list, columns2, previous, keepBlocks) {
|
|
|
12184
12381
|
}
|
|
12185
12382
|
run++;
|
|
12186
12383
|
}
|
|
12187
|
-
if (!tombstoned2) {
|
|
12188
|
-
if (run > blockMaxRun) blockMaxRun = run;
|
|
12189
|
-
blockDocs++;
|
|
12190
|
-
}
|
|
12191
|
-
entry += run;
|
|
12192
|
-
if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === length) {
|
|
12193
|
-
entryEnd[blockIndex] = entry;
|
|
12194
|
-
maxTermFrequency[blockIndex] = blockMaxTf;
|
|
12195
|
-
minFieldLength[blockIndex] = blockMinLen === Number.POSITIVE_INFINITY ? ABSENT_FIELD_LENGTH_BOUND : blockMinLen;
|
|
12196
|
-
maxEntriesPerDocument[blockIndex] = blockMaxRun;
|
|
12197
|
-
documentCount[blockIndex] = blockDocs;
|
|
12198
|
-
blockIndex++;
|
|
12199
|
-
blockStart = entry;
|
|
12200
|
-
blockMaxTf = 0;
|
|
12201
|
-
blockMinLen = Number.POSITIVE_INFINITY;
|
|
12202
|
-
blockMaxRun = 0;
|
|
12203
|
-
blockDocs = 0;
|
|
12204
|
-
}
|
|
12205
|
-
}
|
|
12206
|
-
return {
|
|
12207
|
-
structureRevision: list.structureRevision,
|
|
12208
|
-
blockCount: blockIndex,
|
|
12209
|
-
entryEnd,
|
|
12210
|
-
maxTermFrequency,
|
|
12211
|
-
minFieldLength,
|
|
12212
|
-
maxEntriesPerDocument,
|
|
12213
|
-
documentCount
|
|
12214
|
-
};
|
|
12215
|
-
}
|
|
12216
|
-
function blockBoundsFor(list, columns2) {
|
|
12217
|
-
const existing = boundsByList.get(list);
|
|
12218
|
-
if (existing !== void 0 && existing.structureRevision === list.structureRevision) {
|
|
12219
|
-
const covered = entriesCovered(existing);
|
|
12220
|
-
if (covered === list.length) return existing;
|
|
12221
|
-
if (list.ordered && covered < list.length) {
|
|
12222
|
-
const extended = scanPostingBlocks(list, columns2, existing, Math.max(0, existing.blockCount - 1));
|
|
12223
|
-
boundsByList.set(list, extended);
|
|
12224
|
-
return extended;
|
|
12225
|
-
}
|
|
12226
|
-
}
|
|
12227
|
-
const stored = list.storedBlockBounds?.() ?? null;
|
|
12228
|
-
const built2 = stored === null ? scanPostingBlocks(list, columns2, null, 0) : { ...stored, structureRevision: list.structureRevision };
|
|
12229
|
-
boundsByList.set(list, built2);
|
|
12230
|
-
return built2;
|
|
12231
|
-
}
|
|
12232
|
-
|
|
12233
|
-
// src/core/partition/top-k-heap.ts
|
|
12234
|
-
function candidateWorse(a, b) {
|
|
12235
|
-
if (a.score < b.score) return true;
|
|
12236
|
-
if (a.score > b.score) return false;
|
|
12237
|
-
return unorderedCandidateWorse(a, b);
|
|
12238
|
-
}
|
|
12239
|
-
function unorderedCandidateWorse(a, b) {
|
|
12240
|
-
return (compareScores(a.score, b.score) || compareCodePoints(a.externalId, b.externalId)) > 0;
|
|
12241
|
-
}
|
|
12242
|
-
function buildMinHeap(heap) {
|
|
12243
|
-
for (let index = (heap.length >> 1) - 1; index >= 0; index--) {
|
|
12244
|
-
siftDown(heap, index);
|
|
12245
|
-
}
|
|
12246
|
-
}
|
|
12247
|
-
function siftDown(heap, start) {
|
|
12248
|
-
const len = heap.length;
|
|
12249
|
-
let current = start;
|
|
12250
|
-
for (; ; ) {
|
|
12251
|
-
let worst = current;
|
|
12252
|
-
const left = 2 * current + 1;
|
|
12253
|
-
const right = 2 * current + 2;
|
|
12254
|
-
if (left < len && candidateWorse(heap[left], heap[worst])) worst = left;
|
|
12255
|
-
if (right < len && candidateWorse(heap[right], heap[worst])) worst = right;
|
|
12256
|
-
if (worst === current) break;
|
|
12257
|
-
const held = heap[current];
|
|
12258
|
-
heap[current] = heap[worst];
|
|
12259
|
-
heap[worst] = held;
|
|
12260
|
-
current = worst;
|
|
12261
|
-
}
|
|
12262
|
-
}
|
|
12263
|
-
function sortSelection(heap) {
|
|
12264
|
-
heap.sort((a, b) => compareScores(a.score, b.score) || compareCodePoints(a.externalId, b.externalId));
|
|
12265
|
-
}
|
|
12266
|
-
|
|
12267
|
-
// src/core/partition/score-buffer.ts
|
|
12268
|
-
var MAX_GENERATION = 2147483647;
|
|
12269
|
-
function createScoreBuffer(capacity) {
|
|
12270
|
-
return {
|
|
12271
|
-
scores: new Float64Array(capacity),
|
|
12272
|
-
stamps: new Int32Array(capacity),
|
|
12273
|
-
touched: new Int32Array(Math.max(1, Math.min(capacity, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY))),
|
|
12274
|
-
touchedCount: 0,
|
|
12275
|
-
generation: 0
|
|
12276
|
-
};
|
|
12277
|
-
}
|
|
12278
|
-
function beginScoring(buffer, capacity) {
|
|
12279
|
-
if (buffer.scores.length < capacity) {
|
|
12280
|
-
buffer.scores = new Float64Array(capacity);
|
|
12281
|
-
buffer.stamps = new Int32Array(capacity);
|
|
12282
|
-
buffer.generation = 0;
|
|
12283
|
-
}
|
|
12284
|
-
if (buffer.generation >= MAX_GENERATION) {
|
|
12285
|
-
buffer.stamps.fill(0);
|
|
12286
|
-
buffer.generation = 0;
|
|
12287
|
-
}
|
|
12288
|
-
buffer.generation++;
|
|
12289
|
-
buffer.touchedCount = 0;
|
|
12290
|
-
}
|
|
12291
|
-
function appendTouched(buffer, internalId) {
|
|
12292
|
-
if (buffer.touchedCount === buffer.touched.length) {
|
|
12293
|
-
const grown4 = new Int32Array(buffer.touched.length * 2);
|
|
12294
|
-
grown4.set(buffer.touched);
|
|
12295
|
-
buffer.touched = grown4;
|
|
12296
|
-
}
|
|
12297
|
-
buffer.touched[buffer.touchedCount] = internalId;
|
|
12298
|
-
buffer.touchedCount++;
|
|
12299
|
-
}
|
|
12300
|
-
function addScore(buffer, internalId, termScore) {
|
|
12301
|
-
if (buffer.stamps[internalId] === buffer.generation) {
|
|
12302
|
-
buffer.scores[internalId] += termScore;
|
|
12303
|
-
return;
|
|
12304
|
-
}
|
|
12305
|
-
buffer.stamps[internalId] = buffer.generation;
|
|
12306
|
-
buffer.scores[internalId] = termScore;
|
|
12307
|
-
appendTouched(buffer, internalId);
|
|
12308
|
-
}
|
|
12309
|
-
function markCounted(buffer, internalId) {
|
|
12310
|
-
const countedStamp = -buffer.generation;
|
|
12311
|
-
const stamp = buffer.stamps[internalId];
|
|
12312
|
-
if (stamp === buffer.generation || stamp === countedStamp) return false;
|
|
12313
|
-
buffer.stamps[internalId] = countedStamp;
|
|
12314
|
-
return true;
|
|
12315
|
-
}
|
|
12316
|
-
function kthBestScore(buffer, selection, resolver) {
|
|
12317
|
-
const wanted = selection.length;
|
|
12318
|
-
const { touched, touchedCount, scores } = buffer;
|
|
12319
|
-
let size = 0;
|
|
12320
|
-
let unordered = 0;
|
|
12321
|
-
for (let index = 0; index < touchedCount; index++) {
|
|
12322
|
-
const internalId = touched[index];
|
|
12323
|
-
const score = scores[internalId];
|
|
12324
|
-
if (size === wanted && !(score > selection[0])) continue;
|
|
12325
|
-
if (!resolver.isLive(internalId)) continue;
|
|
12326
|
-
let at = 0;
|
|
12327
|
-
if (size < wanted) {
|
|
12328
|
-
if (Number.isNaN(score)) {
|
|
12329
|
-
unordered++;
|
|
12330
|
-
continue;
|
|
12331
|
-
}
|
|
12332
|
-
at = size++;
|
|
12333
|
-
while (at > 0) {
|
|
12334
|
-
const parent = at - 1 >> 1;
|
|
12335
|
-
if (selection[parent] <= score) break;
|
|
12336
|
-
selection[at] = selection[parent];
|
|
12337
|
-
at = parent;
|
|
12338
|
-
}
|
|
12339
|
-
} else {
|
|
12340
|
-
for (; ; ) {
|
|
12341
|
-
const left = 2 * at + 1;
|
|
12342
|
-
if (left >= size) break;
|
|
12343
|
-
const right = left + 1;
|
|
12344
|
-
const child = right < size && selection[right] < selection[left] ? right : left;
|
|
12345
|
-
if (selection[child] >= score) break;
|
|
12346
|
-
selection[at] = selection[child];
|
|
12347
|
-
at = child;
|
|
12348
|
-
}
|
|
12349
|
-
}
|
|
12350
|
-
selection[at] = score;
|
|
12351
|
-
}
|
|
12352
|
-
if (wanted === 0) return Number.NEGATIVE_INFINITY;
|
|
12353
|
-
if (size === wanted) return selection[0];
|
|
12354
|
-
return size + unordered >= wanted ? Number.NaN : Number.NEGATIVE_INFINITY;
|
|
12355
|
-
}
|
|
12356
|
-
function hasScore(buffer, internalId) {
|
|
12357
|
-
return buffer.stamps[internalId] === buffer.generation;
|
|
12358
|
-
}
|
|
12359
|
-
function topKFromBuffer(buffer, k, resolver, components) {
|
|
12360
|
-
const wanted = Number.isFinite(k) ? Math.max(0, Math.floor(k)) : 0;
|
|
12361
|
-
if (wanted <= 0) return [];
|
|
12362
|
-
const heap = [];
|
|
12363
|
-
const { touched, touchedCount, scores } = buffer;
|
|
12364
|
-
let full = false;
|
|
12365
|
-
let threshold = 0;
|
|
12366
|
-
for (let index = 0; index < touchedCount; index++) {
|
|
12367
|
-
const internalId = touched[index];
|
|
12368
|
-
const score = scores[internalId];
|
|
12369
|
-
if (full && (score < threshold || Number.isNaN(score) && !Number.isNaN(threshold))) continue;
|
|
12370
|
-
const externalId = resolver.toExternal(internalId);
|
|
12371
|
-
if (externalId === void 0) continue;
|
|
12372
|
-
const candidate = { internalId, externalId, score };
|
|
12373
|
-
if (!full) {
|
|
12374
|
-
heap.push(candidate);
|
|
12375
|
-
if (heap.length === wanted) {
|
|
12376
|
-
buildMinHeap(heap);
|
|
12377
|
-
full = true;
|
|
12378
|
-
threshold = heap[0].score;
|
|
12379
|
-
}
|
|
12380
|
-
continue;
|
|
12384
|
+
if (!tombstoned2) {
|
|
12385
|
+
if (run > blockMaxRun) blockMaxRun = run;
|
|
12386
|
+
blockDocs++;
|
|
12381
12387
|
}
|
|
12382
|
-
|
|
12383
|
-
|
|
12384
|
-
|
|
12385
|
-
|
|
12388
|
+
entry += run;
|
|
12389
|
+
if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === length) {
|
|
12390
|
+
entryEnd[blockIndex] = entry;
|
|
12391
|
+
maxTermFrequency[blockIndex] = blockMaxTf;
|
|
12392
|
+
minFieldLength[blockIndex] = blockMinLen === Number.POSITIVE_INFINITY ? ABSENT_FIELD_LENGTH_BOUND : blockMinLen;
|
|
12393
|
+
maxEntriesPerDocument[blockIndex] = blockMaxRun;
|
|
12394
|
+
documentCount[blockIndex] = blockDocs;
|
|
12395
|
+
blockIndex++;
|
|
12396
|
+
blockStart = entry;
|
|
12397
|
+
blockMaxTf = 0;
|
|
12398
|
+
blockMinLen = Number.POSITIVE_INFINITY;
|
|
12399
|
+
blockMaxRun = 0;
|
|
12400
|
+
blockDocs = 0;
|
|
12386
12401
|
}
|
|
12387
12402
|
}
|
|
12388
|
-
|
|
12389
|
-
|
|
12390
|
-
|
|
12391
|
-
|
|
12392
|
-
|
|
12393
|
-
|
|
12394
|
-
|
|
12395
|
-
|
|
12396
|
-
|
|
12397
|
-
|
|
12398
|
-
|
|
12399
|
-
|
|
12403
|
+
return {
|
|
12404
|
+
structureRevision: list.structureRevision,
|
|
12405
|
+
blockCount: blockIndex,
|
|
12406
|
+
entryEnd,
|
|
12407
|
+
maxTermFrequency,
|
|
12408
|
+
minFieldLength,
|
|
12409
|
+
maxEntriesPerDocument,
|
|
12410
|
+
documentCount
|
|
12411
|
+
};
|
|
12412
|
+
}
|
|
12413
|
+
function blockBoundsFor(list, columns2) {
|
|
12414
|
+
const existing = boundsByList.get(list);
|
|
12415
|
+
if (existing !== void 0 && existing.structureRevision === list.structureRevision) {
|
|
12416
|
+
const covered = entriesCovered(existing);
|
|
12417
|
+
if (covered === list.length) return existing;
|
|
12418
|
+
if (list.ordered && covered < list.length) {
|
|
12419
|
+
const extended = scanPostingBlocks(list, columns2, existing, Math.max(0, existing.blockCount - 1));
|
|
12420
|
+
boundsByList.set(list, extended);
|
|
12421
|
+
return extended;
|
|
12422
|
+
}
|
|
12400
12423
|
}
|
|
12401
|
-
|
|
12424
|
+
const stored = list.storedBlockBounds?.() ?? null;
|
|
12425
|
+
const built2 = stored === null ? scanPostingBlocks(list, columns2, null, 0) : { ...stored, structureRevision: list.structureRevision };
|
|
12426
|
+
boundsByList.set(list, built2);
|
|
12427
|
+
return built2;
|
|
12402
12428
|
}
|
|
12403
12429
|
|
|
12404
12430
|
// src/core/partition/multi-term-topk.ts
|
|
@@ -13043,10 +13069,14 @@ function searchFulltext(state, params) {
|
|
|
13043
13069
|
}
|
|
13044
13070
|
function scoreGroup(group, only) {
|
|
13045
13071
|
if (group === null) return;
|
|
13046
|
-
|
|
13047
|
-
|
|
13048
|
-
|
|
13072
|
+
if (components !== null) {
|
|
13073
|
+
for (const [internalId, contribution] of computeGroupContributions(groupContext, group)) {
|
|
13074
|
+
if (only !== null && !bitsetHas(only, internalId)) continue;
|
|
13075
|
+
mergeGroupContribution(internalId, contribution);
|
|
13076
|
+
}
|
|
13077
|
+
return;
|
|
13049
13078
|
}
|
|
13079
|
+
addGroupScores(groupContext, group, scoreBuffer, only, coverage);
|
|
13050
13080
|
}
|
|
13051
13081
|
const prunableList = phrases === void 0 ? prunableSingleTermList(params, state.invertedIdx, fieldNames) : null;
|
|
13052
13082
|
if (prunableList !== null && maxResults !== void 0) {
|
|
@@ -17120,11 +17150,13 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
17120
17150
|
isTombstoned: (ordinal) => tombstones.has(ordinal),
|
|
17121
17151
|
settleRemovals: counts.settleRemovals
|
|
17122
17152
|
});
|
|
17123
|
-
const
|
|
17124
|
-
|
|
17125
|
-
|
|
17126
|
-
|
|
17153
|
+
const textFieldsOnly = {
|
|
17154
|
+
kind: "fields",
|
|
17155
|
+
include: source.fieldLengthNames.map((fieldPath) => fieldPath.split(".")),
|
|
17156
|
+
exclude: []
|
|
17127
17157
|
};
|
|
17158
|
+
const textAt2 = (ordinal) => documentSource.readAt(ordinal, textFieldsOnly);
|
|
17159
|
+
const textDocuments = { count: documentSource.count, textFields: source.fieldLengthNames, documentAt: textAt2 };
|
|
17128
17160
|
const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
|
|
17129
17161
|
const surfaces = createFrozenSurfaceSlot(
|
|
17130
17162
|
loadSurfaceForms,
|
|
@@ -17196,7 +17228,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
17196
17228
|
source.fieldNames,
|
|
17197
17229
|
source.fieldLengthNames,
|
|
17198
17230
|
ordinal,
|
|
17199
|
-
|
|
17231
|
+
textAt2(ordinal),
|
|
17200
17232
|
analysis,
|
|
17201
17233
|
surfaces,
|
|
17202
17234
|
partialWords
|
|
@@ -18140,11 +18172,12 @@ function bindSegmentCodecHost(_host, _msgpack) {
|
|
|
18140
18172
|
// src/serialization/msgpack-without-fields.ts
|
|
18141
18173
|
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
18142
18174
|
var keyDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
18175
|
+
var walkedHeader = { headerBytes: 0, bodyBytes: 0, children: 0 };
|
|
18143
18176
|
function endOfValue(view, start) {
|
|
18144
18177
|
let offset = start;
|
|
18145
18178
|
let pending = 1;
|
|
18146
18179
|
while (pending > 0) {
|
|
18147
|
-
const header = headerAt(view, offset);
|
|
18180
|
+
const header = headerAt(view, offset, walkedHeader);
|
|
18148
18181
|
if (header === null) return null;
|
|
18149
18182
|
offset += header.headerBytes + header.bodyBytes;
|
|
18150
18183
|
if (offset > view.byteLength) return null;
|
|
@@ -18213,6 +18246,8 @@ function decodeMapWithoutFields(bytes, skippedFields) {
|
|
|
18213
18246
|
var encoder5 = new TextEncoder();
|
|
18214
18247
|
var PROTO_KEY_BYTES = encoder5.encode("__proto__");
|
|
18215
18248
|
var MISSING = { found: false, value: void 0 };
|
|
18249
|
+
var mapHeader = { headerBytes: 0, bodyBytes: 0, children: 0 };
|
|
18250
|
+
var keyHeader = { headerBytes: 0, bodyBytes: 0, children: 0 };
|
|
18216
18251
|
var pathDecoder = null;
|
|
18217
18252
|
function encodePathSegments(segments2) {
|
|
18218
18253
|
if (segments2.length === 0) return null;
|
|
@@ -18239,14 +18274,12 @@ function sameBytes(bytes, start, end, target) {
|
|
|
18239
18274
|
}
|
|
18240
18275
|
return true;
|
|
18241
18276
|
}
|
|
18242
|
-
function
|
|
18243
|
-
if (!isStringTag(view.getUint8(offset))) return
|
|
18244
|
-
|
|
18245
|
-
|
|
18246
|
-
const start = offset + header.headerBytes;
|
|
18247
|
-
return { start, end: start + header.bodyBytes };
|
|
18277
|
+
function keyEndAt(view, offset) {
|
|
18278
|
+
if (!isStringTag(view.getUint8(offset))) return -1;
|
|
18279
|
+
if (headerAt(view, offset, keyHeader) === null) return -1;
|
|
18280
|
+
return offset + keyHeader.headerBytes + keyHeader.bodyBytes;
|
|
18248
18281
|
}
|
|
18249
|
-
function readEncodedPath(bytes, segments2, arrays) {
|
|
18282
|
+
function readEncodedPath(bytes, segments2, arrays, foundRead) {
|
|
18250
18283
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
18251
18284
|
let start = 0;
|
|
18252
18285
|
let end = bytes.length;
|
|
@@ -18258,20 +18291,21 @@ function readEncodedPath(bytes, segments2, arrays) {
|
|
|
18258
18291
|
if (isArrayTag(tag2)) return arrays === "missing" ? MISSING : null;
|
|
18259
18292
|
return isScalarTag(tag2) ? MISSING : null;
|
|
18260
18293
|
}
|
|
18261
|
-
|
|
18262
|
-
|
|
18263
|
-
let offset = start +
|
|
18294
|
+
if (headerAt(view, start, mapHeader) === null) return null;
|
|
18295
|
+
const pairs = mapHeader.children / 2;
|
|
18296
|
+
let offset = start + mapHeader.headerBytes;
|
|
18264
18297
|
let valueStart = -1;
|
|
18265
18298
|
let valueEnd = -1;
|
|
18266
|
-
for (let pair = 0; pair <
|
|
18299
|
+
for (let pair = 0; pair < pairs; pair++) {
|
|
18267
18300
|
if (offset >= end) return null;
|
|
18268
|
-
const
|
|
18269
|
-
if (
|
|
18270
|
-
|
|
18271
|
-
|
|
18301
|
+
const keyEnd = keyEndAt(view, offset);
|
|
18302
|
+
if (keyEnd < 0 || keyEnd > end) return null;
|
|
18303
|
+
const keyStart = offset + keyHeader.headerBytes;
|
|
18304
|
+
if (sameBytes(bytes, keyStart, keyEnd, PROTO_KEY_BYTES)) return null;
|
|
18305
|
+
const next = endOfValue(view, keyEnd);
|
|
18272
18306
|
if (next === null || next > end) return null;
|
|
18273
|
-
if (sameBytes(bytes,
|
|
18274
|
-
valueStart =
|
|
18307
|
+
if (sameBytes(bytes, keyStart, keyEnd, segments2[depth])) {
|
|
18308
|
+
valueStart = keyEnd;
|
|
18275
18309
|
valueEnd = next;
|
|
18276
18310
|
}
|
|
18277
18311
|
offset = next;
|
|
@@ -18283,7 +18317,8 @@ function readEncodedPath(bytes, segments2, arrays) {
|
|
|
18283
18317
|
}
|
|
18284
18318
|
try {
|
|
18285
18319
|
pathDecoder ??= new Decoder();
|
|
18286
|
-
|
|
18320
|
+
foundRead.value = pathDecoder.decode(bytes.subarray(start, end));
|
|
18321
|
+
return foundRead;
|
|
18287
18322
|
} catch {
|
|
18288
18323
|
return null;
|
|
18289
18324
|
}
|
|
@@ -18379,20 +18414,35 @@ function wrapEncodedDocumentTable(data) {
|
|
|
18379
18414
|
}
|
|
18380
18415
|
function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
|
|
18381
18416
|
const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
|
|
18382
|
-
const touchesDiscarded = (path) =>
|
|
18417
|
+
const touchesDiscarded = (path) => {
|
|
18418
|
+
for (const discardedPath of discarded) {
|
|
18419
|
+
if (sharesPrefix(discardedPath, path)) return true;
|
|
18420
|
+
}
|
|
18421
|
+
return false;
|
|
18422
|
+
};
|
|
18383
18423
|
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
|
|
18384
|
-
function projectedPathAt(ordinal, path) {
|
|
18424
|
+
function projectedPathAt(ordinal, path, result) {
|
|
18385
18425
|
const segments2 = encodedProjectionPathOf(path);
|
|
18386
18426
|
if (segments2 === null) return { found: false, value: void 0 };
|
|
18387
|
-
const read = touchesDiscarded(path) ? null : readEncodedPath(bytesAt(ordinal), segments2, "missing");
|
|
18427
|
+
const read = touchesDiscarded(path) ? null : readEncodedPath(bytesAt(ordinal), segments2, "missing", result);
|
|
18388
18428
|
if (read === null) return readProjectedPath(docAt(ordinal), path);
|
|
18389
|
-
|
|
18429
|
+
if (!read.found) return read;
|
|
18430
|
+
read.value = valueWithOwnBuffers(read.value);
|
|
18431
|
+
return read;
|
|
18432
|
+
}
|
|
18433
|
+
function projectedAt(ordinal, projection) {
|
|
18434
|
+
if (projection === void 0 || projection.kind === "full") return docAt(ordinal);
|
|
18435
|
+
if (projection.kind === "none") return {};
|
|
18436
|
+
if (projection.include === null) return cloneProjected(docAt(ordinal), projection);
|
|
18437
|
+
const result = { found: true, value: void 0 };
|
|
18438
|
+
return projectIncludedPaths(projection.include, projection.exclude, (path) => projectedPathAt(ordinal, path, result));
|
|
18390
18439
|
}
|
|
18391
18440
|
return {
|
|
18392
18441
|
count,
|
|
18393
18442
|
byteLength,
|
|
18394
18443
|
docAt,
|
|
18395
|
-
readAt(ordinal) {
|
|
18444
|
+
readAt(ordinal, fields) {
|
|
18445
|
+
if (fields !== void 0) return projectedAt(ordinal, fields);
|
|
18396
18446
|
return decodeStoredDocument(bytesAt(ordinal), discardedFields);
|
|
18397
18447
|
},
|
|
18398
18448
|
fieldReader(fieldPath) {
|
|
@@ -18401,16 +18451,15 @@ function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields
|
|
|
18401
18451
|
if (segments2 === null) return () => void 0;
|
|
18402
18452
|
const decodesDocument = touchesDiscarded(path);
|
|
18403
18453
|
return (ordinal) => {
|
|
18404
|
-
const read = decodesDocument ? null : readEncodedPath(bytesAt(ordinal), segments2, "decode-document");
|
|
18454
|
+
const read = decodesDocument ? null : readEncodedPath(bytesAt(ordinal), segments2, "decode-document", { found: true, value: void 0 });
|
|
18405
18455
|
if (read !== null) return valueWithOwnBuffers(read.value);
|
|
18406
18456
|
return getNestedValue(docAt(ordinal), fieldPath);
|
|
18407
18457
|
};
|
|
18408
18458
|
},
|
|
18409
|
-
projectedAt
|
|
18410
|
-
|
|
18411
|
-
|
|
18412
|
-
|
|
18413
|
-
return projectIncludedPaths(projection.include, projection.exclude, (path) => projectedPathAt(ordinal, path));
|
|
18459
|
+
projectedAt,
|
|
18460
|
+
copyForRollbackAt(ordinal) {
|
|
18461
|
+
const bytes = bytesAt(ordinal).slice();
|
|
18462
|
+
return () => documentWithOwnBuffers(decodeStoredDocument(bytes, discardedFields));
|
|
18414
18463
|
}
|
|
18415
18464
|
};
|
|
18416
18465
|
}
|
|
@@ -19946,7 +19995,7 @@ function hasLanguage(name) {
|
|
|
19946
19995
|
|
|
19947
19996
|
// src/narsil/restore-queue-code.ts
|
|
19948
19997
|
var restoreQueue = codeLoader(async () => {
|
|
19949
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
19998
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-MVJTP3E4.mjs')]);
|
|
19950
19999
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
19951
20000
|
return code7;
|
|
19952
20001
|
}, "restore");
|
|
@@ -20511,6 +20560,32 @@ function createSharedFrozenSegment(snapshot, survivingSurfaceForms) {
|
|
|
20511
20560
|
);
|
|
20512
20561
|
}
|
|
20513
20562
|
|
|
20563
|
+
// src/partitioning/manager/nested-values.ts
|
|
20564
|
+
function setNestedValue(obj, path, value) {
|
|
20565
|
+
if (!path.includes(".")) {
|
|
20566
|
+
obj[path] = value;
|
|
20567
|
+
return;
|
|
20568
|
+
}
|
|
20569
|
+
const segments2 = path.split(".");
|
|
20570
|
+
let current = obj;
|
|
20571
|
+
for (let i = 0; i < segments2.length - 1; i++) {
|
|
20572
|
+
let next = current[segments2[i]];
|
|
20573
|
+
if (next === null || next === void 0 || typeof next !== "object") {
|
|
20574
|
+
next = {};
|
|
20575
|
+
current[segments2[i]] = next;
|
|
20576
|
+
}
|
|
20577
|
+
current = next;
|
|
20578
|
+
}
|
|
20579
|
+
current[segments2[segments2.length - 1]] = value;
|
|
20580
|
+
}
|
|
20581
|
+
function copyVectorsInto(document, docId, vectorIndexes, projection) {
|
|
20582
|
+
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
20583
|
+
if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
|
|
20584
|
+
const vector = vectorIndex.getVector(docId);
|
|
20585
|
+
if (vector) setNestedValue(document, fieldPath, vector);
|
|
20586
|
+
}
|
|
20587
|
+
}
|
|
20588
|
+
|
|
20514
20589
|
// src/sync/epoch.ts
|
|
20515
20590
|
var SYNC_EPOCH_BYTES = 16;
|
|
20516
20591
|
var SYNC_EPOCH_PATTERN = /^[0-9a-f]{32}$/;
|
|
@@ -20548,7 +20623,7 @@ var loadEmbedCode = embedCode.load;
|
|
|
20548
20623
|
|
|
20549
20624
|
// src/engine/optional-writes-code.ts
|
|
20550
20625
|
var optionalWrites = codeLoader(async () => {
|
|
20551
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
20626
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-MGFFGKYQ.mjs')]);
|
|
20552
20627
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
20553
20628
|
return code7;
|
|
20554
20629
|
}, "write");
|
|
@@ -21012,6 +21087,51 @@ async function oneAtATime(core2, indexName, action) {
|
|
|
21012
21087
|
function bindOptionalWritesHost(_host, _segmentCodec) {
|
|
21013
21088
|
}
|
|
21014
21089
|
|
|
21090
|
+
// src/engine/mutations/batch-shutdown.ts
|
|
21091
|
+
function isFailureThatShutdownCauses(code7) {
|
|
21092
|
+
return code7 === ErrorCodes.INSTANCE_SHUT_DOWN || code7 === ErrorCodes.INDEX_NOT_FOUND || code7 === ErrorCodes.EMBEDDING_FAILED;
|
|
21093
|
+
}
|
|
21094
|
+
function shutDownStarted(ctx) {
|
|
21095
|
+
return ctx.abortController.signal.aborted;
|
|
21096
|
+
}
|
|
21097
|
+
function shutDownError() {
|
|
21098
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
21099
|
+
}
|
|
21100
|
+
function causedByShutdown(ctx, err) {
|
|
21101
|
+
const { signal } = ctx.abortController;
|
|
21102
|
+
if (!signal.aborted) return false;
|
|
21103
|
+
if (err === signal.reason) return true;
|
|
21104
|
+
if (err instanceof NarsilError) return isFailureThatShutdownCauses(err.code);
|
|
21105
|
+
return err instanceof Error && err.name === "AbortError";
|
|
21106
|
+
}
|
|
21107
|
+
function batchFailureOf(ctx, err, fallbackCode) {
|
|
21108
|
+
if (causedByShutdown(ctx, err)) {
|
|
21109
|
+
return err instanceof NarsilError && err.code === ErrorCodes.INSTANCE_SHUT_DOWN ? err : shutDownError();
|
|
21110
|
+
}
|
|
21111
|
+
return err instanceof NarsilError ? err : new NarsilError(fallbackCode, String(err));
|
|
21112
|
+
}
|
|
21113
|
+
function docIdOf(entry) {
|
|
21114
|
+
return entry.docId;
|
|
21115
|
+
}
|
|
21116
|
+
function failAsShutDown(failed, entries, reportedDocIdOf2, start = 0, end = entries.length) {
|
|
21117
|
+
if (start >= end) return;
|
|
21118
|
+
const error = shutDownError();
|
|
21119
|
+
for (let i = start; i < end; i++) failed.push({ docId: reportedDocIdOf2(entries[i]), error });
|
|
21120
|
+
}
|
|
21121
|
+
function applyUnlessShutDown(ctx, apply) {
|
|
21122
|
+
return () => {
|
|
21123
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
21124
|
+
return apply();
|
|
21125
|
+
};
|
|
21126
|
+
}
|
|
21127
|
+
async function unlessShutDown(ctx, work) {
|
|
21128
|
+
try {
|
|
21129
|
+
await work();
|
|
21130
|
+
} catch (err) {
|
|
21131
|
+
if (!shutDownStarted(ctx)) throw err;
|
|
21132
|
+
}
|
|
21133
|
+
}
|
|
21134
|
+
|
|
21015
21135
|
// src/engine/mutations/batch-admissions.ts
|
|
21016
21136
|
function createPendingAdmissions() {
|
|
21017
21137
|
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
@@ -21022,8 +21142,11 @@ function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
|
21022
21142
|
const partitionId = manager.routePartition(docId);
|
|
21023
21143
|
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
21024
21144
|
}
|
|
21025
|
-
function asBatchInsertError(err) {
|
|
21026
|
-
return
|
|
21145
|
+
function asBatchInsertError(ctx, err) {
|
|
21146
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_VALIDATION_FAILED);
|
|
21147
|
+
}
|
|
21148
|
+
function providedDocIdOrEmpty(document) {
|
|
21149
|
+
return providedDocId(document) ?? "";
|
|
21027
21150
|
}
|
|
21028
21151
|
|
|
21029
21152
|
// src/engine/mutations/outside-hold.ts
|
|
@@ -21253,7 +21376,11 @@ async function prepareInsertDocuments(ctx, indexName, documents2, start, end, ho
|
|
|
21253
21376
|
const pending = [];
|
|
21254
21377
|
for (let i = start; i < end; i++) {
|
|
21255
21378
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
21256
|
-
if (ctx
|
|
21379
|
+
if (shutDownStarted(ctx)) {
|
|
21380
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
21381
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty, i, end);
|
|
21382
|
+
return null;
|
|
21383
|
+
}
|
|
21257
21384
|
const document = documents2[i];
|
|
21258
21385
|
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
21259
21386
|
try {
|
|
@@ -21262,17 +21389,26 @@ async function prepareInsertDocuments(ctx, indexName, documents2, start, end, ho
|
|
|
21262
21389
|
validateDocId(docId);
|
|
21263
21390
|
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
21264
21391
|
} catch (err) {
|
|
21265
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
21392
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
21266
21393
|
}
|
|
21267
21394
|
}
|
|
21268
|
-
|
|
21269
|
-
|
|
21270
|
-
|
|
21271
|
-
|
|
21272
|
-
|
|
21273
|
-
|
|
21274
|
-
|
|
21275
|
-
|
|
21395
|
+
let prepared;
|
|
21396
|
+
try {
|
|
21397
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
21398
|
+
prepared = await prepareDocumentWrites(
|
|
21399
|
+
ctx,
|
|
21400
|
+
indexName,
|
|
21401
|
+
pending.map((item) => item.preparation),
|
|
21402
|
+
hold,
|
|
21403
|
+
true,
|
|
21404
|
+
earlier
|
|
21405
|
+
);
|
|
21406
|
+
} catch (err) {
|
|
21407
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
21408
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
21409
|
+
return null;
|
|
21410
|
+
}
|
|
21411
|
+
const { entry, embedding } = prepared;
|
|
21276
21412
|
const candidates2 = [];
|
|
21277
21413
|
for (const item of pending) {
|
|
21278
21414
|
const { document, failure } = item.preparation;
|
|
@@ -21280,22 +21416,19 @@ async function prepareInsertDocuments(ctx, indexName, documents2, start, end, ho
|
|
|
21280
21416
|
candidates2.push({ docId: item.docId, document });
|
|
21281
21417
|
continue;
|
|
21282
21418
|
}
|
|
21283
|
-
const docId = item.refusedByHook ? item.docId :
|
|
21284
|
-
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
21419
|
+
const docId = item.refusedByHook ? item.docId : providedDocIdOrEmpty(document);
|
|
21420
|
+
failed.push({ docId, error: asBatchInsertError(ctx, failure) });
|
|
21285
21421
|
}
|
|
21286
21422
|
return { entry, candidates: candidates2, embedding };
|
|
21287
21423
|
}
|
|
21288
21424
|
async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
21289
|
-
const
|
|
21290
|
-
|
|
21291
|
-
|
|
21292
|
-
|
|
21293
|
-
|
|
21294
|
-
|
|
21295
|
-
|
|
21296
|
-
true,
|
|
21297
|
-
failed
|
|
21298
|
-
);
|
|
21425
|
+
const prepared = await prepareInsertDocuments(ctx, indexName, documents2, 0, documents2.length, hold, true, failed);
|
|
21426
|
+
if (prepared === null) return { admitted: [], embedding: null };
|
|
21427
|
+
const { entry, candidates: candidates2, embedding } = prepared;
|
|
21428
|
+
if (shutDownStarted(ctx)) {
|
|
21429
|
+
failAsShutDown(failed, candidates2, docIdOf);
|
|
21430
|
+
return { admitted: [], embedding };
|
|
21431
|
+
}
|
|
21299
21432
|
const manager = ctx.requireManager(indexName);
|
|
21300
21433
|
const vecIndexes = manager.getVectorIndexes();
|
|
21301
21434
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
@@ -21305,7 +21438,11 @@ async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
|
21305
21438
|
const pending = createPendingAdmissions();
|
|
21306
21439
|
for (let position = 0; position < candidates2.length; position++) {
|
|
21307
21440
|
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
21308
|
-
if (ctx
|
|
21441
|
+
if (shutDownStarted(ctx)) {
|
|
21442
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
21443
|
+
failAsShutDown(failed, candidates2, docIdOf, position);
|
|
21444
|
+
return { admitted: [], embedding };
|
|
21445
|
+
}
|
|
21309
21446
|
const { docId, document } = candidates2[position];
|
|
21310
21447
|
try {
|
|
21311
21448
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
@@ -21327,7 +21464,7 @@ async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
|
21327
21464
|
recordPendingAdmission(ctx, pending, manager, docId);
|
|
21328
21465
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
21329
21466
|
} catch (err) {
|
|
21330
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
21467
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
21331
21468
|
}
|
|
21332
21469
|
}
|
|
21333
21470
|
return { admitted, embedding };
|
|
@@ -21392,7 +21529,7 @@ async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
21392
21529
|
await manager.publication.write(
|
|
21393
21530
|
() => changeIndexOutsideExternalWaits(async () => {
|
|
21394
21531
|
if (manager.has(docId)) return;
|
|
21395
|
-
const previous = prepareDocumentVectors(restoreDoc, ctx.requireIndex(indexName).vectorFieldPaths);
|
|
21532
|
+
const previous = prepareDocumentVectors(restoreDoc(), ctx.requireIndex(indexName).vectorFieldPaths);
|
|
21396
21533
|
manager.insert(docId, previous.partitionDoc, {
|
|
21397
21534
|
preparedText: { after: preparedText?.before }
|
|
21398
21535
|
});
|
|
@@ -21477,9 +21614,9 @@ async function recordChunk(ctx, indexName, mutations, applies) {
|
|
|
21477
21614
|
function applyInsertDocument(ctx, indexName, doc, options, progress) {
|
|
21478
21615
|
const manager = ctx.requireManager(indexName);
|
|
21479
21616
|
const insertIntoIndex = () => {
|
|
21480
|
-
if (ctx
|
|
21617
|
+
if (shutDownStarted(ctx)) {
|
|
21481
21618
|
progress.skipped = true;
|
|
21482
|
-
throw
|
|
21619
|
+
throw shutDownError();
|
|
21483
21620
|
}
|
|
21484
21621
|
admitInsert(ctx, indexName, manager, doc.docId);
|
|
21485
21622
|
const queued = ctx.bufferIfRebalancing(indexName, {
|
|
@@ -21535,11 +21672,11 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
21535
21672
|
const progress = docs.map(() => ({ inserted: false, buffered: false, skipped: false }));
|
|
21536
21673
|
const applies = docs.map((doc, i) => applyInsertDocument(ctx, indexName, doc, options, progress[i]));
|
|
21537
21674
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
21538
|
-
await ctx.orchestrator.settlePublications(indexName);
|
|
21675
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName));
|
|
21539
21676
|
const applications = [];
|
|
21540
21677
|
for (let i = 0; i < docs.length; i++) {
|
|
21541
21678
|
if (progress[i].skipped) {
|
|
21542
|
-
applications.push({ status: "
|
|
21679
|
+
applications.push({ status: "failed", error: shutDownError() });
|
|
21543
21680
|
continue;
|
|
21544
21681
|
}
|
|
21545
21682
|
const failure = failures[i];
|
|
@@ -21572,27 +21709,33 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
21572
21709
|
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
21573
21710
|
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
21574
21711
|
const manager = ctx.executor.getManager(indexName);
|
|
21575
|
-
await
|
|
21576
|
-
|
|
21577
|
-
|
|
21578
|
-
|
|
21579
|
-
|
|
21580
|
-
|
|
21712
|
+
await unlessShutDown(
|
|
21713
|
+
ctx,
|
|
21714
|
+
() => runOutside(hold, async () => {
|
|
21715
|
+
for (const doc of inserted) {
|
|
21716
|
+
try {
|
|
21717
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
21718
|
+
} catch (err) {
|
|
21719
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
21720
|
+
}
|
|
21581
21721
|
}
|
|
21582
|
-
}
|
|
21583
|
-
|
|
21584
|
-
return ctx.executor.getManager(indexName) !== manager;
|
|
21722
|
+
})
|
|
21723
|
+
);
|
|
21724
|
+
return shutDownStarted(ctx) || ctx.executor.getManager(indexName) !== manager;
|
|
21585
21725
|
}
|
|
21586
21726
|
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
21587
21727
|
const succeeded = [];
|
|
21588
21728
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
21589
21729
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21590
|
-
if (ctx
|
|
21730
|
+
if (shutDownStarted(ctx)) {
|
|
21731
|
+
failAsShutDown(failed, admitted, docIdOf, chunkStart);
|
|
21732
|
+
break;
|
|
21733
|
+
}
|
|
21591
21734
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
21592
21735
|
try {
|
|
21593
21736
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
21594
21737
|
} catch (err) {
|
|
21595
|
-
const error = asBatchInsertError(err);
|
|
21738
|
+
const error = asBatchInsertError(ctx, err);
|
|
21596
21739
|
for (let i = chunkStart; i < admitted.length; i++) failed.push({ docId: admitted[i].docId, error });
|
|
21597
21740
|
break;
|
|
21598
21741
|
}
|
|
@@ -21601,9 +21744,8 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
21601
21744
|
for (let i = 0; i < chunk.length; i++) {
|
|
21602
21745
|
const doc = chunk[i];
|
|
21603
21746
|
const application = applications[i];
|
|
21604
|
-
if (application.status === "skipped") continue;
|
|
21605
21747
|
if (application.status === "failed") {
|
|
21606
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
21748
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
21607
21749
|
continue;
|
|
21608
21750
|
}
|
|
21609
21751
|
succeeded.push(doc.docId);
|
|
@@ -21618,7 +21760,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
21618
21760
|
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
21619
21761
|
}
|
|
21620
21762
|
if (applied < admitted.length) {
|
|
21621
|
-
ctx.checkHeapPressure(indexName);
|
|
21763
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
21622
21764
|
await new Promise((r) => setTimeout(r, 0));
|
|
21623
21765
|
}
|
|
21624
21766
|
}
|
|
@@ -21627,7 +21769,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
21627
21769
|
|
|
21628
21770
|
// src/engine/mutations/insert-batch-chunks.ts
|
|
21629
21771
|
async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
21630
|
-
const
|
|
21772
|
+
const prepared = await prepareInsertDocuments(
|
|
21631
21773
|
ctx,
|
|
21632
21774
|
indexName,
|
|
21633
21775
|
documents2,
|
|
@@ -21638,13 +21780,24 @@ async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold
|
|
|
21638
21780
|
failed,
|
|
21639
21781
|
earlier
|
|
21640
21782
|
);
|
|
21783
|
+
if (prepared === null) return [];
|
|
21784
|
+
const { entry, candidates: candidates2 } = prepared;
|
|
21785
|
+
const admitted = [];
|
|
21786
|
+
if (shutDownStarted(ctx)) {
|
|
21787
|
+
failAsShutDown(failed, candidates2, docIdOf);
|
|
21788
|
+
return admitted;
|
|
21789
|
+
}
|
|
21641
21790
|
const manager = ctx.requireManager(indexName);
|
|
21642
21791
|
const vecIndexes = manager.getVectorIndexes();
|
|
21643
21792
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
21644
|
-
|
|
21645
|
-
for (const { docId, document } of candidates2) {
|
|
21793
|
+
for (let position = 0; position < candidates2.length; position++) {
|
|
21646
21794
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
21647
|
-
if (ctx
|
|
21795
|
+
if (shutDownStarted(ctx)) {
|
|
21796
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
21797
|
+
failAsShutDown(failed, candidates2, docIdOf, position);
|
|
21798
|
+
return [];
|
|
21799
|
+
}
|
|
21800
|
+
const { docId, document } = candidates2[position];
|
|
21648
21801
|
try {
|
|
21649
21802
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
21650
21803
|
document,
|
|
@@ -21656,7 +21809,7 @@ async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold
|
|
|
21656
21809
|
admitInsert(ctx, indexName, manager, docId);
|
|
21657
21810
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
21658
21811
|
} catch (err) {
|
|
21659
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
21812
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
21660
21813
|
}
|
|
21661
21814
|
}
|
|
21662
21815
|
return admitted;
|
|
@@ -21665,17 +21818,25 @@ async function insertInChunks(ctx, indexName, documents2, options, hold, hooked,
|
|
|
21665
21818
|
const succeeded = [];
|
|
21666
21819
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
21667
21820
|
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21668
|
-
if (ctx
|
|
21821
|
+
if (shutDownStarted(ctx)) {
|
|
21822
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty, chunkStart);
|
|
21823
|
+
break;
|
|
21824
|
+
}
|
|
21669
21825
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
21670
21826
|
const earlier = embeddingOfDocuments(embedded2, documents2.slice(chunkStart, chunkEnd));
|
|
21671
21827
|
const prepared = await admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
21828
|
+
if (shutDownStarted(ctx)) {
|
|
21829
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
21830
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty, chunkEnd);
|
|
21831
|
+
break;
|
|
21832
|
+
}
|
|
21672
21833
|
try {
|
|
21673
21834
|
if (prepared.length > 0) ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
21674
21835
|
} catch (err) {
|
|
21675
|
-
const error = asBatchInsertError(err);
|
|
21836
|
+
const error = asBatchInsertError(ctx, err);
|
|
21676
21837
|
for (const doc of prepared) failed.push({ docId: doc.docId, error });
|
|
21677
21838
|
for (let i = chunkEnd; i < documents2.length; i++) {
|
|
21678
|
-
failed.push({ docId:
|
|
21839
|
+
failed.push({ docId: providedDocIdOrEmpty(documents2[i]), error });
|
|
21679
21840
|
}
|
|
21680
21841
|
break;
|
|
21681
21842
|
}
|
|
@@ -21685,9 +21846,8 @@ async function insertInChunks(ctx, indexName, documents2, options, hold, hooked,
|
|
|
21685
21846
|
for (let i = 0; i < prepared.length; i++) {
|
|
21686
21847
|
const doc = prepared[i];
|
|
21687
21848
|
const application = applications[i];
|
|
21688
|
-
if (application.status === "skipped") continue;
|
|
21689
21849
|
if (application.status === "failed") {
|
|
21690
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
21850
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
21691
21851
|
continue;
|
|
21692
21852
|
}
|
|
21693
21853
|
succeeded.push(doc.docId);
|
|
@@ -21698,21 +21858,24 @@ async function insertInChunks(ctx, indexName, documents2, options, hold, hooked,
|
|
|
21698
21858
|
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
21699
21859
|
}
|
|
21700
21860
|
if (chunkEnd < documents2.length) {
|
|
21701
|
-
ctx.checkHeapPressure(indexName);
|
|
21861
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
21702
21862
|
await new Promise((r) => setTimeout(r, 0));
|
|
21703
21863
|
}
|
|
21704
21864
|
}
|
|
21705
21865
|
return { succeeded, touchedVectorFields };
|
|
21706
21866
|
}
|
|
21707
21867
|
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
21868
|
+
if (shutDownStarted(ctx)) return;
|
|
21708
21869
|
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
21709
21870
|
for (const fieldPath of inserts.touchedVectorFields) {
|
|
21710
21871
|
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
21711
21872
|
}
|
|
21712
21873
|
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
21713
21874
|
ctx.checkHeapPressure(indexName);
|
|
21714
|
-
await ctx
|
|
21715
|
-
|
|
21875
|
+
await unlessShutDown(ctx, async () => {
|
|
21876
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
21877
|
+
if (options?.wait === true && !shutDownStarted(ctx)) await awaitWriteVisibility(ctx, indexName);
|
|
21878
|
+
});
|
|
21716
21879
|
}
|
|
21717
21880
|
|
|
21718
21881
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -21721,7 +21884,7 @@ function applySegment(ctx, indexName, attachment, doc, options) {
|
|
|
21721
21884
|
const publish2 = async () => {
|
|
21722
21885
|
if (attachment.failure !== void 0) throw attachment.failure;
|
|
21723
21886
|
if (attachment.attached) return;
|
|
21724
|
-
if (ctx
|
|
21887
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
21725
21888
|
if (ctx.isRebalancing(indexName) || manager.partitionCount !== attachment.partitionCount) {
|
|
21726
21889
|
attachment.individual = true;
|
|
21727
21890
|
}
|
|
@@ -21875,6 +22038,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21875
22038
|
for (let index = 0; index < chunks.length; index++) {
|
|
21876
22039
|
const { docs, attachmentOf } = chunks[index];
|
|
21877
22040
|
try {
|
|
22041
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
21878
22042
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
21879
22043
|
} catch (error) {
|
|
21880
22044
|
for (const later of chunks.slice(index)) {
|
|
@@ -21888,7 +22052,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21888
22052
|
return applySegment(ctx, indexName, attachment, doc, options);
|
|
21889
22053
|
});
|
|
21890
22054
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
21891
|
-
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
22055
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName, "segments"));
|
|
21892
22056
|
const inserted = [];
|
|
21893
22057
|
for (let i = 0; i < docs.length; i++) {
|
|
21894
22058
|
const doc = docs[i];
|
|
@@ -21929,7 +22093,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21929
22093
|
break;
|
|
21930
22094
|
}
|
|
21931
22095
|
if (index + 1 < chunks.length) {
|
|
21932
|
-
ctx.checkHeapPressure(indexName);
|
|
22096
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
21933
22097
|
await new Promise((r) => setTimeout(r, 0));
|
|
21934
22098
|
}
|
|
21935
22099
|
}
|
|
@@ -21940,7 +22104,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21940
22104
|
continue;
|
|
21941
22105
|
}
|
|
21942
22106
|
if (!errorOf.has(doc.docId)) continue;
|
|
21943
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
22107
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, errorOf.get(doc.docId)) });
|
|
21944
22108
|
}
|
|
21945
22109
|
return { succeeded, touchedVectorFields, unrecorded };
|
|
21946
22110
|
}
|
|
@@ -21954,8 +22118,11 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
21954
22118
|
);
|
|
21955
22119
|
}
|
|
21956
22120
|
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
21957
|
-
|
|
21958
|
-
|
|
22121
|
+
const nothingIngested = () => ({ succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] });
|
|
22122
|
+
if (admitted.length === 0) return nothingIngested();
|
|
22123
|
+
if (shutDownStarted(ctx)) {
|
|
22124
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
22125
|
+
return nothingIngested();
|
|
21959
22126
|
}
|
|
21960
22127
|
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
21961
22128
|
const manager = ctx.requireManager(indexName);
|
|
@@ -21973,6 +22140,10 @@ async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
|
21973
22140
|
} catch {
|
|
21974
22141
|
built2 = null;
|
|
21975
22142
|
}
|
|
22143
|
+
if (shutDownStarted(ctx)) {
|
|
22144
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
22145
|
+
return nothingIngested();
|
|
22146
|
+
}
|
|
21976
22147
|
if (built2 === null) {
|
|
21977
22148
|
return applyIndividually();
|
|
21978
22149
|
}
|
|
@@ -22023,7 +22194,9 @@ async function insertBatchViaSegments(ctx, indexName, documents2, options, hold)
|
|
|
22023
22194
|
const failed = [];
|
|
22024
22195
|
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents2, failed, hold);
|
|
22025
22196
|
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
22026
|
-
if (
|
|
22197
|
+
if (shutDownStarted(ctx)) {
|
|
22198
|
+
failAsShutDown(failed, outcome.unrecorded, docIdOf);
|
|
22199
|
+
} else if (outcome.unrecorded.length > 0) {
|
|
22027
22200
|
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
22028
22201
|
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
22029
22202
|
outcome.succeeded.push(...rest.succeeded);
|
|
@@ -22038,20 +22211,54 @@ async function insertDocumentBatch(ctx, indexName, documents2, options, hold) {
|
|
|
22038
22211
|
ctx.guardShutdown();
|
|
22039
22212
|
ctx.requireIndex(indexName);
|
|
22040
22213
|
validateBatchList(documents2, "documents");
|
|
22041
|
-
|
|
22214
|
+
const failed = [];
|
|
22215
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.scaleOutBeforeBatch(indexName, documents2.length));
|
|
22216
|
+
if (shutDownStarted(ctx)) {
|
|
22217
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty);
|
|
22218
|
+
return { succeeded: [], failed };
|
|
22219
|
+
}
|
|
22042
22220
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
22043
22221
|
if (documents2.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
22044
22222
|
return insertBatchViaSegments(ctx, indexName, documents2, options, hold);
|
|
22045
22223
|
}
|
|
22046
|
-
const failed = [];
|
|
22047
22224
|
const inserts = await insertInChunks(ctx, indexName, documents2, options, hold, true, failed);
|
|
22048
22225
|
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
22049
22226
|
return { succeeded: inserts.succeeded, failed };
|
|
22050
22227
|
}
|
|
22051
22228
|
|
|
22229
|
+
// src/engine/mutations/rollback-copy.ts
|
|
22230
|
+
function frozenRollbackCopy(manager, docId) {
|
|
22231
|
+
const partitionId = manager.partitionIdOf(docId);
|
|
22232
|
+
if (partitionId === void 0) return void 0;
|
|
22233
|
+
const partition = manager.getPartition(partitionId);
|
|
22234
|
+
if (!isCompositePartition(partition)) return void 0;
|
|
22235
|
+
for (const segment of partition.frozenSegments()) {
|
|
22236
|
+
const ordinal = segment.docStore.getInternalId(docId);
|
|
22237
|
+
if (ordinal !== void 0) return segment.documentSource.copyForRollbackAt?.(ordinal);
|
|
22238
|
+
}
|
|
22239
|
+
return void 0;
|
|
22240
|
+
}
|
|
22241
|
+
function rollbackCopyOf(manager, docId) {
|
|
22242
|
+
const frozenCopy = frozenRollbackCopy(manager, docId);
|
|
22243
|
+
if (frozenCopy === void 0) {
|
|
22244
|
+
const document = manager.get(docId);
|
|
22245
|
+
return document === void 0 ? void 0 : () => document;
|
|
22246
|
+
}
|
|
22247
|
+
const vectors = [];
|
|
22248
|
+
for (const [fieldPath, vectorIndex] of manager.getVectorIndexes()) {
|
|
22249
|
+
const vector = vectorIndex.getVector(docId);
|
|
22250
|
+
if (vector) vectors.push([fieldPath, vector]);
|
|
22251
|
+
}
|
|
22252
|
+
return () => {
|
|
22253
|
+
const document = frozenCopy();
|
|
22254
|
+
for (const [fieldPath, vector] of vectors) setNestedValue(document, fieldPath, vector);
|
|
22255
|
+
return document;
|
|
22256
|
+
};
|
|
22257
|
+
}
|
|
22258
|
+
|
|
22052
22259
|
// src/engine/mutations/remove.ts
|
|
22053
|
-
function asRemoveError(err) {
|
|
22054
|
-
return
|
|
22260
|
+
function asRemoveError(ctx, err) {
|
|
22261
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
22055
22262
|
}
|
|
22056
22263
|
function validateRemoval(ctx, docId) {
|
|
22057
22264
|
ctx.guardShutdown();
|
|
@@ -22087,7 +22294,7 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
22087
22294
|
prepared.buffered = true;
|
|
22088
22295
|
return queued;
|
|
22089
22296
|
}
|
|
22090
|
-
prepared.restoreDoc = ctx.durability ? manager
|
|
22297
|
+
prepared.restoreDoc = ctx.durability ? rollbackCopyOf(manager, docId) : void 0;
|
|
22091
22298
|
if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
|
|
22092
22299
|
manager.remove(docId, void 0, {
|
|
22093
22300
|
onRemovedSpellings: (spellings) => {
|
|
@@ -22159,18 +22366,31 @@ async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, h
|
|
|
22159
22366
|
}
|
|
22160
22367
|
}
|
|
22161
22368
|
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
22162
|
-
await
|
|
22163
|
-
|
|
22164
|
-
|
|
22165
|
-
|
|
22166
|
-
|
|
22167
|
-
|
|
22168
|
-
|
|
22369
|
+
await unlessShutDown(
|
|
22370
|
+
ctx,
|
|
22371
|
+
() => runOutside(hold, async () => {
|
|
22372
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22373
|
+
if (refused.has(i)) continue;
|
|
22374
|
+
try {
|
|
22375
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
22376
|
+
} catch (err) {
|
|
22377
|
+
refused.set(i, err);
|
|
22378
|
+
}
|
|
22169
22379
|
}
|
|
22170
|
-
}
|
|
22171
|
-
|
|
22380
|
+
})
|
|
22381
|
+
);
|
|
22172
22382
|
return refused;
|
|
22173
22383
|
}
|
|
22384
|
+
function failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused) {
|
|
22385
|
+
const error = shutDownError();
|
|
22386
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22387
|
+
failed.push({ docId: docIds[i], error: refused.has(i) ? asRemoveError(ctx, refused.get(i)) : error });
|
|
22388
|
+
}
|
|
22389
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
22390
|
+
}
|
|
22391
|
+
function removalDocIdOf(docId) {
|
|
22392
|
+
return docId;
|
|
22393
|
+
}
|
|
22174
22394
|
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
22175
22395
|
ctx.guardShutdown();
|
|
22176
22396
|
ctx.requireIndex(indexName);
|
|
@@ -22182,8 +22402,16 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
22182
22402
|
let manager = first;
|
|
22183
22403
|
try {
|
|
22184
22404
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22405
|
+
if (shutDownStarted(ctx)) {
|
|
22406
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkStart);
|
|
22407
|
+
break;
|
|
22408
|
+
}
|
|
22185
22409
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
22186
22410
|
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
22411
|
+
if (shutDownStarted(ctx)) {
|
|
22412
|
+
failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused);
|
|
22413
|
+
break;
|
|
22414
|
+
}
|
|
22187
22415
|
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
22188
22416
|
const removed = [];
|
|
22189
22417
|
const prepared = [];
|
|
@@ -22193,10 +22421,15 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
22193
22421
|
if (refused.has(i)) throw refused.get(i);
|
|
22194
22422
|
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
22195
22423
|
} catch (err) {
|
|
22196
|
-
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
22424
|
+
failed.push({ docId: docIds[i], error: asRemoveError(ctx, err) });
|
|
22197
22425
|
}
|
|
22198
22426
|
}
|
|
22199
|
-
|
|
22427
|
+
if (shutDownStarted(ctx)) {
|
|
22428
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
22429
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
22430
|
+
break;
|
|
22431
|
+
}
|
|
22432
|
+
const applies = prepared.map((removal) => applyUnlessShutDown(ctx, applyOfRemoval(ctx, indexName, removal)));
|
|
22200
22433
|
try {
|
|
22201
22434
|
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
22202
22435
|
for (let i = 0; i < prepared.length; i++) {
|
|
@@ -22217,7 +22450,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
22217
22450
|
error = rollbackError;
|
|
22218
22451
|
}
|
|
22219
22452
|
removal.releaseVectorHolds?.();
|
|
22220
|
-
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
22453
|
+
failed.push({ docId: removal.docId, error: asRemoveError(ctx, error) });
|
|
22221
22454
|
continue;
|
|
22222
22455
|
}
|
|
22223
22456
|
removal.releaseVectorHolds?.();
|
|
@@ -22227,18 +22460,19 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
22227
22460
|
} finally {
|
|
22228
22461
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
22229
22462
|
}
|
|
22230
|
-
await settleRemovals(ctx, indexName, removed, hold);
|
|
22231
|
-
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
22463
|
+
await unlessShutDown(ctx, () => settleRemovals(ctx, indexName, removed, hold));
|
|
22464
|
+
if (!shutDownStarted(ctx)) manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
22232
22465
|
if (chunkEnd < docIds.length) {
|
|
22233
22466
|
await new Promise((r) => setTimeout(r, 0));
|
|
22234
22467
|
}
|
|
22235
22468
|
}
|
|
22236
22469
|
} finally {
|
|
22237
22470
|
const ending = manager;
|
|
22238
|
-
await ending?.publication.write(() => ending.endBatchRemove());
|
|
22471
|
+
await unlessShutDown(ctx, () => ending?.publication.write(() => ending.endBatchRemove()));
|
|
22239
22472
|
}
|
|
22473
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
22240
22474
|
ctx.checkHeapPressure(indexName);
|
|
22241
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22475
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
22242
22476
|
return { succeeded, failed };
|
|
22243
22477
|
}
|
|
22244
22478
|
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
@@ -22278,8 +22512,8 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
22278
22512
|
}
|
|
22279
22513
|
return { partitionDoc };
|
|
22280
22514
|
}
|
|
22281
|
-
function asUpdateError(err) {
|
|
22282
|
-
return
|
|
22515
|
+
function asUpdateError(ctx, err) {
|
|
22516
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
22283
22517
|
}
|
|
22284
22518
|
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
22285
22519
|
for (const update of updates) {
|
|
@@ -22476,24 +22710,39 @@ async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold,
|
|
|
22476
22710
|
const { document } = requireUpdateEntry(update);
|
|
22477
22711
|
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
22478
22712
|
} catch (err) {
|
|
22479
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
22713
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
22480
22714
|
}
|
|
22481
22715
|
}
|
|
22482
|
-
|
|
22483
|
-
|
|
22484
|
-
|
|
22485
|
-
|
|
22486
|
-
|
|
22487
|
-
|
|
22488
|
-
|
|
22716
|
+
let entry;
|
|
22717
|
+
try {
|
|
22718
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
22719
|
+
const writes = await prepareDocumentWrites(
|
|
22720
|
+
ctx,
|
|
22721
|
+
indexName,
|
|
22722
|
+
preparations.map((pending) => pending.preparation),
|
|
22723
|
+
hold,
|
|
22724
|
+
false
|
|
22725
|
+
);
|
|
22726
|
+
entry = writes.entry;
|
|
22727
|
+
} catch (err) {
|
|
22728
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
22729
|
+
failAsShutDown(failed, preparations, docIdOf);
|
|
22730
|
+
return [];
|
|
22731
|
+
}
|
|
22489
22732
|
const prepared = [];
|
|
22490
|
-
for (
|
|
22733
|
+
for (let position = 0; position < preparations.length; position++) {
|
|
22491
22734
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
22735
|
+
if (shutDownStarted(ctx)) {
|
|
22736
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
22737
|
+
failAsShutDown(failed, preparations, docIdOf, position);
|
|
22738
|
+
return [];
|
|
22739
|
+
}
|
|
22740
|
+
const { docId, preparation } = preparations[position];
|
|
22492
22741
|
try {
|
|
22493
22742
|
if (preparation.failure !== void 0) throw preparation.failure;
|
|
22494
22743
|
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
22495
22744
|
} catch (err) {
|
|
22496
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
22745
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
22497
22746
|
}
|
|
22498
22747
|
}
|
|
22499
22748
|
return prepared;
|
|
@@ -22506,17 +22755,26 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
22506
22755
|
const failed = [];
|
|
22507
22756
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
22508
22757
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22758
|
+
if (shutDownStarted(ctx)) {
|
|
22759
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkStart);
|
|
22760
|
+
break;
|
|
22761
|
+
}
|
|
22509
22762
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
22510
22763
|
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
22764
|
+
if (shutDownStarted(ctx)) {
|
|
22765
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
22766
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkEnd);
|
|
22767
|
+
break;
|
|
22768
|
+
}
|
|
22511
22769
|
try {
|
|
22512
22770
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
22513
22771
|
} catch (err) {
|
|
22514
|
-
const error = asUpdateError(err);
|
|
22772
|
+
const error = asUpdateError(ctx, err);
|
|
22515
22773
|
for (const update of prepared) failed.push({ docId: update.docId, error });
|
|
22516
22774
|
for (let i = chunkEnd; i < updates.length; i++) failed.push({ docId: reportedDocIdOf(updates[i]), error });
|
|
22517
22775
|
break;
|
|
22518
22776
|
}
|
|
22519
|
-
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
22777
|
+
const applies = prepared.map((update) => applyUnlessShutDown(ctx, applyOfUpdate(ctx, indexName, update)));
|
|
22520
22778
|
let failures;
|
|
22521
22779
|
try {
|
|
22522
22780
|
failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
@@ -22547,7 +22805,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
22547
22805
|
const update = prepared[i];
|
|
22548
22806
|
const failure = failures[i];
|
|
22549
22807
|
if (failure !== null) {
|
|
22550
|
-
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
22808
|
+
failed.push({ docId: update.docId, error: asUpdateError(ctx, failure.error) });
|
|
22551
22809
|
continue;
|
|
22552
22810
|
}
|
|
22553
22811
|
if (!update.buffered) settled.push(update);
|
|
@@ -22556,18 +22814,19 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
22556
22814
|
}
|
|
22557
22815
|
succeeded.push(update.docId);
|
|
22558
22816
|
}
|
|
22559
|
-
await settleUpdates(ctx, indexName, settled, hold);
|
|
22817
|
+
await unlessShutDown(ctx, () => settleUpdates(ctx, indexName, settled, hold));
|
|
22560
22818
|
if (chunkEnd < updates.length) {
|
|
22561
|
-
ctx.checkHeapPressure(indexName);
|
|
22819
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
22562
22820
|
await new Promise((r) => setTimeout(r, 0));
|
|
22563
22821
|
}
|
|
22564
22822
|
}
|
|
22823
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
22565
22824
|
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
22566
22825
|
for (const fieldPath of touchedVectorFields) {
|
|
22567
22826
|
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
22568
22827
|
}
|
|
22569
22828
|
ctx.checkHeapPressure(indexName);
|
|
22570
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22829
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
22571
22830
|
return { succeeded, failed };
|
|
22572
22831
|
}
|
|
22573
22832
|
|
|
@@ -22613,12 +22872,17 @@ async function withReservation(grouping, indexName, document, write) {
|
|
|
22613
22872
|
reservation.release();
|
|
22614
22873
|
}
|
|
22615
22874
|
}
|
|
22616
|
-
async function writeInGroups(grouping, indexName, items, write) {
|
|
22875
|
+
async function writeInGroups(ctx, grouping, indexName, items, write) {
|
|
22617
22876
|
const succeeded = [];
|
|
22618
22877
|
const failed = [];
|
|
22619
22878
|
await grouping.applyInGroups(indexName, {
|
|
22620
22879
|
items,
|
|
22621
22880
|
failed,
|
|
22881
|
+
refuseFrom: (start) => {
|
|
22882
|
+
if (!shutDownStarted(ctx)) return false;
|
|
22883
|
+
failAsShutDown(failed, items, docIdOf, start);
|
|
22884
|
+
return true;
|
|
22885
|
+
},
|
|
22622
22886
|
apply: async (group) => {
|
|
22623
22887
|
const result = await write(group);
|
|
22624
22888
|
for (const docId of result.succeeded) succeeded.push(docId);
|
|
@@ -22656,6 +22920,7 @@ async function insertDocumentBatch2(ctx, indexName, documents2, options, hold) {
|
|
|
22656
22920
|
}
|
|
22657
22921
|
const items = documents2.map((document) => ({ docId: providedDocId(document) ?? "", document }));
|
|
22658
22922
|
return writeInGroups(
|
|
22923
|
+
ctx,
|
|
22659
22924
|
grouping,
|
|
22660
22925
|
indexName,
|
|
22661
22926
|
items,
|
|
@@ -22688,6 +22953,7 @@ async function updateDocumentBatch2(ctx, indexName, updates, options, hold) {
|
|
|
22688
22953
|
}
|
|
22689
22954
|
noteGroupedChurn(ctx, indexName, updates.length);
|
|
22690
22955
|
return writeInGroups(
|
|
22956
|
+
ctx,
|
|
22691
22957
|
grouping,
|
|
22692
22958
|
indexName,
|
|
22693
22959
|
updates.map(updateItemOf),
|
|
@@ -22726,7 +22992,7 @@ var suggestionsCode = codeLoader(async () => {
|
|
|
22726
22992
|
const [msgpack, suggestionStore2, code7] = await Promise.all([
|
|
22727
22993
|
loadMessagePack(),
|
|
22728
22994
|
loadSuggestionStore(),
|
|
22729
|
-
import('./suggestions-leaf-
|
|
22995
|
+
import('./suggestions-leaf-KDPM3P27.mjs')
|
|
22730
22996
|
]);
|
|
22731
22997
|
code7.bindSuggestionsHost(optionalCodeHost(), msgpack, suggestionStore2);
|
|
22732
22998
|
return code7;
|
|
@@ -23070,32 +23336,6 @@ function createDocumentWrites(source) {
|
|
|
23070
23336
|
};
|
|
23071
23337
|
}
|
|
23072
23338
|
|
|
23073
|
-
// src/partitioning/manager/nested-values.ts
|
|
23074
|
-
function setNestedValue(obj, path, value) {
|
|
23075
|
-
if (!path.includes(".")) {
|
|
23076
|
-
obj[path] = value;
|
|
23077
|
-
return;
|
|
23078
|
-
}
|
|
23079
|
-
const segments2 = path.split(".");
|
|
23080
|
-
let current = obj;
|
|
23081
|
-
for (let i = 0; i < segments2.length - 1; i++) {
|
|
23082
|
-
let next = current[segments2[i]];
|
|
23083
|
-
if (next === null || next === void 0 || typeof next !== "object") {
|
|
23084
|
-
next = {};
|
|
23085
|
-
current[segments2[i]] = next;
|
|
23086
|
-
}
|
|
23087
|
-
current = next;
|
|
23088
|
-
}
|
|
23089
|
-
current[segments2[segments2.length - 1]] = value;
|
|
23090
|
-
}
|
|
23091
|
-
function copyVectorsInto(document, docId, vectorIndexes, projection) {
|
|
23092
|
-
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
23093
|
-
if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
|
|
23094
|
-
const vector = vectorIndex.getVector(docId);
|
|
23095
|
-
if (vector) setNestedValue(document, fieldPath, vector);
|
|
23096
|
-
}
|
|
23097
|
-
}
|
|
23098
|
-
|
|
23099
23339
|
// src/core/stored-values/references.ts
|
|
23100
23340
|
var documents = /* @__PURE__ */ new WeakMap();
|
|
23101
23341
|
function documentStoredFields(document, fields) {
|
|
@@ -24353,7 +24593,7 @@ var workerCopies = codeLoader(async () => {
|
|
|
24353
24593
|
const [segmentCodec2, segmentMerge2, code7] = await Promise.all([
|
|
24354
24594
|
loadSegmentCodec(),
|
|
24355
24595
|
loadSegmentMerge(),
|
|
24356
|
-
import('./worker-copies-
|
|
24596
|
+
import('./worker-copies-FVFMBBH6.mjs')
|
|
24357
24597
|
]);
|
|
24358
24598
|
code7.bindWorkerCopiesHost(optionalCodeHost(), segmentCodec2, segmentMerge2);
|
|
24359
24599
|
return code7;
|
|
@@ -24528,6 +24768,7 @@ function writesHostPart() {
|
|
|
24528
24768
|
requireDocumentWithinSizeLimit,
|
|
24529
24769
|
requireMatchingDocId,
|
|
24530
24770
|
requireSurfaceFormsForPartialFields,
|
|
24771
|
+
setNestedValue,
|
|
24531
24772
|
storedBM25Params,
|
|
24532
24773
|
turnsAreBudgeted,
|
|
24533
24774
|
updateDocumentVectors,
|
|
@@ -24578,7 +24819,7 @@ var loaded2 = null;
|
|
|
24578
24819
|
var loading3 = null;
|
|
24579
24820
|
function importOptionalReads() {
|
|
24580
24821
|
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
24581
|
-
loading3 ??= import('./optional-reads-
|
|
24822
|
+
loading3 ??= import('./optional-reads-TN3PT6M4.mjs').then(
|
|
24582
24823
|
(code7) => {
|
|
24583
24824
|
code7.bindOptionalReadsHost(optionalCodeHost());
|
|
24584
24825
|
loaded2 = code7;
|
|
@@ -24925,6 +25166,7 @@ async function applyInGroups(projection, indexName, write) {
|
|
|
24925
25166
|
);
|
|
24926
25167
|
let start = 0;
|
|
24927
25168
|
while (start < bounded.length) {
|
|
25169
|
+
if (write.refuseFrom?.(start) === true) return null;
|
|
24928
25170
|
let group;
|
|
24929
25171
|
try {
|
|
24930
25172
|
group = projection.takeGroup(indexName, bounds, start, mostDocuments, countExactly);
|
|
@@ -26483,7 +26725,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
26483
26725
|
|
|
26484
26726
|
// src/engine/durability-config-code.ts
|
|
26485
26727
|
var durabilityConfig = codeLoader(
|
|
26486
|
-
() => import('./durability-config-
|
|
26728
|
+
() => import('./durability-config-HJETGZVB.mjs').then((code7) => {
|
|
26487
26729
|
code7.bindDurabilityConfigHost(optionalCodeHost());
|
|
26488
26730
|
return code7;
|
|
26489
26731
|
}),
|
|
@@ -27081,7 +27323,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
27081
27323
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
27082
27324
|
}
|
|
27083
27325
|
const durabilityWiring = durabilityTier === null ? null : (await Promise.all([loadStorageCode(), loadOptionalWrites()]))[0];
|
|
27084
|
-
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
27326
|
+
const filesystemDurability = durabilityTier?.kind === "wal" && durabilityTier.indexedDB === void 0;
|
|
27085
27327
|
const executor = createDirectExecutor({
|
|
27086
27328
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
27087
27329
|
vectorStorage: filesystemDurability ? "disk" : "memory",
|
|
@@ -27359,8 +27601,13 @@ async function shutdownEngine(core2) {
|
|
|
27359
27601
|
await core2.analysisRebuild.shutdown();
|
|
27360
27602
|
await core2.querySuggestionHooks?.shutdown();
|
|
27361
27603
|
core2.indexState.dispose();
|
|
27604
|
+
let durabilityFailure = null;
|
|
27362
27605
|
if (durability) {
|
|
27363
|
-
|
|
27606
|
+
try {
|
|
27607
|
+
await durability.manager.shutdown();
|
|
27608
|
+
} catch (error) {
|
|
27609
|
+
durabilityFailure = { error };
|
|
27610
|
+
}
|
|
27364
27611
|
}
|
|
27365
27612
|
const vectorDisposals = [];
|
|
27366
27613
|
for (const [name] of indexRegistry) {
|
|
@@ -27394,6 +27641,7 @@ async function shutdownEngine(core2) {
|
|
|
27394
27641
|
await Promise.all(vectorDisposals);
|
|
27395
27642
|
eventHandlers.clear();
|
|
27396
27643
|
indexRegistry.clear();
|
|
27644
|
+
if (durabilityFailure !== null) throw durabilityFailure.error;
|
|
27397
27645
|
}
|
|
27398
27646
|
|
|
27399
27647
|
// src/core/keyword-score/layout.ts
|