@delali/narsil 0.4.1 → 0.4.2-next.273
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-56UNHLAF.mjs} +1 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-EKISQBV2.mjs} +283 -244
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-VAPCFSMX.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-VIVG3OXM.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-76Q5SZ5S.mjs} +23 -10
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-3HKIVZFU.mjs} +295 -106
- package/dist/browser/{segment-codec-leaf-A427QJUX.mjs → segment-codec-leaf-GXSRX5VT.mjs} +47 -30
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-35RWLUVY.mjs} +63 -144
- package/dist/{chunk-6GHX35ZM.mjs → chunk-43T3P3BE.mjs} +1 -1
- package/dist/{chunk-IL5DRCNU.mjs → chunk-4JLHHEBN.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-4YIVY3PB.mjs} +2 -2
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-5SHWA6PE.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-6LVQ3GLT.mjs} +3 -3
- package/dist/{chunk-DDUXEJXW.mjs → chunk-6SIEVWMG.mjs} +45 -22
- package/dist/{chunk-WM5KJSQV.mjs → chunk-ACNHJOGC.mjs} +1 -1
- package/dist/{chunk-755NM4PL.mjs → chunk-BWPVDIRN.mjs} +183 -21
- package/dist/{chunk-SMQPZHPS.mjs → chunk-C57GWAXN.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-DRZP5MSG.mjs} +5 -5
- package/dist/{chunk-MYN4K5IS.mjs → chunk-DZMD4OMD.mjs} +1 -1
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-EY5U4P5V.mjs} +1 -1
- package/dist/{chunk-X3EQCQLP.mjs → chunk-EZJ6HBG7.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-FE2XILWN.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-G3KVEHIW.mjs} +2 -2
- package/dist/{chunk-B3MO37YF.mjs → chunk-GTGQLCEI.mjs} +1 -1
- package/dist/{chunk-MSJD74GN.mjs → chunk-H23FB267.mjs} +5 -5
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-WTMCGZ74.mjs → chunk-KXA77DPZ.mjs} +2 -2
- package/dist/{chunk-6I6OO6B2.mjs → chunk-LRIJ7U3T.mjs} +4 -4
- package/dist/{chunk-3IE3A74H.mjs → chunk-PNFP5THV.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-PQWK7KXS.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-OGYSPNL2.mjs → chunk-QNZJYMCF.mjs} +7 -7
- package/dist/{chunk-DBC4EJWB.mjs → chunk-R3PFPFME.mjs} +1 -1
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-MI6UZGUR.mjs → chunk-SPA7GXST.mjs} +3 -3
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-WYH3QHBK.mjs} +698 -451
- package/dist/{chunk-D456ZFPQ.mjs → chunk-X5V25BPN.mjs} +2 -2
- package/dist/{chunk-GXCDWLQI.mjs → chunk-Y7GXGJFZ.mjs} +23 -10
- package/dist/{chunk-NCIGNQJE.mjs → chunk-YGRJLI2Z.mjs} +930 -387
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-ZO7VLAIT.mjs} +2 -2
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-ZTB7AWTW.mjs} +4 -4
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-7G57AYXH.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-KPN7NSOC.mjs} +14 -14
- package/dist/msgpack-leaf-LIT7TKZW.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-CSULLISE.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-BKQADXOT.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-HTSJKZRU.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AOLLV37A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-DN4FT62B.mjs} +23 -10
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-EF4Y5C3Y.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-ESVBC3AT.mjs} +1 -1
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-H6LK5WCF.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-JEYZQA4Z.mjs} +699 -451
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-KJQ3BUVM.mjs} +4 -4
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-QYVRD6C5.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-X3VY2YT4.mjs} +2 -2
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-ZF3VSGEI.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-FOUPTT7J.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-MGFFGKYQ.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-MVJTP3E4.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PSWHL55A.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-WOJWSDLW.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-ZUN6Y44X.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-K5XG46FV.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-KDPM3P27.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-FVFMBBH6.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-3QEJNMFH.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-Q76Q4S35.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-BSDDR7DQ.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-6TT66HC5.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-R5JQ2X2M.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-NCGETZYY.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-TX3GTD6S.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-VLVPAUWC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-NZSBQZKG.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -12
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { encodeDocument } from './chunk-2CKDCC65.mjs';
|
|
2
2
|
import { workerResourceLimits, admittedHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal } from './chunk-2AFZE7Y3.mjs';
|
|
3
|
-
import { headerAt, MAP_TAGS, decode, Decoder } from './chunk-
|
|
3
|
+
import { headerAt, MAP_TAGS, decode, Decoder } from './chunk-Y7GXGJFZ.mjs';
|
|
4
4
|
import { newSyncEpoch, isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
5
5
|
import { english } from './chunk-7T4LAYT4.mjs';
|
|
6
|
-
import { patternSearchModule, PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, matchPattern, claimsTypoPreparation, DEFAULT_PATTERN_WORK_CAP, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, bitsetSet, discardStoredValues, MAX_FILTER_DEPTH, createBitSet, installStoredFields, storedValueBytes, bitsetIterator, bitsetIsEmpty, prepareStoredValues, fuzzyTermMatches, MATCHER_STATE_BUDGET_BYTES, GrowableUint32, checkedSize, buildStoredValues, bitsetFromSet, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyPrefixOf, boundedLevenshtein, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, FILTER_OPERAND_COPY_DEPTH, PATTERN_WORK_FLUSH_UNITS } from './chunk-
|
|
7
|
-
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, isStringValuedBase, flattenSchema, schemaFieldsOf, getNestedValue, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, requireLoadedMessagePack, vectorDimensionOf, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, isWordIndexedBase, PROTOTYPE_POLLUTION_KEYS, tokenizeOptions, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, getFlatSchema, getOrCreateFieldNameIndex, MAX_NESTING_DEPTH, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-
|
|
6
|
+
import { patternSearchModule, PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, matchPattern, claimsTypoPreparation, DEFAULT_PATTERN_WORK_CAP, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, bitsetSet, discardStoredValues, MAX_FILTER_DEPTH, createBitSet, installStoredFields, storedValueBytes, bitsetIterator, bitsetIsEmpty, prepareStoredValues, fuzzyTermMatches, MATCHER_STATE_BUDGET_BYTES, GrowableUint32, checkedSize, buildStoredValues, bitsetFromSet, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyPrefixOf, boundedLevenshtein, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, FILTER_OPERAND_COPY_DEPTH, PATTERN_WORK_FLUSH_UNITS } from './chunk-4JLHHEBN.mjs';
|
|
7
|
+
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, isStringValuedBase, flattenSchema, schemaFieldsOf, getNestedValue, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, requireLoadedMessagePack, vectorDimensionOf, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, isWordIndexedBase, PROTOTYPE_POLLUTION_KEYS, tokenizeOptions, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, getFlatSchema, getOrCreateFieldNameIndex, MAX_NESTING_DEPTH, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-EY5U4P5V.mjs';
|
|
8
8
|
import { detectRuntime, splitWorkerBudget, resolveWorkerCount, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, countEngineThread, requirePinnedState, createDocIdMap, SHUTDOWN_TIMEOUT_MS, stopOfSignal, requireSearchActive, resolveWorkerEntry, missingWorkerEntry, signalOfSearch, answerFromOneState, sharedStopOfSignal, engineThreadYoungGenerationBytes, resolveRequestThreadCount, spawnNodeWorker, bindSearchSignal, WORKER_YOUNG_GENERATION_MB, nodeThreadId, youngGenerationBytesOfThisV8 } from './chunk-FJXJZUWF.mjs';
|
|
9
9
|
import { changeIndexOutsideExternalWaits, turnsAreBudgeted, turnIsSpent, yieldTurn, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, noteThreadSlotLive, threadSlotOfWorker, indexChangeOutsideExternalWaits, PROCESS_LIMIT_REFRESH_INTERVAL_MS, countWrite, PROCESS_STATUS_READ_BYTES, PROCESS_STATUS_READ_CEILING_BYTES, threadSlotGone, noteThreadSlotGone, adoptSharedTurns, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_BLOCK_MAX_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_ATTACH_RETRY_MS, MAX_WASM_PAGES, MAX_VECTOR_ORDINALS, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_DOC_ID_TABLE_BYTES, LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, DISK_STORAGE_BLOCK_BYTES, waitOutsideCountedWrite, collectAndReturnPages } from './chunk-5EI5CXO6.mjs';
|
|
10
10
|
import { MIN_DOCUMENTS_FOR_SEGMENTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, MAX_PARTITIONS, DEFAULT_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, BATCH_CHUNK_SIZE, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_CALLS_ACROSS_INDEX_CHANGES, DEFAULT_PAGE_SIZE, CURSOR_BINDING_BUFFER_BYTES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, DEFAULT_OFFSET, MIN_SWEEP_INTERVAL_MS, MIN_RANK_CONSTANT, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, FUSION_TIE_INSERTION_LIMIT, DEFAULT_RANK_CONSTANT, DEFAULT_LINEAR_ALPHA, STATISTICS_BROADCAST_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, EXPORT_CHUNK_BYTES, DEFAULT_MAX_PARTITIONS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, ANALYSIS_CHANGE_MEMBERS, MIN_HYBRID_DEPTH, MAX_SUGGEST_SCATTER_LIMIT, MAX_SUGGEST_LIMIT, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_SUGGEST_LIMIT, DEFAULT_MAX_PER_GROUP, HEAP_PRESSURE_WARNING_FRACTION, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-3ECIRFS5.mjs';
|
|
@@ -993,7 +993,7 @@ var loaded = null;
|
|
|
993
993
|
var loading = null;
|
|
994
994
|
function importOptionalReads() {
|
|
995
995
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
996
|
-
loading ??= import('./optional-reads-
|
|
996
|
+
loading ??= import('./optional-reads-ESDVS225.mjs').then(
|
|
997
997
|
(code7) => {
|
|
998
998
|
code7.bindOptionalReadsHost(optionalCodeHost());
|
|
999
999
|
loaded = code7;
|
|
@@ -12686,7 +12686,7 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
12686
12686
|
|
|
12687
12687
|
// src/core/segment-codec-code.ts
|
|
12688
12688
|
var segmentCodec = codeLoader(async () => {
|
|
12689
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-
|
|
12689
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-BSDDR7DQ.mjs')]);
|
|
12690
12690
|
code7.bindSegmentCodecHost(optionalCodeHost(), msgpack);
|
|
12691
12691
|
return code7;
|
|
12692
12692
|
}, "segment codec");
|
|
@@ -12694,7 +12694,7 @@ var loadSegmentCodec = segmentCodec.load;
|
|
|
12694
12694
|
|
|
12695
12695
|
// src/query-suggestions/store-code.ts
|
|
12696
12696
|
var suggestionStore = codeLoader(async () => {
|
|
12697
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-
|
|
12697
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-NCGETZYY.mjs')]);
|
|
12698
12698
|
code7.bindSuggestionStoreHost(optionalCodeHost(), msgpack);
|
|
12699
12699
|
return code7;
|
|
12700
12700
|
}, "query suggestions store");
|
|
@@ -12775,7 +12775,7 @@ var storageCode = codeLoader(async () => {
|
|
|
12775
12775
|
loadSegmentMerge(),
|
|
12776
12776
|
loadSuggestionStore(),
|
|
12777
12777
|
loadVectorCore(),
|
|
12778
|
-
import('./storage-leaf-
|
|
12778
|
+
import('./storage-leaf-R5JQ2X2M.mjs')
|
|
12779
12779
|
]);
|
|
12780
12780
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
12781
12781
|
return code7;
|
|
@@ -13614,6 +13614,177 @@ function mergeGroupComponents(components, internalId, contribution) {
|
|
|
13614
13614
|
});
|
|
13615
13615
|
}
|
|
13616
13616
|
|
|
13617
|
+
// src/core/partition/top-k-heap.ts
|
|
13618
|
+
function candidateWorse(a, b) {
|
|
13619
|
+
if (a.score < b.score) return true;
|
|
13620
|
+
if (a.score > b.score) return false;
|
|
13621
|
+
return unorderedCandidateWorse(a, b);
|
|
13622
|
+
}
|
|
13623
|
+
function unorderedCandidateWorse(a, b) {
|
|
13624
|
+
return (compareScores(a.score, b.score) || compareCodePoints(a.externalId, b.externalId)) > 0;
|
|
13625
|
+
}
|
|
13626
|
+
function buildMinHeap(heap) {
|
|
13627
|
+
for (let index = (heap.length >> 1) - 1; index >= 0; index--) {
|
|
13628
|
+
siftDown(heap, index);
|
|
13629
|
+
}
|
|
13630
|
+
}
|
|
13631
|
+
function siftDown(heap, start) {
|
|
13632
|
+
const len = heap.length;
|
|
13633
|
+
let current = start;
|
|
13634
|
+
for (; ; ) {
|
|
13635
|
+
let worst = current;
|
|
13636
|
+
const left = 2 * current + 1;
|
|
13637
|
+
const right = 2 * current + 2;
|
|
13638
|
+
if (left < len && candidateWorse(heap[left], heap[worst])) worst = left;
|
|
13639
|
+
if (right < len && candidateWorse(heap[right], heap[worst])) worst = right;
|
|
13640
|
+
if (worst === current) break;
|
|
13641
|
+
const held = heap[current];
|
|
13642
|
+
heap[current] = heap[worst];
|
|
13643
|
+
heap[worst] = held;
|
|
13644
|
+
current = worst;
|
|
13645
|
+
}
|
|
13646
|
+
}
|
|
13647
|
+
function sortSelection(heap) {
|
|
13648
|
+
heap.sort((a, b) => compareScores(a.score, b.score) || compareCodePoints(a.externalId, b.externalId));
|
|
13649
|
+
}
|
|
13650
|
+
|
|
13651
|
+
// src/core/partition/score-buffer.ts
|
|
13652
|
+
var MAX_GENERATION = 2147483647;
|
|
13653
|
+
function createScoreBuffer(capacity) {
|
|
13654
|
+
return {
|
|
13655
|
+
scores: new Float64Array(capacity),
|
|
13656
|
+
stamps: new Int32Array(capacity),
|
|
13657
|
+
touched: new Int32Array(Math.max(1, Math.min(capacity, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY))),
|
|
13658
|
+
touchedCount: 0,
|
|
13659
|
+
generation: 0
|
|
13660
|
+
};
|
|
13661
|
+
}
|
|
13662
|
+
function beginScoring(buffer, capacity) {
|
|
13663
|
+
if (buffer.scores.length < capacity) {
|
|
13664
|
+
buffer.scores = new Float64Array(capacity);
|
|
13665
|
+
buffer.stamps = new Int32Array(capacity);
|
|
13666
|
+
buffer.generation = 0;
|
|
13667
|
+
}
|
|
13668
|
+
if (buffer.generation >= MAX_GENERATION) {
|
|
13669
|
+
buffer.stamps.fill(0);
|
|
13670
|
+
buffer.generation = 0;
|
|
13671
|
+
}
|
|
13672
|
+
buffer.generation++;
|
|
13673
|
+
buffer.touchedCount = 0;
|
|
13674
|
+
}
|
|
13675
|
+
function appendTouched(buffer, internalId) {
|
|
13676
|
+
if (buffer.touchedCount === buffer.touched.length) {
|
|
13677
|
+
const grown4 = new Int32Array(buffer.touched.length * 2);
|
|
13678
|
+
grown4.set(buffer.touched);
|
|
13679
|
+
buffer.touched = grown4;
|
|
13680
|
+
}
|
|
13681
|
+
buffer.touched[buffer.touchedCount] = internalId;
|
|
13682
|
+
buffer.touchedCount++;
|
|
13683
|
+
}
|
|
13684
|
+
function addScore(buffer, internalId, termScore) {
|
|
13685
|
+
if (buffer.stamps[internalId] === buffer.generation) {
|
|
13686
|
+
buffer.scores[internalId] += termScore;
|
|
13687
|
+
return;
|
|
13688
|
+
}
|
|
13689
|
+
buffer.stamps[internalId] = buffer.generation;
|
|
13690
|
+
buffer.scores[internalId] = termScore;
|
|
13691
|
+
appendTouched(buffer, internalId);
|
|
13692
|
+
}
|
|
13693
|
+
function markCounted(buffer, internalId) {
|
|
13694
|
+
const countedStamp = -buffer.generation;
|
|
13695
|
+
const stamp = buffer.stamps[internalId];
|
|
13696
|
+
if (stamp === buffer.generation || stamp === countedStamp) return false;
|
|
13697
|
+
buffer.stamps[internalId] = countedStamp;
|
|
13698
|
+
return true;
|
|
13699
|
+
}
|
|
13700
|
+
function kthBestScore(buffer, selection, resolver) {
|
|
13701
|
+
const wanted = selection.length;
|
|
13702
|
+
const { touched, touchedCount, scores } = buffer;
|
|
13703
|
+
let size = 0;
|
|
13704
|
+
let unordered = 0;
|
|
13705
|
+
for (let index = 0; index < touchedCount; index++) {
|
|
13706
|
+
const internalId = touched[index];
|
|
13707
|
+
const score = scores[internalId];
|
|
13708
|
+
if (size === wanted && !(score > selection[0])) continue;
|
|
13709
|
+
if (!resolver.isLive(internalId)) continue;
|
|
13710
|
+
let at = 0;
|
|
13711
|
+
if (size < wanted) {
|
|
13712
|
+
if (Number.isNaN(score)) {
|
|
13713
|
+
unordered++;
|
|
13714
|
+
continue;
|
|
13715
|
+
}
|
|
13716
|
+
at = size++;
|
|
13717
|
+
while (at > 0) {
|
|
13718
|
+
const parent = at - 1 >> 1;
|
|
13719
|
+
if (selection[parent] <= score) break;
|
|
13720
|
+
selection[at] = selection[parent];
|
|
13721
|
+
at = parent;
|
|
13722
|
+
}
|
|
13723
|
+
} else {
|
|
13724
|
+
for (; ; ) {
|
|
13725
|
+
const left = 2 * at + 1;
|
|
13726
|
+
if (left >= size) break;
|
|
13727
|
+
const right = left + 1;
|
|
13728
|
+
const child = right < size && selection[right] < selection[left] ? right : left;
|
|
13729
|
+
if (selection[child] >= score) break;
|
|
13730
|
+
selection[at] = selection[child];
|
|
13731
|
+
at = child;
|
|
13732
|
+
}
|
|
13733
|
+
}
|
|
13734
|
+
selection[at] = score;
|
|
13735
|
+
}
|
|
13736
|
+
if (wanted === 0) return Number.NEGATIVE_INFINITY;
|
|
13737
|
+
if (size === wanted) return selection[0];
|
|
13738
|
+
return size + unordered >= wanted ? Number.NaN : Number.NEGATIVE_INFINITY;
|
|
13739
|
+
}
|
|
13740
|
+
function hasScore(buffer, internalId) {
|
|
13741
|
+
return buffer.stamps[internalId] === buffer.generation;
|
|
13742
|
+
}
|
|
13743
|
+
function topKFromBuffer(buffer, k, resolver, components) {
|
|
13744
|
+
const wanted = Number.isFinite(k) ? Math.max(0, Math.floor(k)) : 0;
|
|
13745
|
+
if (wanted <= 0) return [];
|
|
13746
|
+
const heap = [];
|
|
13747
|
+
const { touched, touchedCount, scores } = buffer;
|
|
13748
|
+
let full = false;
|
|
13749
|
+
let threshold = 0;
|
|
13750
|
+
for (let index = 0; index < touchedCount; index++) {
|
|
13751
|
+
const internalId = touched[index];
|
|
13752
|
+
const score = scores[internalId];
|
|
13753
|
+
if (full && (score < threshold || Number.isNaN(score) && !Number.isNaN(threshold))) continue;
|
|
13754
|
+
const externalId = resolver.toExternal(internalId);
|
|
13755
|
+
if (externalId === void 0) continue;
|
|
13756
|
+
const candidate = { internalId, externalId, score };
|
|
13757
|
+
if (!full) {
|
|
13758
|
+
heap.push(candidate);
|
|
13759
|
+
if (heap.length === wanted) {
|
|
13760
|
+
buildMinHeap(heap);
|
|
13761
|
+
full = true;
|
|
13762
|
+
threshold = heap[0].score;
|
|
13763
|
+
}
|
|
13764
|
+
continue;
|
|
13765
|
+
}
|
|
13766
|
+
if (candidateWorse(heap[0], candidate)) {
|
|
13767
|
+
heap[0] = candidate;
|
|
13768
|
+
siftDown(heap, 0);
|
|
13769
|
+
threshold = heap[0].score;
|
|
13770
|
+
}
|
|
13771
|
+
}
|
|
13772
|
+
sortSelection(heap);
|
|
13773
|
+
const result = new Array(heap.length);
|
|
13774
|
+
for (let index = 0; index < heap.length; index++) {
|
|
13775
|
+
const entry = heap[index];
|
|
13776
|
+
const record2 = components?.get(entry.internalId);
|
|
13777
|
+
result[index] = {
|
|
13778
|
+
docId: entry.externalId,
|
|
13779
|
+
score: entry.score,
|
|
13780
|
+
termFrequencies: record2?.termFrequencies ?? EMPTY_COMPONENTS,
|
|
13781
|
+
fieldLengths: record2?.fieldLengths ?? EMPTY_COMPONENTS,
|
|
13782
|
+
idf: record2?.idf ?? EMPTY_COMPONENTS
|
|
13783
|
+
};
|
|
13784
|
+
}
|
|
13785
|
+
return result;
|
|
13786
|
+
}
|
|
13787
|
+
|
|
13617
13788
|
// src/core/partition/group-scoring.ts
|
|
13618
13789
|
function partialFieldMarks(fieldNames, partialFields) {
|
|
13619
13790
|
const marks = new Uint8Array(fieldNames.length);
|
|
@@ -13667,72 +13838,98 @@ function groupPostingCount(matches) {
|
|
|
13667
13838
|
for (const match of matches) total += match.postingList.length;
|
|
13668
13839
|
return total;
|
|
13669
13840
|
}
|
|
13670
|
-
function
|
|
13671
|
-
const { fields,
|
|
13841
|
+
function scoreMatchPostings(context, match, visit) {
|
|
13842
|
+
const { fields, filterBitset, partialMarks } = context;
|
|
13672
13843
|
const scoresAreZero = context.totalDocs === 0;
|
|
13673
13844
|
const onePair = fields.boundPairs.length === 2;
|
|
13845
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
13846
|
+
match.postingList
|
|
13847
|
+
);
|
|
13848
|
+
let runStart = 0;
|
|
13849
|
+
let runEnd = 0;
|
|
13850
|
+
for (let at = 0; at < count; at++) {
|
|
13851
|
+
const internalId = docIds[at];
|
|
13852
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
13853
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
13854
|
+
let pi = at;
|
|
13855
|
+
if (at < runEnd) {
|
|
13856
|
+
pi = fieldRun.entries[at - runStart];
|
|
13857
|
+
} else if (at + 2 < count && docIds[at + 2] === internalId) {
|
|
13858
|
+
pi = orderRunByFieldName(docIds, fieldNameIndices, fields.nameRanks, at, count)[0];
|
|
13859
|
+
runStart = at;
|
|
13860
|
+
runEnd = at + fieldRun.length;
|
|
13861
|
+
}
|
|
13862
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
13863
|
+
if (fields.searchable[fieldIndex] === 0) continue;
|
|
13864
|
+
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
13865
|
+
if (factor === 0) continue;
|
|
13866
|
+
const termFrequency = termFrequencies[pi];
|
|
13867
|
+
const fieldBoost = fields.boosts[fieldIndex];
|
|
13868
|
+
const avgLen = fields.averageLengths[fieldIndex];
|
|
13869
|
+
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
13870
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
13871
|
+
termFrequency,
|
|
13872
|
+
match.idf,
|
|
13873
|
+
actualFieldLength,
|
|
13874
|
+
avgLen,
|
|
13875
|
+
onePair ? fields.boundPairs[0] : fields.k1s[fieldIndex],
|
|
13876
|
+
onePair ? fields.boundPairs[1] : fields.bs[fieldIndex]
|
|
13877
|
+
)) * fieldBoost * factor;
|
|
13878
|
+
visit(internalId, fieldIndex, termFrequency, actualFieldLength, termScore);
|
|
13879
|
+
}
|
|
13880
|
+
}
|
|
13881
|
+
var termScores = null;
|
|
13882
|
+
var bestScores = null;
|
|
13883
|
+
function addGroupScores(context, matches, target, only, coverage) {
|
|
13884
|
+
const best = bestGroupScores(context, matches, target.scores.length);
|
|
13885
|
+
for (let index = 0; index < best.touchedCount; index++) {
|
|
13886
|
+
const internalId = best.touched[index];
|
|
13887
|
+
if (only !== null && !bitsetHas(only, internalId)) continue;
|
|
13888
|
+
coverage?.mark(internalId);
|
|
13889
|
+
addScore(target, internalId, best.scores[internalId]);
|
|
13890
|
+
}
|
|
13891
|
+
}
|
|
13892
|
+
function bestGroupScores(context, matches, capacity) {
|
|
13893
|
+
termScores ??= createScoreBuffer(capacity);
|
|
13894
|
+
bestScores ??= createScoreBuffer(capacity);
|
|
13895
|
+
const perTerm = termScores;
|
|
13896
|
+
const best = bestScores;
|
|
13897
|
+
const addTermScore = (internalId, _fieldIndex, _termFrequency, _fieldLength, termScore) => addScore(perTerm, internalId, termScore);
|
|
13898
|
+
beginScoring(best, capacity);
|
|
13899
|
+
for (const match of matches) {
|
|
13900
|
+
beginScoring(perTerm, capacity);
|
|
13901
|
+
scoreMatchPostings(context, match, addTermScore);
|
|
13902
|
+
for (let index = 0; index < perTerm.touchedCount; index++) {
|
|
13903
|
+
const internalId = perTerm.touched[index];
|
|
13904
|
+
const score = perTerm.scores[internalId];
|
|
13905
|
+
if (!hasScore(best, internalId)) addScore(best, internalId, score);
|
|
13906
|
+
else if (compareScores(score, best.scores[internalId]) < 0) best.scores[internalId] = score;
|
|
13907
|
+
}
|
|
13908
|
+
}
|
|
13909
|
+
return best;
|
|
13910
|
+
}
|
|
13911
|
+
function computeGroupContributions(context, matches) {
|
|
13912
|
+
const { fieldNames } = context;
|
|
13674
13913
|
const best = /* @__PURE__ */ new Map();
|
|
13675
13914
|
for (const match of matches) {
|
|
13676
13915
|
const perTerm = /* @__PURE__ */ new Map();
|
|
13677
|
-
|
|
13678
|
-
match.postingList
|
|
13679
|
-
);
|
|
13680
|
-
let runStart = 0;
|
|
13681
|
-
let runEnd = 0;
|
|
13682
|
-
for (let at = 0; at < count; at++) {
|
|
13683
|
-
const internalId = docIds[at];
|
|
13684
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
13685
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
13686
|
-
let pi = at;
|
|
13687
|
-
if (at < runEnd) {
|
|
13688
|
-
pi = fieldRun.entries[at - runStart];
|
|
13689
|
-
} else if (at + 2 < count && docIds[at + 2] === internalId) {
|
|
13690
|
-
pi = orderRunByFieldName(docIds, fieldNameIndices, fields.nameRanks, at, count)[0];
|
|
13691
|
-
runStart = at;
|
|
13692
|
-
runEnd = at + fieldRun.length;
|
|
13693
|
-
}
|
|
13694
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
13695
|
-
if (fields.searchable[fieldIndex] === 0) continue;
|
|
13696
|
-
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
13697
|
-
if (factor === 0) continue;
|
|
13916
|
+
scoreMatchPostings(context, match, (internalId, fieldIndex, termFrequency, actualFieldLength, termScore) => {
|
|
13698
13917
|
const fieldName = fieldNames[fieldIndex];
|
|
13699
|
-
const termFrequency = termFrequencies[pi];
|
|
13700
|
-
const fieldBoost = fields.boosts[fieldIndex];
|
|
13701
|
-
const avgLen = fields.averageLengths[fieldIndex];
|
|
13702
|
-
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
13703
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
13704
|
-
termFrequency,
|
|
13705
|
-
match.idf,
|
|
13706
|
-
actualFieldLength,
|
|
13707
|
-
avgLen,
|
|
13708
|
-
onePair ? fields.boundPairs[0] : fields.k1s[fieldIndex],
|
|
13709
|
-
onePair ? fields.boundPairs[1] : fields.bs[fieldIndex]
|
|
13710
|
-
)) * fieldBoost * factor;
|
|
13711
13918
|
const existing = perTerm.get(internalId);
|
|
13712
13919
|
if (existing) {
|
|
13713
13920
|
existing.score += termScore;
|
|
13714
|
-
|
|
13715
|
-
|
|
13716
|
-
|
|
13717
|
-
}
|
|
13718
|
-
} else if (collect3) {
|
|
13719
|
-
perTerm.set(internalId, {
|
|
13720
|
-
score: termScore,
|
|
13721
|
-
token: match.token,
|
|
13722
|
-
idf: match.idf,
|
|
13723
|
-
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
13724
|
-
fieldLengths: { [fieldName]: actualFieldLength }
|
|
13725
|
-
});
|
|
13726
|
-
} else {
|
|
13727
|
-
perTerm.set(internalId, {
|
|
13728
|
-
score: termScore,
|
|
13729
|
-
token: match.token,
|
|
13730
|
-
idf: match.idf,
|
|
13731
|
-
termFrequencies: EMPTY_COMPONENTS,
|
|
13732
|
-
fieldLengths: EMPTY_COMPONENTS
|
|
13733
|
-
});
|
|
13921
|
+
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
13922
|
+
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
13923
|
+
return;
|
|
13734
13924
|
}
|
|
13735
|
-
|
|
13925
|
+
perTerm.set(internalId, {
|
|
13926
|
+
score: termScore,
|
|
13927
|
+
token: match.token,
|
|
13928
|
+
idf: match.idf,
|
|
13929
|
+
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
13930
|
+
fieldLengths: { [fieldName]: actualFieldLength }
|
|
13931
|
+
});
|
|
13932
|
+
});
|
|
13736
13933
|
for (const [internalId, contribution] of perTerm) {
|
|
13737
13934
|
const current = best.get(internalId);
|
|
13738
13935
|
if (!current || compareScores(contribution.score, current.score) < 0) {
|
|
@@ -14213,221 +14410,50 @@ function scanPostingBlocks(list, columns2, previous, keepBlocks) {
|
|
|
14213
14410
|
}
|
|
14214
14411
|
run++;
|
|
14215
14412
|
}
|
|
14216
|
-
if (!tombstoned2) {
|
|
14217
|
-
if (run > blockMaxRun) blockMaxRun = run;
|
|
14218
|
-
blockDocs++;
|
|
14219
|
-
}
|
|
14220
|
-
entry += run;
|
|
14221
|
-
if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === length) {
|
|
14222
|
-
entryEnd[blockIndex] = entry;
|
|
14223
|
-
maxTermFrequency[blockIndex] = blockMaxTf;
|
|
14224
|
-
minFieldLength[blockIndex] = blockMinLen === Number.POSITIVE_INFINITY ? ABSENT_FIELD_LENGTH_BOUND : blockMinLen;
|
|
14225
|
-
maxEntriesPerDocument[blockIndex] = blockMaxRun;
|
|
14226
|
-
documentCount[blockIndex] = blockDocs;
|
|
14227
|
-
blockIndex++;
|
|
14228
|
-
blockStart = entry;
|
|
14229
|
-
blockMaxTf = 0;
|
|
14230
|
-
blockMinLen = Number.POSITIVE_INFINITY;
|
|
14231
|
-
blockMaxRun = 0;
|
|
14232
|
-
blockDocs = 0;
|
|
14233
|
-
}
|
|
14234
|
-
}
|
|
14235
|
-
return {
|
|
14236
|
-
structureRevision: list.structureRevision,
|
|
14237
|
-
blockCount: blockIndex,
|
|
14238
|
-
entryEnd,
|
|
14239
|
-
maxTermFrequency,
|
|
14240
|
-
minFieldLength,
|
|
14241
|
-
maxEntriesPerDocument,
|
|
14242
|
-
documentCount
|
|
14243
|
-
};
|
|
14244
|
-
}
|
|
14245
|
-
function blockBoundsFor(list, columns2) {
|
|
14246
|
-
const existing = boundsByList.get(list);
|
|
14247
|
-
if (existing !== void 0 && existing.structureRevision === list.structureRevision) {
|
|
14248
|
-
const covered = entriesCovered(existing);
|
|
14249
|
-
if (covered === list.length) return existing;
|
|
14250
|
-
if (list.ordered && covered < list.length) {
|
|
14251
|
-
const extended = scanPostingBlocks(list, columns2, existing, Math.max(0, existing.blockCount - 1));
|
|
14252
|
-
boundsByList.set(list, extended);
|
|
14253
|
-
return extended;
|
|
14254
|
-
}
|
|
14255
|
-
}
|
|
14256
|
-
const stored = list.storedBlockBounds?.() ?? null;
|
|
14257
|
-
const built2 = stored === null ? scanPostingBlocks(list, columns2, null, 0) : { ...stored, structureRevision: list.structureRevision };
|
|
14258
|
-
boundsByList.set(list, built2);
|
|
14259
|
-
return built2;
|
|
14260
|
-
}
|
|
14261
|
-
|
|
14262
|
-
// src/core/partition/top-k-heap.ts
|
|
14263
|
-
function candidateWorse(a, b) {
|
|
14264
|
-
if (a.score < b.score) return true;
|
|
14265
|
-
if (a.score > b.score) return false;
|
|
14266
|
-
return unorderedCandidateWorse(a, b);
|
|
14267
|
-
}
|
|
14268
|
-
function unorderedCandidateWorse(a, b) {
|
|
14269
|
-
return (compareScores(a.score, b.score) || compareCodePoints(a.externalId, b.externalId)) > 0;
|
|
14270
|
-
}
|
|
14271
|
-
function buildMinHeap(heap) {
|
|
14272
|
-
for (let index = (heap.length >> 1) - 1; index >= 0; index--) {
|
|
14273
|
-
siftDown(heap, index);
|
|
14274
|
-
}
|
|
14275
|
-
}
|
|
14276
|
-
function siftDown(heap, start) {
|
|
14277
|
-
const len = heap.length;
|
|
14278
|
-
let current = start;
|
|
14279
|
-
for (; ; ) {
|
|
14280
|
-
let worst = current;
|
|
14281
|
-
const left = 2 * current + 1;
|
|
14282
|
-
const right = 2 * current + 2;
|
|
14283
|
-
if (left < len && candidateWorse(heap[left], heap[worst])) worst = left;
|
|
14284
|
-
if (right < len && candidateWorse(heap[right], heap[worst])) worst = right;
|
|
14285
|
-
if (worst === current) break;
|
|
14286
|
-
const held = heap[current];
|
|
14287
|
-
heap[current] = heap[worst];
|
|
14288
|
-
heap[worst] = held;
|
|
14289
|
-
current = worst;
|
|
14290
|
-
}
|
|
14291
|
-
}
|
|
14292
|
-
function sortSelection(heap) {
|
|
14293
|
-
heap.sort((a, b) => compareScores(a.score, b.score) || compareCodePoints(a.externalId, b.externalId));
|
|
14294
|
-
}
|
|
14295
|
-
|
|
14296
|
-
// src/core/partition/score-buffer.ts
|
|
14297
|
-
var MAX_GENERATION = 2147483647;
|
|
14298
|
-
function createScoreBuffer(capacity) {
|
|
14299
|
-
return {
|
|
14300
|
-
scores: new Float64Array(capacity),
|
|
14301
|
-
stamps: new Int32Array(capacity),
|
|
14302
|
-
touched: new Int32Array(Math.max(1, Math.min(capacity, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY))),
|
|
14303
|
-
touchedCount: 0,
|
|
14304
|
-
generation: 0
|
|
14305
|
-
};
|
|
14306
|
-
}
|
|
14307
|
-
function beginScoring(buffer, capacity) {
|
|
14308
|
-
if (buffer.scores.length < capacity) {
|
|
14309
|
-
buffer.scores = new Float64Array(capacity);
|
|
14310
|
-
buffer.stamps = new Int32Array(capacity);
|
|
14311
|
-
buffer.generation = 0;
|
|
14312
|
-
}
|
|
14313
|
-
if (buffer.generation >= MAX_GENERATION) {
|
|
14314
|
-
buffer.stamps.fill(0);
|
|
14315
|
-
buffer.generation = 0;
|
|
14316
|
-
}
|
|
14317
|
-
buffer.generation++;
|
|
14318
|
-
buffer.touchedCount = 0;
|
|
14319
|
-
}
|
|
14320
|
-
function appendTouched(buffer, internalId) {
|
|
14321
|
-
if (buffer.touchedCount === buffer.touched.length) {
|
|
14322
|
-
const grown4 = new Int32Array(buffer.touched.length * 2);
|
|
14323
|
-
grown4.set(buffer.touched);
|
|
14324
|
-
buffer.touched = grown4;
|
|
14325
|
-
}
|
|
14326
|
-
buffer.touched[buffer.touchedCount] = internalId;
|
|
14327
|
-
buffer.touchedCount++;
|
|
14328
|
-
}
|
|
14329
|
-
function addScore(buffer, internalId, termScore) {
|
|
14330
|
-
if (buffer.stamps[internalId] === buffer.generation) {
|
|
14331
|
-
buffer.scores[internalId] += termScore;
|
|
14332
|
-
return;
|
|
14333
|
-
}
|
|
14334
|
-
buffer.stamps[internalId] = buffer.generation;
|
|
14335
|
-
buffer.scores[internalId] = termScore;
|
|
14336
|
-
appendTouched(buffer, internalId);
|
|
14337
|
-
}
|
|
14338
|
-
function markCounted(buffer, internalId) {
|
|
14339
|
-
const countedStamp = -buffer.generation;
|
|
14340
|
-
const stamp = buffer.stamps[internalId];
|
|
14341
|
-
if (stamp === buffer.generation || stamp === countedStamp) return false;
|
|
14342
|
-
buffer.stamps[internalId] = countedStamp;
|
|
14343
|
-
return true;
|
|
14344
|
-
}
|
|
14345
|
-
function kthBestScore(buffer, selection, resolver) {
|
|
14346
|
-
const wanted = selection.length;
|
|
14347
|
-
const { touched, touchedCount, scores } = buffer;
|
|
14348
|
-
let size = 0;
|
|
14349
|
-
let unordered = 0;
|
|
14350
|
-
for (let index = 0; index < touchedCount; index++) {
|
|
14351
|
-
const internalId = touched[index];
|
|
14352
|
-
const score = scores[internalId];
|
|
14353
|
-
if (size === wanted && !(score > selection[0])) continue;
|
|
14354
|
-
if (!resolver.isLive(internalId)) continue;
|
|
14355
|
-
let at = 0;
|
|
14356
|
-
if (size < wanted) {
|
|
14357
|
-
if (Number.isNaN(score)) {
|
|
14358
|
-
unordered++;
|
|
14359
|
-
continue;
|
|
14360
|
-
}
|
|
14361
|
-
at = size++;
|
|
14362
|
-
while (at > 0) {
|
|
14363
|
-
const parent = at - 1 >> 1;
|
|
14364
|
-
if (selection[parent] <= score) break;
|
|
14365
|
-
selection[at] = selection[parent];
|
|
14366
|
-
at = parent;
|
|
14367
|
-
}
|
|
14368
|
-
} else {
|
|
14369
|
-
for (; ; ) {
|
|
14370
|
-
const left = 2 * at + 1;
|
|
14371
|
-
if (left >= size) break;
|
|
14372
|
-
const right = left + 1;
|
|
14373
|
-
const child = right < size && selection[right] < selection[left] ? right : left;
|
|
14374
|
-
if (selection[child] >= score) break;
|
|
14375
|
-
selection[at] = selection[child];
|
|
14376
|
-
at = child;
|
|
14377
|
-
}
|
|
14378
|
-
}
|
|
14379
|
-
selection[at] = score;
|
|
14380
|
-
}
|
|
14381
|
-
if (wanted === 0) return Number.NEGATIVE_INFINITY;
|
|
14382
|
-
if (size === wanted) return selection[0];
|
|
14383
|
-
return size + unordered >= wanted ? Number.NaN : Number.NEGATIVE_INFINITY;
|
|
14384
|
-
}
|
|
14385
|
-
function hasScore(buffer, internalId) {
|
|
14386
|
-
return buffer.stamps[internalId] === buffer.generation;
|
|
14387
|
-
}
|
|
14388
|
-
function topKFromBuffer(buffer, k, resolver, components) {
|
|
14389
|
-
const wanted = Number.isFinite(k) ? Math.max(0, Math.floor(k)) : 0;
|
|
14390
|
-
if (wanted <= 0) return [];
|
|
14391
|
-
const heap = [];
|
|
14392
|
-
const { touched, touchedCount, scores } = buffer;
|
|
14393
|
-
let full = false;
|
|
14394
|
-
let threshold = 0;
|
|
14395
|
-
for (let index = 0; index < touchedCount; index++) {
|
|
14396
|
-
const internalId = touched[index];
|
|
14397
|
-
const score = scores[internalId];
|
|
14398
|
-
if (full && (score < threshold || Number.isNaN(score) && !Number.isNaN(threshold))) continue;
|
|
14399
|
-
const externalId = resolver.toExternal(internalId);
|
|
14400
|
-
if (externalId === void 0) continue;
|
|
14401
|
-
const candidate = { internalId, externalId, score };
|
|
14402
|
-
if (!full) {
|
|
14403
|
-
heap.push(candidate);
|
|
14404
|
-
if (heap.length === wanted) {
|
|
14405
|
-
buildMinHeap(heap);
|
|
14406
|
-
full = true;
|
|
14407
|
-
threshold = heap[0].score;
|
|
14408
|
-
}
|
|
14409
|
-
continue;
|
|
14413
|
+
if (!tombstoned2) {
|
|
14414
|
+
if (run > blockMaxRun) blockMaxRun = run;
|
|
14415
|
+
blockDocs++;
|
|
14410
14416
|
}
|
|
14411
|
-
|
|
14412
|
-
|
|
14413
|
-
|
|
14414
|
-
|
|
14417
|
+
entry += run;
|
|
14418
|
+
if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === length) {
|
|
14419
|
+
entryEnd[blockIndex] = entry;
|
|
14420
|
+
maxTermFrequency[blockIndex] = blockMaxTf;
|
|
14421
|
+
minFieldLength[blockIndex] = blockMinLen === Number.POSITIVE_INFINITY ? ABSENT_FIELD_LENGTH_BOUND : blockMinLen;
|
|
14422
|
+
maxEntriesPerDocument[blockIndex] = blockMaxRun;
|
|
14423
|
+
documentCount[blockIndex] = blockDocs;
|
|
14424
|
+
blockIndex++;
|
|
14425
|
+
blockStart = entry;
|
|
14426
|
+
blockMaxTf = 0;
|
|
14427
|
+
blockMinLen = Number.POSITIVE_INFINITY;
|
|
14428
|
+
blockMaxRun = 0;
|
|
14429
|
+
blockDocs = 0;
|
|
14415
14430
|
}
|
|
14416
14431
|
}
|
|
14417
|
-
|
|
14418
|
-
|
|
14419
|
-
|
|
14420
|
-
|
|
14421
|
-
|
|
14422
|
-
|
|
14423
|
-
|
|
14424
|
-
|
|
14425
|
-
|
|
14426
|
-
|
|
14427
|
-
|
|
14428
|
-
|
|
14432
|
+
return {
|
|
14433
|
+
structureRevision: list.structureRevision,
|
|
14434
|
+
blockCount: blockIndex,
|
|
14435
|
+
entryEnd,
|
|
14436
|
+
maxTermFrequency,
|
|
14437
|
+
minFieldLength,
|
|
14438
|
+
maxEntriesPerDocument,
|
|
14439
|
+
documentCount
|
|
14440
|
+
};
|
|
14441
|
+
}
|
|
14442
|
+
function blockBoundsFor(list, columns2) {
|
|
14443
|
+
const existing = boundsByList.get(list);
|
|
14444
|
+
if (existing !== void 0 && existing.structureRevision === list.structureRevision) {
|
|
14445
|
+
const covered = entriesCovered(existing);
|
|
14446
|
+
if (covered === list.length) return existing;
|
|
14447
|
+
if (list.ordered && covered < list.length) {
|
|
14448
|
+
const extended = scanPostingBlocks(list, columns2, existing, Math.max(0, existing.blockCount - 1));
|
|
14449
|
+
boundsByList.set(list, extended);
|
|
14450
|
+
return extended;
|
|
14451
|
+
}
|
|
14429
14452
|
}
|
|
14430
|
-
|
|
14453
|
+
const stored = list.storedBlockBounds?.() ?? null;
|
|
14454
|
+
const built2 = stored === null ? scanPostingBlocks(list, columns2, null, 0) : { ...stored, structureRevision: list.structureRevision };
|
|
14455
|
+
boundsByList.set(list, built2);
|
|
14456
|
+
return built2;
|
|
14431
14457
|
}
|
|
14432
14458
|
|
|
14433
14459
|
// src/core/partition/multi-term-topk.ts
|
|
@@ -15072,10 +15098,14 @@ function searchFulltext(state, params) {
|
|
|
15072
15098
|
}
|
|
15073
15099
|
function scoreGroup(group, only) {
|
|
15074
15100
|
if (group === null) return;
|
|
15075
|
-
|
|
15076
|
-
|
|
15077
|
-
|
|
15101
|
+
if (components !== null) {
|
|
15102
|
+
for (const [internalId, contribution] of computeGroupContributions(groupContext, group)) {
|
|
15103
|
+
if (only !== null && !bitsetHas(only, internalId)) continue;
|
|
15104
|
+
mergeGroupContribution(internalId, contribution);
|
|
15105
|
+
}
|
|
15106
|
+
return;
|
|
15078
15107
|
}
|
|
15108
|
+
addGroupScores(groupContext, group, scoreBuffer, only, coverage);
|
|
15079
15109
|
}
|
|
15080
15110
|
const prunableList = phrases === void 0 ? prunableSingleTermList(params, state.invertedIdx, fieldNames) : null;
|
|
15081
15111
|
if (prunableList !== null && maxResults !== void 0) {
|
|
@@ -19092,11 +19122,13 @@ function assembleFrozenSegment(segmentId, source2, documentSource, sharedSnapsho
|
|
|
19092
19122
|
isTombstoned: (ordinal) => tombstones.has(ordinal),
|
|
19093
19123
|
settleRemovals: counts.settleRemovals
|
|
19094
19124
|
});
|
|
19095
|
-
const
|
|
19096
|
-
|
|
19097
|
-
|
|
19098
|
-
|
|
19125
|
+
const textFieldsOnly = {
|
|
19126
|
+
kind: "fields",
|
|
19127
|
+
include: source2.fieldLengthNames.map((fieldPath) => fieldPath.split(".")),
|
|
19128
|
+
exclude: []
|
|
19099
19129
|
};
|
|
19130
|
+
const textAt2 = (ordinal) => documentSource.readAt(ordinal, textFieldsOnly);
|
|
19131
|
+
const textDocuments = { count: documentSource.count, textFields: source2.fieldLengthNames, documentAt: textAt2 };
|
|
19100
19132
|
const loadSurfaceForms = survivingSurfaceForms === void 0 ? source2.readSurfaceForms : () => survivingSurfaceForms;
|
|
19101
19133
|
const surfaces = createFrozenSurfaceSlot(
|
|
19102
19134
|
loadSurfaceForms,
|
|
@@ -19168,7 +19200,7 @@ function assembleFrozenSegment(segmentId, source2, documentSource, sharedSnapsho
|
|
|
19168
19200
|
source2.fieldNames,
|
|
19169
19201
|
source2.fieldLengthNames,
|
|
19170
19202
|
ordinal,
|
|
19171
|
-
|
|
19203
|
+
textAt2(ordinal),
|
|
19172
19204
|
analysis,
|
|
19173
19205
|
surfaces,
|
|
19174
19206
|
partialWords
|
|
@@ -20105,11 +20137,12 @@ function bindSegmentCodecHost(_host, _msgpack) {
|
|
|
20105
20137
|
// src/serialization/msgpack-without-fields.ts
|
|
20106
20138
|
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
20107
20139
|
var keyDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
20140
|
+
var walkedHeader = { headerBytes: 0, bodyBytes: 0, children: 0 };
|
|
20108
20141
|
function endOfValue(view, start) {
|
|
20109
20142
|
let offset = start;
|
|
20110
20143
|
let pending = 1;
|
|
20111
20144
|
while (pending > 0) {
|
|
20112
|
-
const header = headerAt(view, offset);
|
|
20145
|
+
const header = headerAt(view, offset, walkedHeader);
|
|
20113
20146
|
if (header === null) return null;
|
|
20114
20147
|
offset += header.headerBytes + header.bodyBytes;
|
|
20115
20148
|
if (offset > view.byteLength) return null;
|
|
@@ -20178,6 +20211,8 @@ function decodeMapWithoutFields(bytes, skippedFields) {
|
|
|
20178
20211
|
var encoder6 = new TextEncoder();
|
|
20179
20212
|
var PROTO_KEY_BYTES = encoder6.encode("__proto__");
|
|
20180
20213
|
var MISSING = { found: false, value: void 0 };
|
|
20214
|
+
var mapHeader = { headerBytes: 0, bodyBytes: 0, children: 0 };
|
|
20215
|
+
var keyHeader = { headerBytes: 0, bodyBytes: 0, children: 0 };
|
|
20181
20216
|
var pathDecoder = null;
|
|
20182
20217
|
function encodePathSegments(segments2) {
|
|
20183
20218
|
if (segments2.length === 0) return null;
|
|
@@ -20204,14 +20239,12 @@ function sameBytes(bytes, start, end, target) {
|
|
|
20204
20239
|
}
|
|
20205
20240
|
return true;
|
|
20206
20241
|
}
|
|
20207
|
-
function
|
|
20208
|
-
if (!isStringTag(view.getUint8(offset))) return
|
|
20209
|
-
|
|
20210
|
-
|
|
20211
|
-
const start = offset + header.headerBytes;
|
|
20212
|
-
return { start, end: start + header.bodyBytes };
|
|
20242
|
+
function keyEndAt(view, offset) {
|
|
20243
|
+
if (!isStringTag(view.getUint8(offset))) return -1;
|
|
20244
|
+
if (headerAt(view, offset, keyHeader) === null) return -1;
|
|
20245
|
+
return offset + keyHeader.headerBytes + keyHeader.bodyBytes;
|
|
20213
20246
|
}
|
|
20214
|
-
function readEncodedPath(bytes, segments2, arrays) {
|
|
20247
|
+
function readEncodedPath(bytes, segments2, arrays, foundRead) {
|
|
20215
20248
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
20216
20249
|
let start = 0;
|
|
20217
20250
|
let end = bytes.length;
|
|
@@ -20223,20 +20256,21 @@ function readEncodedPath(bytes, segments2, arrays) {
|
|
|
20223
20256
|
if (isArrayTag(tag2)) return arrays === "missing" ? MISSING : null;
|
|
20224
20257
|
return isScalarTag(tag2) ? MISSING : null;
|
|
20225
20258
|
}
|
|
20226
|
-
|
|
20227
|
-
|
|
20228
|
-
let offset = start +
|
|
20259
|
+
if (headerAt(view, start, mapHeader) === null) return null;
|
|
20260
|
+
const pairs = mapHeader.children / 2;
|
|
20261
|
+
let offset = start + mapHeader.headerBytes;
|
|
20229
20262
|
let valueStart = -1;
|
|
20230
20263
|
let valueEnd = -1;
|
|
20231
|
-
for (let pair = 0; pair <
|
|
20264
|
+
for (let pair = 0; pair < pairs; pair++) {
|
|
20232
20265
|
if (offset >= end) return null;
|
|
20233
|
-
const
|
|
20234
|
-
if (
|
|
20235
|
-
|
|
20236
|
-
|
|
20266
|
+
const keyEnd = keyEndAt(view, offset);
|
|
20267
|
+
if (keyEnd < 0 || keyEnd > end) return null;
|
|
20268
|
+
const keyStart = offset + keyHeader.headerBytes;
|
|
20269
|
+
if (sameBytes(bytes, keyStart, keyEnd, PROTO_KEY_BYTES)) return null;
|
|
20270
|
+
const next = endOfValue(view, keyEnd);
|
|
20237
20271
|
if (next === null || next > end) return null;
|
|
20238
|
-
if (sameBytes(bytes,
|
|
20239
|
-
valueStart =
|
|
20272
|
+
if (sameBytes(bytes, keyStart, keyEnd, segments2[depth])) {
|
|
20273
|
+
valueStart = keyEnd;
|
|
20240
20274
|
valueEnd = next;
|
|
20241
20275
|
}
|
|
20242
20276
|
offset = next;
|
|
@@ -20248,7 +20282,8 @@ function readEncodedPath(bytes, segments2, arrays) {
|
|
|
20248
20282
|
}
|
|
20249
20283
|
try {
|
|
20250
20284
|
pathDecoder ??= new Decoder();
|
|
20251
|
-
|
|
20285
|
+
foundRead.value = pathDecoder.decode(bytes.subarray(start, end));
|
|
20286
|
+
return foundRead;
|
|
20252
20287
|
} catch {
|
|
20253
20288
|
return null;
|
|
20254
20289
|
}
|
|
@@ -20344,20 +20379,35 @@ function wrapEncodedDocumentTable(data) {
|
|
|
20344
20379
|
}
|
|
20345
20380
|
function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
|
|
20346
20381
|
const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
|
|
20347
|
-
const touchesDiscarded = (path) =>
|
|
20382
|
+
const touchesDiscarded = (path) => {
|
|
20383
|
+
for (const discardedPath of discarded) {
|
|
20384
|
+
if (sharesPrefix(discardedPath, path)) return true;
|
|
20385
|
+
}
|
|
20386
|
+
return false;
|
|
20387
|
+
};
|
|
20348
20388
|
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
|
|
20349
|
-
function projectedPathAt(ordinal, path) {
|
|
20389
|
+
function projectedPathAt(ordinal, path, result) {
|
|
20350
20390
|
const segments2 = encodedProjectionPathOf(path);
|
|
20351
20391
|
if (segments2 === null) return { found: false, value: void 0 };
|
|
20352
|
-
const read = touchesDiscarded(path) ? null : readEncodedPath(bytesAt(ordinal), segments2, "missing");
|
|
20392
|
+
const read = touchesDiscarded(path) ? null : readEncodedPath(bytesAt(ordinal), segments2, "missing", result);
|
|
20353
20393
|
if (read === null) return readProjectedPath(docAt(ordinal), path);
|
|
20354
|
-
|
|
20394
|
+
if (!read.found) return read;
|
|
20395
|
+
read.value = valueWithOwnBuffers(read.value);
|
|
20396
|
+
return read;
|
|
20397
|
+
}
|
|
20398
|
+
function projectedAt(ordinal, projection) {
|
|
20399
|
+
if (projection === void 0 || projection.kind === "full") return docAt(ordinal);
|
|
20400
|
+
if (projection.kind === "none") return {};
|
|
20401
|
+
if (projection.include === null) return cloneProjected(docAt(ordinal), projection);
|
|
20402
|
+
const result = { found: true, value: void 0 };
|
|
20403
|
+
return projectIncludedPaths(projection.include, projection.exclude, (path) => projectedPathAt(ordinal, path, result));
|
|
20355
20404
|
}
|
|
20356
20405
|
return {
|
|
20357
20406
|
count,
|
|
20358
20407
|
byteLength,
|
|
20359
20408
|
docAt,
|
|
20360
|
-
readAt(ordinal) {
|
|
20409
|
+
readAt(ordinal, fields) {
|
|
20410
|
+
if (fields !== void 0) return projectedAt(ordinal, fields);
|
|
20361
20411
|
return decodeStoredDocument(bytesAt(ordinal), discardedFields);
|
|
20362
20412
|
},
|
|
20363
20413
|
fieldReader(fieldPath) {
|
|
@@ -20366,16 +20416,15 @@ function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields
|
|
|
20366
20416
|
if (segments2 === null) return () => void 0;
|
|
20367
20417
|
const decodesDocument = touchesDiscarded(path);
|
|
20368
20418
|
return (ordinal) => {
|
|
20369
|
-
const read = decodesDocument ? null : readEncodedPath(bytesAt(ordinal), segments2, "decode-document");
|
|
20419
|
+
const read = decodesDocument ? null : readEncodedPath(bytesAt(ordinal), segments2, "decode-document", { found: true, value: void 0 });
|
|
20370
20420
|
if (read !== null) return valueWithOwnBuffers(read.value);
|
|
20371
20421
|
return getNestedValue(docAt(ordinal), fieldPath);
|
|
20372
20422
|
};
|
|
20373
20423
|
},
|
|
20374
|
-
projectedAt
|
|
20375
|
-
|
|
20376
|
-
|
|
20377
|
-
|
|
20378
|
-
return projectIncludedPaths(projection.include, projection.exclude, (path) => projectedPathAt(ordinal, path));
|
|
20424
|
+
projectedAt,
|
|
20425
|
+
copyForRollbackAt(ordinal) {
|
|
20426
|
+
const bytes = bytesAt(ordinal).slice();
|
|
20427
|
+
return () => documentWithOwnBuffers(decodeStoredDocument(bytes, discardedFields));
|
|
20379
20428
|
}
|
|
20380
20429
|
};
|
|
20381
20430
|
}
|
|
@@ -21549,7 +21598,7 @@ function hasLanguage(name) {
|
|
|
21549
21598
|
|
|
21550
21599
|
// src/narsil/restore-queue-code.ts
|
|
21551
21600
|
var restoreQueue = codeLoader(async () => {
|
|
21552
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
21601
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-Q76Q4S35.mjs')]);
|
|
21553
21602
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
21554
21603
|
return code7;
|
|
21555
21604
|
}, "restore");
|
|
@@ -22103,6 +22152,32 @@ function createSharedFrozenSegment(snapshot, survivingSurfaceForms) {
|
|
|
22103
22152
|
);
|
|
22104
22153
|
}
|
|
22105
22154
|
|
|
22155
|
+
// src/partitioning/manager/nested-values.ts
|
|
22156
|
+
function setNestedValue(obj, path, value) {
|
|
22157
|
+
if (!path.includes(".")) {
|
|
22158
|
+
obj[path] = value;
|
|
22159
|
+
return;
|
|
22160
|
+
}
|
|
22161
|
+
const segments2 = path.split(".");
|
|
22162
|
+
let current = obj;
|
|
22163
|
+
for (let i = 0; i < segments2.length - 1; i++) {
|
|
22164
|
+
let next = current[segments2[i]];
|
|
22165
|
+
if (next === null || next === void 0 || typeof next !== "object") {
|
|
22166
|
+
next = {};
|
|
22167
|
+
current[segments2[i]] = next;
|
|
22168
|
+
}
|
|
22169
|
+
current = next;
|
|
22170
|
+
}
|
|
22171
|
+
current[segments2[segments2.length - 1]] = value;
|
|
22172
|
+
}
|
|
22173
|
+
function copyVectorsInto(document, docId, vectorIndexes, projection) {
|
|
22174
|
+
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
22175
|
+
if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
|
|
22176
|
+
const vector = vectorIndex.getVector(docId);
|
|
22177
|
+
if (vector) setNestedValue(document, fieldPath, vector);
|
|
22178
|
+
}
|
|
22179
|
+
}
|
|
22180
|
+
|
|
22106
22181
|
// src/engine/array-positions.ts
|
|
22107
22182
|
function holdsStringArrayField(schema) {
|
|
22108
22183
|
return schemaFieldsOf(schema).some((field) => field.base === "string[]");
|
|
@@ -22126,7 +22201,7 @@ var loadEmbedCode = embedCode.load;
|
|
|
22126
22201
|
|
|
22127
22202
|
// src/engine/optional-writes-code.ts
|
|
22128
22203
|
var optionalWrites = codeLoader(async () => {
|
|
22129
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
22204
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-CSULLISE.mjs')]);
|
|
22130
22205
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
22131
22206
|
return code7;
|
|
22132
22207
|
}, "write");
|
|
@@ -22561,6 +22636,51 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
22561
22636
|
function bindOptionalWritesHost(_host, _segmentCodec) {
|
|
22562
22637
|
}
|
|
22563
22638
|
|
|
22639
|
+
// src/engine/mutations/batch-shutdown.ts
|
|
22640
|
+
function isFailureThatShutdownCauses(code7) {
|
|
22641
|
+
return code7 === ErrorCodes.INSTANCE_SHUT_DOWN || code7 === ErrorCodes.INDEX_NOT_FOUND || code7 === ErrorCodes.EMBEDDING_FAILED;
|
|
22642
|
+
}
|
|
22643
|
+
function shutDownStarted(ctx) {
|
|
22644
|
+
return ctx.abortController.signal.aborted;
|
|
22645
|
+
}
|
|
22646
|
+
function shutDownError() {
|
|
22647
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
22648
|
+
}
|
|
22649
|
+
function causedByShutdown(ctx, err) {
|
|
22650
|
+
const { signal } = ctx.abortController;
|
|
22651
|
+
if (!signal.aborted) return false;
|
|
22652
|
+
if (err === signal.reason) return true;
|
|
22653
|
+
if (err instanceof NarsilError) return isFailureThatShutdownCauses(err.code);
|
|
22654
|
+
return err instanceof Error && err.name === "AbortError";
|
|
22655
|
+
}
|
|
22656
|
+
function batchFailureOf(ctx, err, fallbackCode) {
|
|
22657
|
+
if (causedByShutdown(ctx, err)) {
|
|
22658
|
+
return err instanceof NarsilError && err.code === ErrorCodes.INSTANCE_SHUT_DOWN ? err : shutDownError();
|
|
22659
|
+
}
|
|
22660
|
+
return err instanceof NarsilError ? err : new NarsilError(fallbackCode, String(err));
|
|
22661
|
+
}
|
|
22662
|
+
function docIdOf(entry) {
|
|
22663
|
+
return entry.docId;
|
|
22664
|
+
}
|
|
22665
|
+
function failAsShutDown(failed, entries, reportedDocIdOf2, start = 0, end = entries.length) {
|
|
22666
|
+
if (start >= end) return;
|
|
22667
|
+
const error = shutDownError();
|
|
22668
|
+
for (let i = start; i < end; i++) failed.push({ docId: reportedDocIdOf2(entries[i]), error });
|
|
22669
|
+
}
|
|
22670
|
+
function applyUnlessShutDown(ctx, apply) {
|
|
22671
|
+
return () => {
|
|
22672
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
22673
|
+
return apply();
|
|
22674
|
+
};
|
|
22675
|
+
}
|
|
22676
|
+
async function unlessShutDown(ctx, work) {
|
|
22677
|
+
try {
|
|
22678
|
+
await work();
|
|
22679
|
+
} catch (err) {
|
|
22680
|
+
if (!shutDownStarted(ctx)) throw err;
|
|
22681
|
+
}
|
|
22682
|
+
}
|
|
22683
|
+
|
|
22564
22684
|
// src/engine/mutations/batch-admissions.ts
|
|
22565
22685
|
function createPendingAdmissions() {
|
|
22566
22686
|
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
@@ -22571,8 +22691,11 @@ function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
|
22571
22691
|
const partitionId = manager.routePartition(docId);
|
|
22572
22692
|
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
22573
22693
|
}
|
|
22574
|
-
function asBatchInsertError(err) {
|
|
22575
|
-
return
|
|
22694
|
+
function asBatchInsertError(ctx, err) {
|
|
22695
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_VALIDATION_FAILED);
|
|
22696
|
+
}
|
|
22697
|
+
function providedDocIdOrEmpty(document) {
|
|
22698
|
+
return providedDocId(document) ?? "";
|
|
22576
22699
|
}
|
|
22577
22700
|
|
|
22578
22701
|
// src/engine/mutations/outside-hold.ts
|
|
@@ -22802,7 +22925,11 @@ async function prepareInsertDocuments(ctx, indexName, documents2, start, end, ho
|
|
|
22802
22925
|
const pending = [];
|
|
22803
22926
|
for (let i = start; i < end; i++) {
|
|
22804
22927
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
22805
|
-
if (ctx
|
|
22928
|
+
if (shutDownStarted(ctx)) {
|
|
22929
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
22930
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty, i, end);
|
|
22931
|
+
return null;
|
|
22932
|
+
}
|
|
22806
22933
|
const document = documents2[i];
|
|
22807
22934
|
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
22808
22935
|
try {
|
|
@@ -22811,17 +22938,26 @@ async function prepareInsertDocuments(ctx, indexName, documents2, start, end, ho
|
|
|
22811
22938
|
validateDocId(docId);
|
|
22812
22939
|
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
22813
22940
|
} catch (err) {
|
|
22814
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
22941
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
22815
22942
|
}
|
|
22816
22943
|
}
|
|
22817
|
-
|
|
22818
|
-
|
|
22819
|
-
|
|
22820
|
-
|
|
22821
|
-
|
|
22822
|
-
|
|
22823
|
-
|
|
22824
|
-
|
|
22944
|
+
let prepared;
|
|
22945
|
+
try {
|
|
22946
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
22947
|
+
prepared = await prepareDocumentWrites(
|
|
22948
|
+
ctx,
|
|
22949
|
+
indexName,
|
|
22950
|
+
pending.map((item) => item.preparation),
|
|
22951
|
+
hold,
|
|
22952
|
+
true,
|
|
22953
|
+
earlier
|
|
22954
|
+
);
|
|
22955
|
+
} catch (err) {
|
|
22956
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
22957
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
22958
|
+
return null;
|
|
22959
|
+
}
|
|
22960
|
+
const { entry, embedding } = prepared;
|
|
22825
22961
|
const candidates2 = [];
|
|
22826
22962
|
for (const item of pending) {
|
|
22827
22963
|
const { document, failure } = item.preparation;
|
|
@@ -22829,22 +22965,19 @@ async function prepareInsertDocuments(ctx, indexName, documents2, start, end, ho
|
|
|
22829
22965
|
candidates2.push({ docId: item.docId, document });
|
|
22830
22966
|
continue;
|
|
22831
22967
|
}
|
|
22832
|
-
const docId = item.refusedByHook ? item.docId :
|
|
22833
|
-
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
22968
|
+
const docId = item.refusedByHook ? item.docId : providedDocIdOrEmpty(document);
|
|
22969
|
+
failed.push({ docId, error: asBatchInsertError(ctx, failure) });
|
|
22834
22970
|
}
|
|
22835
22971
|
return { entry, candidates: candidates2, embedding };
|
|
22836
22972
|
}
|
|
22837
22973
|
async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
22838
|
-
const
|
|
22839
|
-
|
|
22840
|
-
|
|
22841
|
-
|
|
22842
|
-
|
|
22843
|
-
|
|
22844
|
-
|
|
22845
|
-
true,
|
|
22846
|
-
failed
|
|
22847
|
-
);
|
|
22974
|
+
const prepared = await prepareInsertDocuments(ctx, indexName, documents2, 0, documents2.length, hold, true, failed);
|
|
22975
|
+
if (prepared === null) return { admitted: [], embedding: null };
|
|
22976
|
+
const { entry, candidates: candidates2, embedding } = prepared;
|
|
22977
|
+
if (shutDownStarted(ctx)) {
|
|
22978
|
+
failAsShutDown(failed, candidates2, docIdOf);
|
|
22979
|
+
return { admitted: [], embedding };
|
|
22980
|
+
}
|
|
22848
22981
|
const manager = ctx.requireManager(indexName);
|
|
22849
22982
|
const vecIndexes = manager.getVectorIndexes();
|
|
22850
22983
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
@@ -22854,7 +22987,11 @@ async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
|
22854
22987
|
const pending = createPendingAdmissions();
|
|
22855
22988
|
for (let position = 0; position < candidates2.length; position++) {
|
|
22856
22989
|
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
22857
|
-
if (ctx
|
|
22990
|
+
if (shutDownStarted(ctx)) {
|
|
22991
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
22992
|
+
failAsShutDown(failed, candidates2, docIdOf, position);
|
|
22993
|
+
return { admitted: [], embedding };
|
|
22994
|
+
}
|
|
22858
22995
|
const { docId, document } = candidates2[position];
|
|
22859
22996
|
try {
|
|
22860
22997
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
@@ -22876,7 +23013,7 @@ async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
|
22876
23013
|
recordPendingAdmission(ctx, pending, manager, docId);
|
|
22877
23014
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
22878
23015
|
} catch (err) {
|
|
22879
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
23016
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
22880
23017
|
}
|
|
22881
23018
|
}
|
|
22882
23019
|
return { admitted, embedding };
|
|
@@ -22941,7 +23078,7 @@ async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
22941
23078
|
await manager.publication.write(
|
|
22942
23079
|
() => changeIndexOutsideExternalWaits(async () => {
|
|
22943
23080
|
if (manager.has(docId)) return;
|
|
22944
|
-
const previous = prepareDocumentVectors(restoreDoc, ctx.requireIndex(indexName).vectorFieldPaths);
|
|
23081
|
+
const previous = prepareDocumentVectors(restoreDoc(), ctx.requireIndex(indexName).vectorFieldPaths);
|
|
22945
23082
|
manager.insert(docId, previous.partitionDoc, {
|
|
22946
23083
|
preparedText: { after: preparedText?.before }
|
|
22947
23084
|
});
|
|
@@ -23026,9 +23163,9 @@ async function recordChunk(ctx, indexName, mutations, applies) {
|
|
|
23026
23163
|
function applyInsertDocument(ctx, indexName, doc, options, progress) {
|
|
23027
23164
|
const manager = ctx.requireManager(indexName);
|
|
23028
23165
|
const insertIntoIndex = () => {
|
|
23029
|
-
if (ctx
|
|
23166
|
+
if (shutDownStarted(ctx)) {
|
|
23030
23167
|
progress.skipped = true;
|
|
23031
|
-
throw
|
|
23168
|
+
throw shutDownError();
|
|
23032
23169
|
}
|
|
23033
23170
|
admitInsert(ctx, indexName, manager, doc.docId);
|
|
23034
23171
|
const queued = ctx.bufferIfRebalancing(indexName, {
|
|
@@ -23084,11 +23221,11 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
23084
23221
|
const progress = docs.map(() => ({ inserted: false, buffered: false, skipped: false }));
|
|
23085
23222
|
const applies = docs.map((doc, i) => applyInsertDocument(ctx, indexName, doc, options, progress[i]));
|
|
23086
23223
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
23087
|
-
await ctx.orchestrator.settlePublications(indexName);
|
|
23224
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName));
|
|
23088
23225
|
const applications = [];
|
|
23089
23226
|
for (let i = 0; i < docs.length; i++) {
|
|
23090
23227
|
if (progress[i].skipped) {
|
|
23091
|
-
applications.push({ status: "
|
|
23228
|
+
applications.push({ status: "failed", error: shutDownError() });
|
|
23092
23229
|
continue;
|
|
23093
23230
|
}
|
|
23094
23231
|
const failure = failures[i];
|
|
@@ -23121,27 +23258,33 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
23121
23258
|
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
23122
23259
|
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
23123
23260
|
const manager = ctx.executor.getManager(indexName);
|
|
23124
|
-
await
|
|
23125
|
-
|
|
23126
|
-
|
|
23127
|
-
|
|
23128
|
-
|
|
23129
|
-
|
|
23261
|
+
await unlessShutDown(
|
|
23262
|
+
ctx,
|
|
23263
|
+
() => runOutside(hold, async () => {
|
|
23264
|
+
for (const doc of inserted) {
|
|
23265
|
+
try {
|
|
23266
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
23267
|
+
} catch (err) {
|
|
23268
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23269
|
+
}
|
|
23130
23270
|
}
|
|
23131
|
-
}
|
|
23132
|
-
|
|
23133
|
-
return ctx.executor.getManager(indexName) !== manager;
|
|
23271
|
+
})
|
|
23272
|
+
);
|
|
23273
|
+
return shutDownStarted(ctx) || ctx.executor.getManager(indexName) !== manager;
|
|
23134
23274
|
}
|
|
23135
23275
|
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
23136
23276
|
const succeeded = [];
|
|
23137
23277
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23138
23278
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23139
|
-
if (ctx
|
|
23279
|
+
if (shutDownStarted(ctx)) {
|
|
23280
|
+
failAsShutDown(failed, admitted, docIdOf, chunkStart);
|
|
23281
|
+
break;
|
|
23282
|
+
}
|
|
23140
23283
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
23141
23284
|
try {
|
|
23142
23285
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
23143
23286
|
} catch (err) {
|
|
23144
|
-
const error = asBatchInsertError(err);
|
|
23287
|
+
const error = asBatchInsertError(ctx, err);
|
|
23145
23288
|
for (let i = chunkStart; i < admitted.length; i++) failed.push({ docId: admitted[i].docId, error });
|
|
23146
23289
|
break;
|
|
23147
23290
|
}
|
|
@@ -23150,9 +23293,8 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
23150
23293
|
for (let i = 0; i < chunk.length; i++) {
|
|
23151
23294
|
const doc = chunk[i];
|
|
23152
23295
|
const application = applications[i];
|
|
23153
|
-
if (application.status === "skipped") continue;
|
|
23154
23296
|
if (application.status === "failed") {
|
|
23155
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
23297
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
23156
23298
|
continue;
|
|
23157
23299
|
}
|
|
23158
23300
|
succeeded.push(doc.docId);
|
|
@@ -23167,7 +23309,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
23167
23309
|
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
23168
23310
|
}
|
|
23169
23311
|
if (applied < admitted.length) {
|
|
23170
|
-
ctx.checkHeapPressure(indexName);
|
|
23312
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
23171
23313
|
await new Promise((r) => setTimeout(r, 0));
|
|
23172
23314
|
}
|
|
23173
23315
|
}
|
|
@@ -23176,7 +23318,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
23176
23318
|
|
|
23177
23319
|
// src/engine/mutations/insert-batch-chunks.ts
|
|
23178
23320
|
async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
23179
|
-
const
|
|
23321
|
+
const prepared = await prepareInsertDocuments(
|
|
23180
23322
|
ctx,
|
|
23181
23323
|
indexName,
|
|
23182
23324
|
documents2,
|
|
@@ -23187,13 +23329,24 @@ async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold
|
|
|
23187
23329
|
failed,
|
|
23188
23330
|
earlier
|
|
23189
23331
|
);
|
|
23332
|
+
if (prepared === null) return [];
|
|
23333
|
+
const { entry, candidates: candidates2 } = prepared;
|
|
23334
|
+
const admitted = [];
|
|
23335
|
+
if (shutDownStarted(ctx)) {
|
|
23336
|
+
failAsShutDown(failed, candidates2, docIdOf);
|
|
23337
|
+
return admitted;
|
|
23338
|
+
}
|
|
23190
23339
|
const manager = ctx.requireManager(indexName);
|
|
23191
23340
|
const vecIndexes = manager.getVectorIndexes();
|
|
23192
23341
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
23193
|
-
|
|
23194
|
-
for (const { docId, document } of candidates2) {
|
|
23342
|
+
for (let position = 0; position < candidates2.length; position++) {
|
|
23195
23343
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23196
|
-
if (ctx
|
|
23344
|
+
if (shutDownStarted(ctx)) {
|
|
23345
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
23346
|
+
failAsShutDown(failed, candidates2, docIdOf, position);
|
|
23347
|
+
return [];
|
|
23348
|
+
}
|
|
23349
|
+
const { docId, document } = candidates2[position];
|
|
23197
23350
|
try {
|
|
23198
23351
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
23199
23352
|
document,
|
|
@@ -23205,7 +23358,7 @@ async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold
|
|
|
23205
23358
|
admitInsert(ctx, indexName, manager, docId);
|
|
23206
23359
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
23207
23360
|
} catch (err) {
|
|
23208
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
23361
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
23209
23362
|
}
|
|
23210
23363
|
}
|
|
23211
23364
|
return admitted;
|
|
@@ -23214,17 +23367,25 @@ async function insertInChunks(ctx, indexName, documents2, options, hold, hooked,
|
|
|
23214
23367
|
const succeeded = [];
|
|
23215
23368
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23216
23369
|
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23217
|
-
if (ctx
|
|
23370
|
+
if (shutDownStarted(ctx)) {
|
|
23371
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty, chunkStart);
|
|
23372
|
+
break;
|
|
23373
|
+
}
|
|
23218
23374
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
23219
23375
|
const earlier = embeddingOfDocuments(embedded2, documents2.slice(chunkStart, chunkEnd));
|
|
23220
23376
|
const prepared = await admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
23377
|
+
if (shutDownStarted(ctx)) {
|
|
23378
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
23379
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty, chunkEnd);
|
|
23380
|
+
break;
|
|
23381
|
+
}
|
|
23221
23382
|
try {
|
|
23222
23383
|
if (prepared.length > 0) ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
23223
23384
|
} catch (err) {
|
|
23224
|
-
const error = asBatchInsertError(err);
|
|
23385
|
+
const error = asBatchInsertError(ctx, err);
|
|
23225
23386
|
for (const doc of prepared) failed.push({ docId: doc.docId, error });
|
|
23226
23387
|
for (let i = chunkEnd; i < documents2.length; i++) {
|
|
23227
|
-
failed.push({ docId:
|
|
23388
|
+
failed.push({ docId: providedDocIdOrEmpty(documents2[i]), error });
|
|
23228
23389
|
}
|
|
23229
23390
|
break;
|
|
23230
23391
|
}
|
|
@@ -23234,9 +23395,8 @@ async function insertInChunks(ctx, indexName, documents2, options, hold, hooked,
|
|
|
23234
23395
|
for (let i = 0; i < prepared.length; i++) {
|
|
23235
23396
|
const doc = prepared[i];
|
|
23236
23397
|
const application = applications[i];
|
|
23237
|
-
if (application.status === "skipped") continue;
|
|
23238
23398
|
if (application.status === "failed") {
|
|
23239
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
23399
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
23240
23400
|
continue;
|
|
23241
23401
|
}
|
|
23242
23402
|
succeeded.push(doc.docId);
|
|
@@ -23247,21 +23407,24 @@ async function insertInChunks(ctx, indexName, documents2, options, hold, hooked,
|
|
|
23247
23407
|
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
23248
23408
|
}
|
|
23249
23409
|
if (chunkEnd < documents2.length) {
|
|
23250
|
-
ctx.checkHeapPressure(indexName);
|
|
23410
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
23251
23411
|
await new Promise((r) => setTimeout(r, 0));
|
|
23252
23412
|
}
|
|
23253
23413
|
}
|
|
23254
23414
|
return { succeeded, touchedVectorFields };
|
|
23255
23415
|
}
|
|
23256
23416
|
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
23417
|
+
if (shutDownStarted(ctx)) return;
|
|
23257
23418
|
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
23258
23419
|
for (const fieldPath of inserts.touchedVectorFields) {
|
|
23259
23420
|
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
23260
23421
|
}
|
|
23261
23422
|
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
23262
23423
|
ctx.checkHeapPressure(indexName);
|
|
23263
|
-
await ctx
|
|
23264
|
-
|
|
23424
|
+
await unlessShutDown(ctx, async () => {
|
|
23425
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
23426
|
+
if (options?.wait === true && !shutDownStarted(ctx)) await awaitWriteVisibility(ctx, indexName);
|
|
23427
|
+
});
|
|
23265
23428
|
}
|
|
23266
23429
|
|
|
23267
23430
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -23270,7 +23433,7 @@ function applySegment(ctx, indexName, attachment, doc, options) {
|
|
|
23270
23433
|
const publish2 = async () => {
|
|
23271
23434
|
if (attachment.failure !== void 0) throw attachment.failure;
|
|
23272
23435
|
if (attachment.attached) return;
|
|
23273
|
-
if (ctx
|
|
23436
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
23274
23437
|
if (ctx.isRebalancing(indexName) || manager.partitionCount !== attachment.partitionCount) {
|
|
23275
23438
|
attachment.individual = true;
|
|
23276
23439
|
}
|
|
@@ -23424,6 +23587,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23424
23587
|
for (let index = 0; index < chunks.length; index++) {
|
|
23425
23588
|
const { docs, attachmentOf } = chunks[index];
|
|
23426
23589
|
try {
|
|
23590
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
23427
23591
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
23428
23592
|
} catch (error) {
|
|
23429
23593
|
for (const later of chunks.slice(index)) {
|
|
@@ -23437,7 +23601,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23437
23601
|
return applySegment(ctx, indexName, attachment, doc, options);
|
|
23438
23602
|
});
|
|
23439
23603
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
23440
|
-
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
23604
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName, "segments"));
|
|
23441
23605
|
const inserted = [];
|
|
23442
23606
|
for (let i = 0; i < docs.length; i++) {
|
|
23443
23607
|
const doc = docs[i];
|
|
@@ -23478,7 +23642,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23478
23642
|
break;
|
|
23479
23643
|
}
|
|
23480
23644
|
if (index + 1 < chunks.length) {
|
|
23481
|
-
ctx.checkHeapPressure(indexName);
|
|
23645
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
23482
23646
|
await new Promise((r) => setTimeout(r, 0));
|
|
23483
23647
|
}
|
|
23484
23648
|
}
|
|
@@ -23489,7 +23653,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23489
23653
|
continue;
|
|
23490
23654
|
}
|
|
23491
23655
|
if (!errorOf.has(doc.docId)) continue;
|
|
23492
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
23656
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, errorOf.get(doc.docId)) });
|
|
23493
23657
|
}
|
|
23494
23658
|
return { succeeded, touchedVectorFields, unrecorded };
|
|
23495
23659
|
}
|
|
@@ -23503,8 +23667,11 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
23503
23667
|
);
|
|
23504
23668
|
}
|
|
23505
23669
|
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
23506
|
-
|
|
23507
|
-
|
|
23670
|
+
const nothingIngested = () => ({ succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] });
|
|
23671
|
+
if (admitted.length === 0) return nothingIngested();
|
|
23672
|
+
if (shutDownStarted(ctx)) {
|
|
23673
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
23674
|
+
return nothingIngested();
|
|
23508
23675
|
}
|
|
23509
23676
|
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
23510
23677
|
const manager = ctx.requireManager(indexName);
|
|
@@ -23522,6 +23689,10 @@ async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
|
23522
23689
|
} catch {
|
|
23523
23690
|
built2 = null;
|
|
23524
23691
|
}
|
|
23692
|
+
if (shutDownStarted(ctx)) {
|
|
23693
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
23694
|
+
return nothingIngested();
|
|
23695
|
+
}
|
|
23525
23696
|
if (built2 === null) {
|
|
23526
23697
|
return applyIndividually();
|
|
23527
23698
|
}
|
|
@@ -23572,7 +23743,9 @@ async function insertBatchViaSegments(ctx, indexName, documents2, options, hold)
|
|
|
23572
23743
|
const failed = [];
|
|
23573
23744
|
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents2, failed, hold);
|
|
23574
23745
|
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
23575
|
-
if (
|
|
23746
|
+
if (shutDownStarted(ctx)) {
|
|
23747
|
+
failAsShutDown(failed, outcome.unrecorded, docIdOf);
|
|
23748
|
+
} else if (outcome.unrecorded.length > 0) {
|
|
23576
23749
|
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
23577
23750
|
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
23578
23751
|
outcome.succeeded.push(...rest.succeeded);
|
|
@@ -23587,20 +23760,54 @@ async function insertDocumentBatch(ctx, indexName, documents2, options, hold) {
|
|
|
23587
23760
|
ctx.guardShutdown();
|
|
23588
23761
|
ctx.requireIndex(indexName);
|
|
23589
23762
|
validateBatchList(documents2, "documents");
|
|
23590
|
-
|
|
23763
|
+
const failed = [];
|
|
23764
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.scaleOutBeforeBatch(indexName, documents2.length));
|
|
23765
|
+
if (shutDownStarted(ctx)) {
|
|
23766
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty);
|
|
23767
|
+
return { succeeded: [], failed };
|
|
23768
|
+
}
|
|
23591
23769
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
23592
23770
|
if (documents2.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
23593
23771
|
return insertBatchViaSegments(ctx, indexName, documents2, options, hold);
|
|
23594
23772
|
}
|
|
23595
|
-
const failed = [];
|
|
23596
23773
|
const inserts = await insertInChunks(ctx, indexName, documents2, options, hold, true, failed);
|
|
23597
23774
|
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
23598
23775
|
return { succeeded: inserts.succeeded, failed };
|
|
23599
23776
|
}
|
|
23600
23777
|
|
|
23778
|
+
// src/engine/mutations/rollback-copy.ts
|
|
23779
|
+
function frozenRollbackCopy(manager, docId) {
|
|
23780
|
+
const partitionId = manager.partitionIdOf(docId);
|
|
23781
|
+
if (partitionId === void 0) return void 0;
|
|
23782
|
+
const partition = manager.getPartition(partitionId);
|
|
23783
|
+
if (!isCompositePartition(partition)) return void 0;
|
|
23784
|
+
for (const segment of partition.frozenSegments()) {
|
|
23785
|
+
const ordinal = segment.docStore.getInternalId(docId);
|
|
23786
|
+
if (ordinal !== void 0) return segment.documentSource.copyForRollbackAt?.(ordinal);
|
|
23787
|
+
}
|
|
23788
|
+
return void 0;
|
|
23789
|
+
}
|
|
23790
|
+
function rollbackCopyOf(manager, docId) {
|
|
23791
|
+
const frozenCopy = frozenRollbackCopy(manager, docId);
|
|
23792
|
+
if (frozenCopy === void 0) {
|
|
23793
|
+
const document = manager.get(docId);
|
|
23794
|
+
return document === void 0 ? void 0 : () => document;
|
|
23795
|
+
}
|
|
23796
|
+
const vectors = [];
|
|
23797
|
+
for (const [fieldPath, vectorIndex] of manager.getVectorIndexes()) {
|
|
23798
|
+
const vector = vectorIndex.getVector(docId);
|
|
23799
|
+
if (vector) vectors.push([fieldPath, vector]);
|
|
23800
|
+
}
|
|
23801
|
+
return () => {
|
|
23802
|
+
const document = frozenCopy();
|
|
23803
|
+
for (const [fieldPath, vector] of vectors) setNestedValue(document, fieldPath, vector);
|
|
23804
|
+
return document;
|
|
23805
|
+
};
|
|
23806
|
+
}
|
|
23807
|
+
|
|
23601
23808
|
// src/engine/mutations/remove.ts
|
|
23602
|
-
function asRemoveError(err) {
|
|
23603
|
-
return
|
|
23809
|
+
function asRemoveError(ctx, err) {
|
|
23810
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
23604
23811
|
}
|
|
23605
23812
|
function validateRemoval(ctx, docId) {
|
|
23606
23813
|
ctx.guardShutdown();
|
|
@@ -23636,7 +23843,7 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
23636
23843
|
prepared.buffered = true;
|
|
23637
23844
|
return queued;
|
|
23638
23845
|
}
|
|
23639
|
-
prepared.restoreDoc = ctx.durability ? manager
|
|
23846
|
+
prepared.restoreDoc = ctx.durability ? rollbackCopyOf(manager, docId) : void 0;
|
|
23640
23847
|
if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
|
|
23641
23848
|
manager.remove(docId, void 0, {
|
|
23642
23849
|
onRemovedSpellings: (spellings) => {
|
|
@@ -23708,18 +23915,31 @@ async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, h
|
|
|
23708
23915
|
}
|
|
23709
23916
|
}
|
|
23710
23917
|
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
23711
|
-
await
|
|
23712
|
-
|
|
23713
|
-
|
|
23714
|
-
|
|
23715
|
-
|
|
23716
|
-
|
|
23717
|
-
|
|
23918
|
+
await unlessShutDown(
|
|
23919
|
+
ctx,
|
|
23920
|
+
() => runOutside(hold, async () => {
|
|
23921
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23922
|
+
if (refused.has(i)) continue;
|
|
23923
|
+
try {
|
|
23924
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
23925
|
+
} catch (err) {
|
|
23926
|
+
refused.set(i, err);
|
|
23927
|
+
}
|
|
23718
23928
|
}
|
|
23719
|
-
}
|
|
23720
|
-
|
|
23929
|
+
})
|
|
23930
|
+
);
|
|
23721
23931
|
return refused;
|
|
23722
23932
|
}
|
|
23933
|
+
function failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused) {
|
|
23934
|
+
const error = shutDownError();
|
|
23935
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23936
|
+
failed.push({ docId: docIds[i], error: refused.has(i) ? asRemoveError(ctx, refused.get(i)) : error });
|
|
23937
|
+
}
|
|
23938
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
23939
|
+
}
|
|
23940
|
+
function removalDocIdOf(docId) {
|
|
23941
|
+
return docId;
|
|
23942
|
+
}
|
|
23723
23943
|
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
23724
23944
|
ctx.guardShutdown();
|
|
23725
23945
|
ctx.requireIndex(indexName);
|
|
@@ -23731,8 +23951,16 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
23731
23951
|
let manager = first;
|
|
23732
23952
|
try {
|
|
23733
23953
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23954
|
+
if (shutDownStarted(ctx)) {
|
|
23955
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkStart);
|
|
23956
|
+
break;
|
|
23957
|
+
}
|
|
23734
23958
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
23735
23959
|
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
23960
|
+
if (shutDownStarted(ctx)) {
|
|
23961
|
+
failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused);
|
|
23962
|
+
break;
|
|
23963
|
+
}
|
|
23736
23964
|
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
23737
23965
|
const removed = [];
|
|
23738
23966
|
const prepared = [];
|
|
@@ -23742,10 +23970,15 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
23742
23970
|
if (refused.has(i)) throw refused.get(i);
|
|
23743
23971
|
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
23744
23972
|
} catch (err) {
|
|
23745
|
-
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
23973
|
+
failed.push({ docId: docIds[i], error: asRemoveError(ctx, err) });
|
|
23746
23974
|
}
|
|
23747
23975
|
}
|
|
23748
|
-
|
|
23976
|
+
if (shutDownStarted(ctx)) {
|
|
23977
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
23978
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
23979
|
+
break;
|
|
23980
|
+
}
|
|
23981
|
+
const applies = prepared.map((removal) => applyUnlessShutDown(ctx, applyOfRemoval(ctx, indexName, removal)));
|
|
23749
23982
|
try {
|
|
23750
23983
|
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
23751
23984
|
for (let i = 0; i < prepared.length; i++) {
|
|
@@ -23766,7 +23999,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
23766
23999
|
error = rollbackError;
|
|
23767
24000
|
}
|
|
23768
24001
|
removal.releaseVectorHolds?.();
|
|
23769
|
-
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
24002
|
+
failed.push({ docId: removal.docId, error: asRemoveError(ctx, error) });
|
|
23770
24003
|
continue;
|
|
23771
24004
|
}
|
|
23772
24005
|
removal.releaseVectorHolds?.();
|
|
@@ -23776,18 +24009,19 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
23776
24009
|
} finally {
|
|
23777
24010
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
23778
24011
|
}
|
|
23779
|
-
await settleRemovals(ctx, indexName, removed, hold);
|
|
23780
|
-
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
24012
|
+
await unlessShutDown(ctx, () => settleRemovals(ctx, indexName, removed, hold));
|
|
24013
|
+
if (!shutDownStarted(ctx)) manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
23781
24014
|
if (chunkEnd < docIds.length) {
|
|
23782
24015
|
await new Promise((r) => setTimeout(r, 0));
|
|
23783
24016
|
}
|
|
23784
24017
|
}
|
|
23785
24018
|
} finally {
|
|
23786
24019
|
const ending = manager;
|
|
23787
|
-
await ending?.publication.write(() => ending.endBatchRemove());
|
|
24020
|
+
await unlessShutDown(ctx, () => ending?.publication.write(() => ending.endBatchRemove()));
|
|
23788
24021
|
}
|
|
24022
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
23789
24023
|
ctx.checkHeapPressure(indexName);
|
|
23790
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
24024
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
23791
24025
|
return { succeeded, failed };
|
|
23792
24026
|
}
|
|
23793
24027
|
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
@@ -23827,8 +24061,8 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
23827
24061
|
}
|
|
23828
24062
|
return { partitionDoc };
|
|
23829
24063
|
}
|
|
23830
|
-
function asUpdateError(err) {
|
|
23831
|
-
return
|
|
24064
|
+
function asUpdateError(ctx, err) {
|
|
24065
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
23832
24066
|
}
|
|
23833
24067
|
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
23834
24068
|
for (const update of updates) {
|
|
@@ -24025,24 +24259,39 @@ async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold,
|
|
|
24025
24259
|
const { document } = requireUpdateEntry(update);
|
|
24026
24260
|
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
24027
24261
|
} catch (err) {
|
|
24028
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
24262
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
24029
24263
|
}
|
|
24030
24264
|
}
|
|
24031
|
-
|
|
24032
|
-
|
|
24033
|
-
|
|
24034
|
-
|
|
24035
|
-
|
|
24036
|
-
|
|
24037
|
-
|
|
24265
|
+
let entry;
|
|
24266
|
+
try {
|
|
24267
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
24268
|
+
const writes = await prepareDocumentWrites(
|
|
24269
|
+
ctx,
|
|
24270
|
+
indexName,
|
|
24271
|
+
preparations.map((pending) => pending.preparation),
|
|
24272
|
+
hold,
|
|
24273
|
+
false
|
|
24274
|
+
);
|
|
24275
|
+
entry = writes.entry;
|
|
24276
|
+
} catch (err) {
|
|
24277
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
24278
|
+
failAsShutDown(failed, preparations, docIdOf);
|
|
24279
|
+
return [];
|
|
24280
|
+
}
|
|
24038
24281
|
const prepared = [];
|
|
24039
|
-
for (
|
|
24282
|
+
for (let position = 0; position < preparations.length; position++) {
|
|
24040
24283
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
24284
|
+
if (shutDownStarted(ctx)) {
|
|
24285
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
24286
|
+
failAsShutDown(failed, preparations, docIdOf, position);
|
|
24287
|
+
return [];
|
|
24288
|
+
}
|
|
24289
|
+
const { docId, preparation } = preparations[position];
|
|
24041
24290
|
try {
|
|
24042
24291
|
if (preparation.failure !== void 0) throw preparation.failure;
|
|
24043
24292
|
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
24044
24293
|
} catch (err) {
|
|
24045
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
24294
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
24046
24295
|
}
|
|
24047
24296
|
}
|
|
24048
24297
|
return prepared;
|
|
@@ -24055,17 +24304,26 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
24055
24304
|
const failed = [];
|
|
24056
24305
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
24057
24306
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
24307
|
+
if (shutDownStarted(ctx)) {
|
|
24308
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkStart);
|
|
24309
|
+
break;
|
|
24310
|
+
}
|
|
24058
24311
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
24059
24312
|
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
24313
|
+
if (shutDownStarted(ctx)) {
|
|
24314
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
24315
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkEnd);
|
|
24316
|
+
break;
|
|
24317
|
+
}
|
|
24060
24318
|
try {
|
|
24061
24319
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
24062
24320
|
} catch (err) {
|
|
24063
|
-
const error = asUpdateError(err);
|
|
24321
|
+
const error = asUpdateError(ctx, err);
|
|
24064
24322
|
for (const update of prepared) failed.push({ docId: update.docId, error });
|
|
24065
24323
|
for (let i = chunkEnd; i < updates.length; i++) failed.push({ docId: reportedDocIdOf(updates[i]), error });
|
|
24066
24324
|
break;
|
|
24067
24325
|
}
|
|
24068
|
-
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
24326
|
+
const applies = prepared.map((update) => applyUnlessShutDown(ctx, applyOfUpdate(ctx, indexName, update)));
|
|
24069
24327
|
let failures;
|
|
24070
24328
|
try {
|
|
24071
24329
|
failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
@@ -24096,7 +24354,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
24096
24354
|
const update = prepared[i];
|
|
24097
24355
|
const failure = failures[i];
|
|
24098
24356
|
if (failure !== null) {
|
|
24099
|
-
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
24357
|
+
failed.push({ docId: update.docId, error: asUpdateError(ctx, failure.error) });
|
|
24100
24358
|
continue;
|
|
24101
24359
|
}
|
|
24102
24360
|
if (!update.buffered) settled.push(update);
|
|
@@ -24105,18 +24363,19 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
24105
24363
|
}
|
|
24106
24364
|
succeeded.push(update.docId);
|
|
24107
24365
|
}
|
|
24108
|
-
await settleUpdates(ctx, indexName, settled, hold);
|
|
24366
|
+
await unlessShutDown(ctx, () => settleUpdates(ctx, indexName, settled, hold));
|
|
24109
24367
|
if (chunkEnd < updates.length) {
|
|
24110
|
-
ctx.checkHeapPressure(indexName);
|
|
24368
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
24111
24369
|
await new Promise((r) => setTimeout(r, 0));
|
|
24112
24370
|
}
|
|
24113
24371
|
}
|
|
24372
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
24114
24373
|
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
24115
24374
|
for (const fieldPath of touchedVectorFields) {
|
|
24116
24375
|
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
24117
24376
|
}
|
|
24118
24377
|
ctx.checkHeapPressure(indexName);
|
|
24119
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
24378
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
24120
24379
|
return { succeeded, failed };
|
|
24121
24380
|
}
|
|
24122
24381
|
|
|
@@ -24162,12 +24421,17 @@ async function withReservation(grouping, indexName, document, write) {
|
|
|
24162
24421
|
reservation.release();
|
|
24163
24422
|
}
|
|
24164
24423
|
}
|
|
24165
|
-
async function writeInGroups(grouping, indexName, items, write) {
|
|
24424
|
+
async function writeInGroups(ctx, grouping, indexName, items, write) {
|
|
24166
24425
|
const succeeded = [];
|
|
24167
24426
|
const failed = [];
|
|
24168
24427
|
await grouping.applyInGroups(indexName, {
|
|
24169
24428
|
items,
|
|
24170
24429
|
failed,
|
|
24430
|
+
refuseFrom: (start) => {
|
|
24431
|
+
if (!shutDownStarted(ctx)) return false;
|
|
24432
|
+
failAsShutDown(failed, items, docIdOf, start);
|
|
24433
|
+
return true;
|
|
24434
|
+
},
|
|
24171
24435
|
apply: async (group) => {
|
|
24172
24436
|
const result = await write(group);
|
|
24173
24437
|
for (const docId of result.succeeded) succeeded.push(docId);
|
|
@@ -24205,6 +24469,7 @@ async function insertDocumentBatch2(ctx, indexName, documents2, options, hold) {
|
|
|
24205
24469
|
}
|
|
24206
24470
|
const items = documents2.map((document) => ({ docId: providedDocId(document) ?? "", document }));
|
|
24207
24471
|
return writeInGroups(
|
|
24472
|
+
ctx,
|
|
24208
24473
|
grouping,
|
|
24209
24474
|
indexName,
|
|
24210
24475
|
items,
|
|
@@ -24237,6 +24502,7 @@ async function updateDocumentBatch2(ctx, indexName, updates, options, hold) {
|
|
|
24237
24502
|
}
|
|
24238
24503
|
noteGroupedChurn(ctx, indexName, updates.length);
|
|
24239
24504
|
return writeInGroups(
|
|
24505
|
+
ctx,
|
|
24240
24506
|
grouping,
|
|
24241
24507
|
indexName,
|
|
24242
24508
|
updates.map(updateItemOf),
|
|
@@ -25055,7 +25321,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
25055
25321
|
|
|
25056
25322
|
// src/engine/durability-config-code.ts
|
|
25057
25323
|
var durabilityConfig = codeLoader(
|
|
25058
|
-
() => import('./durability-config-
|
|
25324
|
+
() => import('./durability-config-HWKWP35M.mjs').then((code7) => {
|
|
25059
25325
|
code7.bindDurabilityConfigHost(optionalCodeHost());
|
|
25060
25326
|
return code7;
|
|
25061
25327
|
}),
|
|
@@ -25628,7 +25894,7 @@ var workerCopies = codeLoader(async () => {
|
|
|
25628
25894
|
const [segmentCodec2, segmentMerge2, code7] = await Promise.all([
|
|
25629
25895
|
loadSegmentCodec(),
|
|
25630
25896
|
loadSegmentMerge(),
|
|
25631
|
-
import('./worker-copies-
|
|
25897
|
+
import('./worker-copies-NZSBQZKG.mjs')
|
|
25632
25898
|
]);
|
|
25633
25899
|
code7.bindWorkerCopiesHost(optionalCodeHost(), segmentCodec2, segmentMerge2);
|
|
25634
25900
|
return code7;
|
|
@@ -25753,7 +26019,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
25753
26019
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
25754
26020
|
}
|
|
25755
26021
|
const durabilityWiring = durabilityTier === null ? null : (await Promise.all([loadStorageCode(), loadOptionalWrites()]))[0];
|
|
25756
|
-
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
26022
|
+
const filesystemDurability = durabilityTier?.kind === "wal" && durabilityTier.indexedDB === void 0;
|
|
25757
26023
|
const executor = createDirectExecutor({
|
|
25758
26024
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
25759
26025
|
vectorStorage: filesystemDurability ? "disk" : "memory",
|
|
@@ -26029,8 +26295,13 @@ async function shutdownEngine(core2) {
|
|
|
26029
26295
|
await core2.analysisRebuild.shutdown();
|
|
26030
26296
|
await core2.querySuggestionHooks?.shutdown();
|
|
26031
26297
|
core2.indexState.dispose();
|
|
26298
|
+
let durabilityFailure = null;
|
|
26032
26299
|
if (durability) {
|
|
26033
|
-
|
|
26300
|
+
try {
|
|
26301
|
+
await durability.manager.shutdown();
|
|
26302
|
+
} catch (error) {
|
|
26303
|
+
durabilityFailure = { error };
|
|
26304
|
+
}
|
|
26034
26305
|
}
|
|
26035
26306
|
const vectorDisposals = [];
|
|
26036
26307
|
for (const [name] of indexRegistry) {
|
|
@@ -26064,6 +26335,7 @@ async function shutdownEngine(core2) {
|
|
|
26064
26335
|
await Promise.all(vectorDisposals);
|
|
26065
26336
|
eventHandlers.clear();
|
|
26066
26337
|
indexRegistry.clear();
|
|
26338
|
+
if (durabilityFailure !== null) throw durabilityFailure.error;
|
|
26067
26339
|
}
|
|
26068
26340
|
|
|
26069
26341
|
// src/core/keyword-score/layout.ts
|
|
@@ -28709,32 +28981,6 @@ function partitionStoredFields(partition, docId, fields) {
|
|
|
28709
28981
|
return void 0;
|
|
28710
28982
|
}
|
|
28711
28983
|
|
|
28712
|
-
// src/partitioning/manager/nested-values.ts
|
|
28713
|
-
function setNestedValue(obj, path, value) {
|
|
28714
|
-
if (!path.includes(".")) {
|
|
28715
|
-
obj[path] = value;
|
|
28716
|
-
return;
|
|
28717
|
-
}
|
|
28718
|
-
const segments2 = path.split(".");
|
|
28719
|
-
let current = obj;
|
|
28720
|
-
for (let i = 0; i < segments2.length - 1; i++) {
|
|
28721
|
-
let next = current[segments2[i]];
|
|
28722
|
-
if (next === null || next === void 0 || typeof next !== "object") {
|
|
28723
|
-
next = {};
|
|
28724
|
-
current[segments2[i]] = next;
|
|
28725
|
-
}
|
|
28726
|
-
current = next;
|
|
28727
|
-
}
|
|
28728
|
-
current[segments2[segments2.length - 1]] = value;
|
|
28729
|
-
}
|
|
28730
|
-
function copyVectorsInto(document, docId, vectorIndexes, projection) {
|
|
28731
|
-
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
28732
|
-
if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
|
|
28733
|
-
const vector = vectorIndex.getVector(docId);
|
|
28734
|
-
if (vector) setNestedValue(document, fieldPath, vector);
|
|
28735
|
-
}
|
|
28736
|
-
}
|
|
28737
|
-
|
|
28738
28984
|
// src/partitioning/manager/state-pins.ts
|
|
28739
28985
|
function documentFieldValues(ref, fieldPaths) {
|
|
28740
28986
|
return fieldPaths.map((fieldPath) => getNestedValue(ref, fieldPath));
|
|
@@ -30126,7 +30372,7 @@ var suggestionsCode = codeLoader(async () => {
|
|
|
30126
30372
|
const [msgpack, suggestionStore2, code7] = await Promise.all([
|
|
30127
30373
|
loadMessagePack(),
|
|
30128
30374
|
loadSuggestionStore(),
|
|
30129
|
-
import('./suggestions-leaf-
|
|
30375
|
+
import('./suggestions-leaf-TX3GTD6S.mjs')
|
|
30130
30376
|
]);
|
|
30131
30377
|
code7.bindSuggestionsHost(optionalCodeHost(), msgpack, suggestionStore2);
|
|
30132
30378
|
return code7;
|
|
@@ -31238,6 +31484,7 @@ function writesHostPart() {
|
|
|
31238
31484
|
requireDocumentWithinSizeLimit,
|
|
31239
31485
|
requireMatchingDocId,
|
|
31240
31486
|
requireSurfaceFormsForPartialFields,
|
|
31487
|
+
setNestedValue,
|
|
31241
31488
|
storedBM25Params,
|
|
31242
31489
|
turnsAreBudgeted,
|
|
31243
31490
|
updateDocumentVectors,
|
|
@@ -31287,7 +31534,7 @@ var segmentMerge = codeLoader(async () => {
|
|
|
31287
31534
|
const [msgpack, codec, code7] = await Promise.all([
|
|
31288
31535
|
loadMessagePack(),
|
|
31289
31536
|
loadSegmentCodec(),
|
|
31290
|
-
import('./segment-merge-leaf-
|
|
31537
|
+
import('./segment-merge-leaf-6TT66HC5.mjs')
|
|
31291
31538
|
]);
|
|
31292
31539
|
code7.bindSegmentMergeHost(optionalCodeHost(), msgpack, codec);
|
|
31293
31540
|
return code7;
|