@delali/narsil 0.4.1-next.215 → 0.4.2-next.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-DPRCAVC2.mjs} +14 -8
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-WRRXR57N.mjs} +1 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-XBWABEL3.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-Y5CSZDOH.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-FEPCLXXY.mjs} +7 -1
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-HVPPDHSY.mjs} +259 -104
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-DBC4EJWB.mjs → chunk-33ZMFOAR.mjs} +1 -1
- package/dist/{chunk-NCIGNQJE.mjs → chunk-4GCHJ3XD.mjs} +930 -387
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-4WQNWAQE.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-5E3TJ4P3.mjs} +5 -5
- package/dist/{chunk-GXCDWLQI.mjs → chunk-5GYGHPCH.mjs} +7 -1
- package/dist/{chunk-SMQPZHPS.mjs → chunk-ATQ6JR3I.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-BBIAUHAB.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-DA32WXBT.mjs} +2 -2
- package/dist/{chunk-MSJD74GN.mjs → chunk-EALKCTWH.mjs} +5 -5
- package/dist/{chunk-755NM4PL.mjs → chunk-FOGTBAYN.mjs} +183 -21
- package/dist/{chunk-6GHX35ZM.mjs → chunk-GTBJSNN2.mjs} +1 -1
- package/dist/{chunk-DDUXEJXW.mjs → chunk-H6LWGMPQ.mjs} +45 -22
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-IL5DRCNU.mjs → chunk-KQWTS7AM.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-KX3OP6VP.mjs} +2 -2
- package/dist/{chunk-D456ZFPQ.mjs → chunk-MIIJ5O3O.mjs} +2 -2
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-MS746XNY.mjs} +285 -118
- package/dist/{chunk-WM5KJSQV.mjs → chunk-NFA3WHJU.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-OETLM4EH.mjs} +1 -1
- package/dist/{chunk-MYN4K5IS.mjs → chunk-OLXFIUIZ.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-X3EQCQLP.mjs → chunk-Q4NLL66U.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-QJFKL4NK.mjs} +4 -4
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-3IE3A74H.mjs → chunk-SSITKQU6.mjs} +3 -3
- package/dist/{chunk-B3MO37YF.mjs → chunk-TANKI7TG.mjs} +1 -1
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-TI4G3SIG.mjs} +63 -144
- package/dist/{chunk-OGYSPNL2.mjs → chunk-TT3UDDL3.mjs} +7 -7
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-VHIOGDVD.mjs} +1 -1
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-W3JH2IK6.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-WIT6B5OB.mjs} +2 -2
- package/dist/{chunk-MI6UZGUR.mjs → chunk-XGVSG6ZU.mjs} +3 -3
- package/dist/{chunk-6I6OO6B2.mjs → chunk-XWPNRHTG.mjs} +4 -4
- package/dist/{chunk-WTMCGZ74.mjs → chunk-YHZSSPG3.mjs} +2 -2
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-PCBB6FCO.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-4ALYIIC4.mjs} +14 -14
- package/dist/msgpack-leaf-6ZBZWFRD.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-2OK4TOLY.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-Y5OAKNRB.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-UQVQZFIJ.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-4OOUY2K7.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-5MYHYBET.mjs} +286 -118
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AVCMTWET.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-GS6LGXY3.mjs} +1 -1
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-LE4DTMDL.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-VIO5IPF7.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-VRI5XH2M.mjs} +7 -1
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-XGS2UYVI.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-XW6GZQWP.mjs} +4 -4
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-Z44S6WW5.mjs} +2 -2
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-4MIGXWZT.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-LZDHB7FX.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-J7SB263E.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PEKX7TBL.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-I2EC7H6R.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-RQXM7HBO.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-EZGCLSLM.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-JNGH2JYC.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-2OWLLBCV.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-TQ4SU76L.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-W7U2X6V3.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-WGP4FBXF.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-FYWJQSPL.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-GRG44RQA.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-YDJVH73G.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-5UFJV7SC.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-6YLVL2NC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-IP4L7KEW.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -10
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { reservedHeapBytes, reserveHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, admittedHeapBytes } from './chunk-HQEVQEDJ.mjs';
|
|
2
2
|
import { fnv1aUtf16, changeIndexOutsideExternalWaits, turnsAreBudgeted, turnIsSpent, yieldTurn, createDocIdMap, detectRuntime, fnv1a, indexChangeOutsideExternalWaits, stopOfSignal, splitWorkerBudget, resolveWorkerCount, sharedStopOfSignal, requireSearchActive, yieldBackgroundTurn, backgroundTurnIsSpent, FNV_OFFSET_BASIS, DEFAULT_REQUEST_TIMEOUT_MS, countWrite, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, answerFromOneState, signalOfSearch, requirePinnedState, fnv1aContinue, fnv1aContinueTaggedNumbers, FNV_PRIME, collectAndReturnPages, shardEntryRangeOf, threadSlotOfWorker, resolveRequestThreadCount, SHUTDOWN_TIMEOUT_MS, DEFAULT_BACKPRESSURE_LIMIT, threadSlotGone, spawnNodeWorker, resolveWorkerEntry, noteThreadSlotLive, noteThreadSlotGone, bindSearchSignal, adoptSharedTurns, WORKER_YOUNG_GENERATION_MB, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_BLOCK_MAX_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_ATTACH_RETRY_MS, MAX_WASM_PAGES, MAX_VECTOR_ORDINALS, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_DOC_ID_TABLE_BYTES, LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, DISK_STORAGE_BLOCK_BYTES, waitOutsideCountedWrite, nodeThreadId } from './chunk-AKRYHAVG.mjs';
|
|
3
3
|
import { MIN_DOCUMENTS_FOR_SEGMENTS, MAX_PARTITIONS, MAX_INDEX_CONFIGURATION_BYTES, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_CALLS_ACROSS_INDEX_CHANGES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MIN_SWEEP_INTERVAL_MS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, DEFAULT_PAGE_SIZE, CURSOR_BINDING_BUFFER_BYTES, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, MIN_RANK_CONSTANT, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, FUSION_TIE_INSERTION_LIMIT, DEFAULT_RANK_CONSTANT, DEFAULT_LINEAR_ALPHA, STATISTICS_BROADCAST_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, EXPORT_CHUNK_BYTES, DEFAULT_MAX_PARTITIONS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, ANALYSIS_CHANGE_MEMBERS, MIN_HYBRID_DEPTH, MAX_SUGGEST_SCATTER_LIMIT, MAX_SUGGEST_LIMIT, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_SUGGEST_LIMIT, DEFAULT_MAX_PER_GROUP } from './chunk-LX62Y4B5.mjs';
|
|
4
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, DEFAULT_PATTERN_WORK_CAP, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, bitsetSet, discardStoredValues, createBitSet, installStoredFields, storedValueBytes, bitsetIsEmpty, prepareStoredValues, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyPrefixOf, boundedLevenshtein, checkedSize, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, buildStoredValues, FILTER_OPERAND_COPY_DEPTH, fuzzyTermMatches, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIterator, GrowableUint32, bitsetFromSet } from './chunk-
|
|
4
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, DEFAULT_PATTERN_WORK_CAP, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, bitsetSet, discardStoredValues, createBitSet, installStoredFields, storedValueBytes, bitsetIsEmpty, prepareStoredValues, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyPrefixOf, boundedLevenshtein, checkedSize, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, buildStoredValues, FILTER_OPERAND_COPY_DEPTH, fuzzyTermMatches, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIterator, GrowableUint32, bitsetFromSet } from './chunk-GS6LGXY3.mjs';
|
|
5
5
|
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_NEEDS_ROOM, NATIVE_OK } from './chunk-UYKIV2KT.mjs';
|
|
6
6
|
import { reactNativeHost } from './chunk-CW2ZFXSE.mjs';
|
|
7
|
-
import { headerAt, MAP_TAGS, decode, encodeDocument, Decoder } from './chunk-
|
|
8
|
-
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, schemaFieldsOf, getNestedValue, tokenizeOptions, flattenSchema, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, vectorDimensionOf, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, PROTOTYPE_POLLUTION_KEYS, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, MAX_NESTING_DEPTH, getFlatSchema, getOrCreateFieldNameIndex, isWordIndexedBase, isStringValuedBase, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-
|
|
7
|
+
import { headerAt, MAP_TAGS, decode, encodeDocument, Decoder } from './chunk-VRI5XH2M.mjs';
|
|
8
|
+
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, schemaFieldsOf, getNestedValue, tokenizeOptions, flattenSchema, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, vectorDimensionOf, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, PROTOTYPE_POLLUTION_KEYS, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, MAX_NESTING_DEPTH, getFlatSchema, getOrCreateFieldNameIndex, isWordIndexedBase, isStringValuedBase, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-AVCMTWET.mjs';
|
|
9
9
|
import { withUnpairedSurrogatesReplaced, compareCodePoints, holdsUnpairedSurrogate, defaultSortMode, toComparableSortValue, exceedsCodePoints, SORT_VALUE_MAX_CODE_POINTS, sortModeOf, PLACEMENT_ROOM_REQUESTS, NEAREST_LINK_FALLBACK_HOSTS, MIN_M, MAX_M, MAX_LAYER_CAP, INITIAL_UPPER_SLOTS, INITIAL_LIST_CAPACITY, FARTHEST_REJECTED_LINKS_KEPT, EARLIER_RELEASE_MIN_M, DEFAULT_EF_SEARCH, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, CONNECT_RESTARTS_BEFORE_ONE_PASS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, truncateSortString, SORT_MODES, isSortMode, codePointCount } from './chunk-HRDMIQ63.mjs';
|
|
10
10
|
import { MAX_DOCUMENT_ENCODING_DEPTH, MAX_FILE_BYTES, VALID_HNSW_METRICS, NATIVE_CRC32_MIN_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, DOCUMENT_REPLY_ENVELOPE_DEPTH, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER } from './chunk-VUJCSQG5.mjs';
|
|
11
11
|
import { NarsilError, ErrorCodes, createNarsilError, isNarsilError } from './chunk-43R4GCM7.mjs';
|
|
@@ -8682,7 +8682,7 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
8682
8682
|
|
|
8683
8683
|
// src/core/segment-codec-code.ts
|
|
8684
8684
|
var segmentCodec = codeLoader(async () => {
|
|
8685
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-
|
|
8685
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-PEKX7TBL.mjs')]);
|
|
8686
8686
|
code7.bindSegmentCodecHost(optionalCodeHost(), msgpack);
|
|
8687
8687
|
return code7;
|
|
8688
8688
|
}, "segment codec");
|
|
@@ -8693,7 +8693,7 @@ var segmentMerge = codeLoader(async () => {
|
|
|
8693
8693
|
const [msgpack, codec, code7] = await Promise.all([
|
|
8694
8694
|
loadMessagePack(),
|
|
8695
8695
|
loadSegmentCodec(),
|
|
8696
|
-
import('./segment-merge-leaf-
|
|
8696
|
+
import('./segment-merge-leaf-I2EC7H6R.mjs')
|
|
8697
8697
|
]);
|
|
8698
8698
|
code7.bindSegmentMergeHost(optionalCodeHost(), msgpack, codec);
|
|
8699
8699
|
return code7;
|
|
@@ -8702,7 +8702,7 @@ var loadSegmentMerge = segmentMerge.load;
|
|
|
8702
8702
|
|
|
8703
8703
|
// src/query-suggestions/store-code.ts
|
|
8704
8704
|
var suggestionStore = codeLoader(async () => {
|
|
8705
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-
|
|
8705
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-EZGCLSLM.mjs')]);
|
|
8706
8706
|
code7.bindSuggestionStoreHost(optionalCodeHost(), msgpack);
|
|
8707
8707
|
return code7;
|
|
8708
8708
|
}, "query suggestions store");
|
|
@@ -8783,7 +8783,7 @@ var storageCode = codeLoader(async () => {
|
|
|
8783
8783
|
loadSegmentMerge(),
|
|
8784
8784
|
loadSuggestionStore(),
|
|
8785
8785
|
loadVectorCore(),
|
|
8786
|
-
import('./storage-leaf-
|
|
8786
|
+
import('./storage-leaf-RQXM7HBO.mjs')
|
|
8787
8787
|
]);
|
|
8788
8788
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
8789
8789
|
return code7;
|
|
@@ -19946,7 +19946,7 @@ function hasLanguage(name) {
|
|
|
19946
19946
|
|
|
19947
19947
|
// src/narsil/restore-queue-code.ts
|
|
19948
19948
|
var restoreQueue = codeLoader(async () => {
|
|
19949
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
19949
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-J7SB263E.mjs')]);
|
|
19950
19950
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
19951
19951
|
return code7;
|
|
19952
19952
|
}, "restore");
|
|
@@ -20548,7 +20548,7 @@ var loadEmbedCode = embedCode.load;
|
|
|
20548
20548
|
|
|
20549
20549
|
// src/engine/optional-writes-code.ts
|
|
20550
20550
|
var optionalWrites = codeLoader(async () => {
|
|
20551
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
20551
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-LZDHB7FX.mjs')]);
|
|
20552
20552
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
20553
20553
|
return code7;
|
|
20554
20554
|
}, "write");
|
|
@@ -21012,6 +21012,51 @@ async function oneAtATime(core2, indexName, action) {
|
|
|
21012
21012
|
function bindOptionalWritesHost(_host, _segmentCodec) {
|
|
21013
21013
|
}
|
|
21014
21014
|
|
|
21015
|
+
// src/engine/mutations/batch-shutdown.ts
|
|
21016
|
+
function isFailureThatShutdownCauses(code7) {
|
|
21017
|
+
return code7 === ErrorCodes.INSTANCE_SHUT_DOWN || code7 === ErrorCodes.INDEX_NOT_FOUND || code7 === ErrorCodes.EMBEDDING_FAILED;
|
|
21018
|
+
}
|
|
21019
|
+
function shutDownStarted(ctx) {
|
|
21020
|
+
return ctx.abortController.signal.aborted;
|
|
21021
|
+
}
|
|
21022
|
+
function shutDownError() {
|
|
21023
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
21024
|
+
}
|
|
21025
|
+
function causedByShutdown(ctx, err) {
|
|
21026
|
+
const { signal } = ctx.abortController;
|
|
21027
|
+
if (!signal.aborted) return false;
|
|
21028
|
+
if (err === signal.reason) return true;
|
|
21029
|
+
if (err instanceof NarsilError) return isFailureThatShutdownCauses(err.code);
|
|
21030
|
+
return err instanceof Error && err.name === "AbortError";
|
|
21031
|
+
}
|
|
21032
|
+
function batchFailureOf(ctx, err, fallbackCode) {
|
|
21033
|
+
if (causedByShutdown(ctx, err)) {
|
|
21034
|
+
return err instanceof NarsilError && err.code === ErrorCodes.INSTANCE_SHUT_DOWN ? err : shutDownError();
|
|
21035
|
+
}
|
|
21036
|
+
return err instanceof NarsilError ? err : new NarsilError(fallbackCode, String(err));
|
|
21037
|
+
}
|
|
21038
|
+
function docIdOf(entry) {
|
|
21039
|
+
return entry.docId;
|
|
21040
|
+
}
|
|
21041
|
+
function failAsShutDown(failed, entries, reportedDocIdOf2, start = 0, end = entries.length) {
|
|
21042
|
+
if (start >= end) return;
|
|
21043
|
+
const error = shutDownError();
|
|
21044
|
+
for (let i = start; i < end; i++) failed.push({ docId: reportedDocIdOf2(entries[i]), error });
|
|
21045
|
+
}
|
|
21046
|
+
function applyUnlessShutDown(ctx, apply) {
|
|
21047
|
+
return () => {
|
|
21048
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
21049
|
+
return apply();
|
|
21050
|
+
};
|
|
21051
|
+
}
|
|
21052
|
+
async function unlessShutDown(ctx, work) {
|
|
21053
|
+
try {
|
|
21054
|
+
await work();
|
|
21055
|
+
} catch (err) {
|
|
21056
|
+
if (!shutDownStarted(ctx)) throw err;
|
|
21057
|
+
}
|
|
21058
|
+
}
|
|
21059
|
+
|
|
21015
21060
|
// src/engine/mutations/batch-admissions.ts
|
|
21016
21061
|
function createPendingAdmissions() {
|
|
21017
21062
|
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
@@ -21022,8 +21067,11 @@ function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
|
21022
21067
|
const partitionId = manager.routePartition(docId);
|
|
21023
21068
|
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
21024
21069
|
}
|
|
21025
|
-
function asBatchInsertError(err) {
|
|
21026
|
-
return
|
|
21070
|
+
function asBatchInsertError(ctx, err) {
|
|
21071
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_VALIDATION_FAILED);
|
|
21072
|
+
}
|
|
21073
|
+
function providedDocIdOrEmpty(document) {
|
|
21074
|
+
return providedDocId(document) ?? "";
|
|
21027
21075
|
}
|
|
21028
21076
|
|
|
21029
21077
|
// src/engine/mutations/outside-hold.ts
|
|
@@ -21253,7 +21301,11 @@ async function prepareInsertDocuments(ctx, indexName, documents2, start, end, ho
|
|
|
21253
21301
|
const pending = [];
|
|
21254
21302
|
for (let i = start; i < end; i++) {
|
|
21255
21303
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
21256
|
-
if (ctx
|
|
21304
|
+
if (shutDownStarted(ctx)) {
|
|
21305
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
21306
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty, i, end);
|
|
21307
|
+
return null;
|
|
21308
|
+
}
|
|
21257
21309
|
const document = documents2[i];
|
|
21258
21310
|
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
21259
21311
|
try {
|
|
@@ -21262,17 +21314,26 @@ async function prepareInsertDocuments(ctx, indexName, documents2, start, end, ho
|
|
|
21262
21314
|
validateDocId(docId);
|
|
21263
21315
|
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
21264
21316
|
} catch (err) {
|
|
21265
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
21317
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
21266
21318
|
}
|
|
21267
21319
|
}
|
|
21268
|
-
|
|
21269
|
-
|
|
21270
|
-
|
|
21271
|
-
|
|
21272
|
-
|
|
21273
|
-
|
|
21274
|
-
|
|
21275
|
-
|
|
21320
|
+
let prepared;
|
|
21321
|
+
try {
|
|
21322
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
21323
|
+
prepared = await prepareDocumentWrites(
|
|
21324
|
+
ctx,
|
|
21325
|
+
indexName,
|
|
21326
|
+
pending.map((item) => item.preparation),
|
|
21327
|
+
hold,
|
|
21328
|
+
true,
|
|
21329
|
+
earlier
|
|
21330
|
+
);
|
|
21331
|
+
} catch (err) {
|
|
21332
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
21333
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
21334
|
+
return null;
|
|
21335
|
+
}
|
|
21336
|
+
const { entry, embedding } = prepared;
|
|
21276
21337
|
const candidates2 = [];
|
|
21277
21338
|
for (const item of pending) {
|
|
21278
21339
|
const { document, failure } = item.preparation;
|
|
@@ -21280,22 +21341,19 @@ async function prepareInsertDocuments(ctx, indexName, documents2, start, end, ho
|
|
|
21280
21341
|
candidates2.push({ docId: item.docId, document });
|
|
21281
21342
|
continue;
|
|
21282
21343
|
}
|
|
21283
|
-
const docId = item.refusedByHook ? item.docId :
|
|
21284
|
-
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
21344
|
+
const docId = item.refusedByHook ? item.docId : providedDocIdOrEmpty(document);
|
|
21345
|
+
failed.push({ docId, error: asBatchInsertError(ctx, failure) });
|
|
21285
21346
|
}
|
|
21286
21347
|
return { entry, candidates: candidates2, embedding };
|
|
21287
21348
|
}
|
|
21288
21349
|
async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
21289
|
-
const
|
|
21290
|
-
|
|
21291
|
-
|
|
21292
|
-
|
|
21293
|
-
|
|
21294
|
-
|
|
21295
|
-
|
|
21296
|
-
true,
|
|
21297
|
-
failed
|
|
21298
|
-
);
|
|
21350
|
+
const prepared = await prepareInsertDocuments(ctx, indexName, documents2, 0, documents2.length, hold, true, failed);
|
|
21351
|
+
if (prepared === null) return { admitted: [], embedding: null };
|
|
21352
|
+
const { entry, candidates: candidates2, embedding } = prepared;
|
|
21353
|
+
if (shutDownStarted(ctx)) {
|
|
21354
|
+
failAsShutDown(failed, candidates2, docIdOf);
|
|
21355
|
+
return { admitted: [], embedding };
|
|
21356
|
+
}
|
|
21299
21357
|
const manager = ctx.requireManager(indexName);
|
|
21300
21358
|
const vecIndexes = manager.getVectorIndexes();
|
|
21301
21359
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
@@ -21305,7 +21363,11 @@ async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
|
21305
21363
|
const pending = createPendingAdmissions();
|
|
21306
21364
|
for (let position = 0; position < candidates2.length; position++) {
|
|
21307
21365
|
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
21308
|
-
if (ctx
|
|
21366
|
+
if (shutDownStarted(ctx)) {
|
|
21367
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
21368
|
+
failAsShutDown(failed, candidates2, docIdOf, position);
|
|
21369
|
+
return { admitted: [], embedding };
|
|
21370
|
+
}
|
|
21309
21371
|
const { docId, document } = candidates2[position];
|
|
21310
21372
|
try {
|
|
21311
21373
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
@@ -21327,7 +21389,7 @@ async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
|
21327
21389
|
recordPendingAdmission(ctx, pending, manager, docId);
|
|
21328
21390
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
21329
21391
|
} catch (err) {
|
|
21330
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
21392
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
21331
21393
|
}
|
|
21332
21394
|
}
|
|
21333
21395
|
return { admitted, embedding };
|
|
@@ -21477,9 +21539,9 @@ async function recordChunk(ctx, indexName, mutations, applies) {
|
|
|
21477
21539
|
function applyInsertDocument(ctx, indexName, doc, options, progress) {
|
|
21478
21540
|
const manager = ctx.requireManager(indexName);
|
|
21479
21541
|
const insertIntoIndex = () => {
|
|
21480
|
-
if (ctx
|
|
21542
|
+
if (shutDownStarted(ctx)) {
|
|
21481
21543
|
progress.skipped = true;
|
|
21482
|
-
throw
|
|
21544
|
+
throw shutDownError();
|
|
21483
21545
|
}
|
|
21484
21546
|
admitInsert(ctx, indexName, manager, doc.docId);
|
|
21485
21547
|
const queued = ctx.bufferIfRebalancing(indexName, {
|
|
@@ -21535,11 +21597,11 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
21535
21597
|
const progress = docs.map(() => ({ inserted: false, buffered: false, skipped: false }));
|
|
21536
21598
|
const applies = docs.map((doc, i) => applyInsertDocument(ctx, indexName, doc, options, progress[i]));
|
|
21537
21599
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
21538
|
-
await ctx.orchestrator.settlePublications(indexName);
|
|
21600
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName));
|
|
21539
21601
|
const applications = [];
|
|
21540
21602
|
for (let i = 0; i < docs.length; i++) {
|
|
21541
21603
|
if (progress[i].skipped) {
|
|
21542
|
-
applications.push({ status: "
|
|
21604
|
+
applications.push({ status: "failed", error: shutDownError() });
|
|
21543
21605
|
continue;
|
|
21544
21606
|
}
|
|
21545
21607
|
const failure = failures[i];
|
|
@@ -21572,27 +21634,33 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
21572
21634
|
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
21573
21635
|
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
21574
21636
|
const manager = ctx.executor.getManager(indexName);
|
|
21575
|
-
await
|
|
21576
|
-
|
|
21577
|
-
|
|
21578
|
-
|
|
21579
|
-
|
|
21580
|
-
|
|
21637
|
+
await unlessShutDown(
|
|
21638
|
+
ctx,
|
|
21639
|
+
() => runOutside(hold, async () => {
|
|
21640
|
+
for (const doc of inserted) {
|
|
21641
|
+
try {
|
|
21642
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
21643
|
+
} catch (err) {
|
|
21644
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
21645
|
+
}
|
|
21581
21646
|
}
|
|
21582
|
-
}
|
|
21583
|
-
|
|
21584
|
-
return ctx.executor.getManager(indexName) !== manager;
|
|
21647
|
+
})
|
|
21648
|
+
);
|
|
21649
|
+
return shutDownStarted(ctx) || ctx.executor.getManager(indexName) !== manager;
|
|
21585
21650
|
}
|
|
21586
21651
|
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
21587
21652
|
const succeeded = [];
|
|
21588
21653
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
21589
21654
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21590
|
-
if (ctx
|
|
21655
|
+
if (shutDownStarted(ctx)) {
|
|
21656
|
+
failAsShutDown(failed, admitted, docIdOf, chunkStart);
|
|
21657
|
+
break;
|
|
21658
|
+
}
|
|
21591
21659
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
21592
21660
|
try {
|
|
21593
21661
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
21594
21662
|
} catch (err) {
|
|
21595
|
-
const error = asBatchInsertError(err);
|
|
21663
|
+
const error = asBatchInsertError(ctx, err);
|
|
21596
21664
|
for (let i = chunkStart; i < admitted.length; i++) failed.push({ docId: admitted[i].docId, error });
|
|
21597
21665
|
break;
|
|
21598
21666
|
}
|
|
@@ -21601,9 +21669,8 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
21601
21669
|
for (let i = 0; i < chunk.length; i++) {
|
|
21602
21670
|
const doc = chunk[i];
|
|
21603
21671
|
const application = applications[i];
|
|
21604
|
-
if (application.status === "skipped") continue;
|
|
21605
21672
|
if (application.status === "failed") {
|
|
21606
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
21673
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
21607
21674
|
continue;
|
|
21608
21675
|
}
|
|
21609
21676
|
succeeded.push(doc.docId);
|
|
@@ -21618,7 +21685,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
21618
21685
|
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
21619
21686
|
}
|
|
21620
21687
|
if (applied < admitted.length) {
|
|
21621
|
-
ctx.checkHeapPressure(indexName);
|
|
21688
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
21622
21689
|
await new Promise((r) => setTimeout(r, 0));
|
|
21623
21690
|
}
|
|
21624
21691
|
}
|
|
@@ -21627,7 +21694,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
21627
21694
|
|
|
21628
21695
|
// src/engine/mutations/insert-batch-chunks.ts
|
|
21629
21696
|
async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
21630
|
-
const
|
|
21697
|
+
const prepared = await prepareInsertDocuments(
|
|
21631
21698
|
ctx,
|
|
21632
21699
|
indexName,
|
|
21633
21700
|
documents2,
|
|
@@ -21638,13 +21705,24 @@ async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold
|
|
|
21638
21705
|
failed,
|
|
21639
21706
|
earlier
|
|
21640
21707
|
);
|
|
21708
|
+
if (prepared === null) return [];
|
|
21709
|
+
const { entry, candidates: candidates2 } = prepared;
|
|
21710
|
+
const admitted = [];
|
|
21711
|
+
if (shutDownStarted(ctx)) {
|
|
21712
|
+
failAsShutDown(failed, candidates2, docIdOf);
|
|
21713
|
+
return admitted;
|
|
21714
|
+
}
|
|
21641
21715
|
const manager = ctx.requireManager(indexName);
|
|
21642
21716
|
const vecIndexes = manager.getVectorIndexes();
|
|
21643
21717
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
21644
|
-
|
|
21645
|
-
for (const { docId, document } of candidates2) {
|
|
21718
|
+
for (let position = 0; position < candidates2.length; position++) {
|
|
21646
21719
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
21647
|
-
if (ctx
|
|
21720
|
+
if (shutDownStarted(ctx)) {
|
|
21721
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
21722
|
+
failAsShutDown(failed, candidates2, docIdOf, position);
|
|
21723
|
+
return [];
|
|
21724
|
+
}
|
|
21725
|
+
const { docId, document } = candidates2[position];
|
|
21648
21726
|
try {
|
|
21649
21727
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
21650
21728
|
document,
|
|
@@ -21656,7 +21734,7 @@ async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold
|
|
|
21656
21734
|
admitInsert(ctx, indexName, manager, docId);
|
|
21657
21735
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
21658
21736
|
} catch (err) {
|
|
21659
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
21737
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
21660
21738
|
}
|
|
21661
21739
|
}
|
|
21662
21740
|
return admitted;
|
|
@@ -21665,17 +21743,25 @@ async function insertInChunks(ctx, indexName, documents2, options, hold, hooked,
|
|
|
21665
21743
|
const succeeded = [];
|
|
21666
21744
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
21667
21745
|
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21668
|
-
if (ctx
|
|
21746
|
+
if (shutDownStarted(ctx)) {
|
|
21747
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty, chunkStart);
|
|
21748
|
+
break;
|
|
21749
|
+
}
|
|
21669
21750
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
21670
21751
|
const earlier = embeddingOfDocuments(embedded2, documents2.slice(chunkStart, chunkEnd));
|
|
21671
21752
|
const prepared = await admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
21753
|
+
if (shutDownStarted(ctx)) {
|
|
21754
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
21755
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty, chunkEnd);
|
|
21756
|
+
break;
|
|
21757
|
+
}
|
|
21672
21758
|
try {
|
|
21673
21759
|
if (prepared.length > 0) ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
21674
21760
|
} catch (err) {
|
|
21675
|
-
const error = asBatchInsertError(err);
|
|
21761
|
+
const error = asBatchInsertError(ctx, err);
|
|
21676
21762
|
for (const doc of prepared) failed.push({ docId: doc.docId, error });
|
|
21677
21763
|
for (let i = chunkEnd; i < documents2.length; i++) {
|
|
21678
|
-
failed.push({ docId:
|
|
21764
|
+
failed.push({ docId: providedDocIdOrEmpty(documents2[i]), error });
|
|
21679
21765
|
}
|
|
21680
21766
|
break;
|
|
21681
21767
|
}
|
|
@@ -21685,9 +21771,8 @@ async function insertInChunks(ctx, indexName, documents2, options, hold, hooked,
|
|
|
21685
21771
|
for (let i = 0; i < prepared.length; i++) {
|
|
21686
21772
|
const doc = prepared[i];
|
|
21687
21773
|
const application = applications[i];
|
|
21688
|
-
if (application.status === "skipped") continue;
|
|
21689
21774
|
if (application.status === "failed") {
|
|
21690
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
21775
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
21691
21776
|
continue;
|
|
21692
21777
|
}
|
|
21693
21778
|
succeeded.push(doc.docId);
|
|
@@ -21698,21 +21783,24 @@ async function insertInChunks(ctx, indexName, documents2, options, hold, hooked,
|
|
|
21698
21783
|
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
21699
21784
|
}
|
|
21700
21785
|
if (chunkEnd < documents2.length) {
|
|
21701
|
-
ctx.checkHeapPressure(indexName);
|
|
21786
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
21702
21787
|
await new Promise((r) => setTimeout(r, 0));
|
|
21703
21788
|
}
|
|
21704
21789
|
}
|
|
21705
21790
|
return { succeeded, touchedVectorFields };
|
|
21706
21791
|
}
|
|
21707
21792
|
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
21793
|
+
if (shutDownStarted(ctx)) return;
|
|
21708
21794
|
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
21709
21795
|
for (const fieldPath of inserts.touchedVectorFields) {
|
|
21710
21796
|
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
21711
21797
|
}
|
|
21712
21798
|
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
21713
21799
|
ctx.checkHeapPressure(indexName);
|
|
21714
|
-
await ctx
|
|
21715
|
-
|
|
21800
|
+
await unlessShutDown(ctx, async () => {
|
|
21801
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
21802
|
+
if (options?.wait === true && !shutDownStarted(ctx)) await awaitWriteVisibility(ctx, indexName);
|
|
21803
|
+
});
|
|
21716
21804
|
}
|
|
21717
21805
|
|
|
21718
21806
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -21721,7 +21809,7 @@ function applySegment(ctx, indexName, attachment, doc, options) {
|
|
|
21721
21809
|
const publish2 = async () => {
|
|
21722
21810
|
if (attachment.failure !== void 0) throw attachment.failure;
|
|
21723
21811
|
if (attachment.attached) return;
|
|
21724
|
-
if (ctx
|
|
21812
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
21725
21813
|
if (ctx.isRebalancing(indexName) || manager.partitionCount !== attachment.partitionCount) {
|
|
21726
21814
|
attachment.individual = true;
|
|
21727
21815
|
}
|
|
@@ -21875,6 +21963,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21875
21963
|
for (let index = 0; index < chunks.length; index++) {
|
|
21876
21964
|
const { docs, attachmentOf } = chunks[index];
|
|
21877
21965
|
try {
|
|
21966
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
21878
21967
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
21879
21968
|
} catch (error) {
|
|
21880
21969
|
for (const later of chunks.slice(index)) {
|
|
@@ -21888,7 +21977,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21888
21977
|
return applySegment(ctx, indexName, attachment, doc, options);
|
|
21889
21978
|
});
|
|
21890
21979
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
21891
|
-
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
21980
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName, "segments"));
|
|
21892
21981
|
const inserted = [];
|
|
21893
21982
|
for (let i = 0; i < docs.length; i++) {
|
|
21894
21983
|
const doc = docs[i];
|
|
@@ -21929,7 +22018,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21929
22018
|
break;
|
|
21930
22019
|
}
|
|
21931
22020
|
if (index + 1 < chunks.length) {
|
|
21932
|
-
ctx.checkHeapPressure(indexName);
|
|
22021
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
21933
22022
|
await new Promise((r) => setTimeout(r, 0));
|
|
21934
22023
|
}
|
|
21935
22024
|
}
|
|
@@ -21940,7 +22029,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21940
22029
|
continue;
|
|
21941
22030
|
}
|
|
21942
22031
|
if (!errorOf.has(doc.docId)) continue;
|
|
21943
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
22032
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, errorOf.get(doc.docId)) });
|
|
21944
22033
|
}
|
|
21945
22034
|
return { succeeded, touchedVectorFields, unrecorded };
|
|
21946
22035
|
}
|
|
@@ -21954,8 +22043,11 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
21954
22043
|
);
|
|
21955
22044
|
}
|
|
21956
22045
|
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
21957
|
-
|
|
21958
|
-
|
|
22046
|
+
const nothingIngested = () => ({ succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] });
|
|
22047
|
+
if (admitted.length === 0) return nothingIngested();
|
|
22048
|
+
if (shutDownStarted(ctx)) {
|
|
22049
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
22050
|
+
return nothingIngested();
|
|
21959
22051
|
}
|
|
21960
22052
|
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
21961
22053
|
const manager = ctx.requireManager(indexName);
|
|
@@ -21973,6 +22065,10 @@ async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
|
21973
22065
|
} catch {
|
|
21974
22066
|
built2 = null;
|
|
21975
22067
|
}
|
|
22068
|
+
if (shutDownStarted(ctx)) {
|
|
22069
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
22070
|
+
return nothingIngested();
|
|
22071
|
+
}
|
|
21976
22072
|
if (built2 === null) {
|
|
21977
22073
|
return applyIndividually();
|
|
21978
22074
|
}
|
|
@@ -22023,7 +22119,9 @@ async function insertBatchViaSegments(ctx, indexName, documents2, options, hold)
|
|
|
22023
22119
|
const failed = [];
|
|
22024
22120
|
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents2, failed, hold);
|
|
22025
22121
|
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
22026
|
-
if (
|
|
22122
|
+
if (shutDownStarted(ctx)) {
|
|
22123
|
+
failAsShutDown(failed, outcome.unrecorded, docIdOf);
|
|
22124
|
+
} else if (outcome.unrecorded.length > 0) {
|
|
22027
22125
|
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
22028
22126
|
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
22029
22127
|
outcome.succeeded.push(...rest.succeeded);
|
|
@@ -22038,20 +22136,24 @@ async function insertDocumentBatch(ctx, indexName, documents2, options, hold) {
|
|
|
22038
22136
|
ctx.guardShutdown();
|
|
22039
22137
|
ctx.requireIndex(indexName);
|
|
22040
22138
|
validateBatchList(documents2, "documents");
|
|
22041
|
-
|
|
22139
|
+
const failed = [];
|
|
22140
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.scaleOutBeforeBatch(indexName, documents2.length));
|
|
22141
|
+
if (shutDownStarted(ctx)) {
|
|
22142
|
+
failAsShutDown(failed, documents2, providedDocIdOrEmpty);
|
|
22143
|
+
return { succeeded: [], failed };
|
|
22144
|
+
}
|
|
22042
22145
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
22043
22146
|
if (documents2.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
22044
22147
|
return insertBatchViaSegments(ctx, indexName, documents2, options, hold);
|
|
22045
22148
|
}
|
|
22046
|
-
const failed = [];
|
|
22047
22149
|
const inserts = await insertInChunks(ctx, indexName, documents2, options, hold, true, failed);
|
|
22048
22150
|
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
22049
22151
|
return { succeeded: inserts.succeeded, failed };
|
|
22050
22152
|
}
|
|
22051
22153
|
|
|
22052
22154
|
// src/engine/mutations/remove.ts
|
|
22053
|
-
function asRemoveError(err) {
|
|
22054
|
-
return
|
|
22155
|
+
function asRemoveError(ctx, err) {
|
|
22156
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
22055
22157
|
}
|
|
22056
22158
|
function validateRemoval(ctx, docId) {
|
|
22057
22159
|
ctx.guardShutdown();
|
|
@@ -22159,18 +22261,31 @@ async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, h
|
|
|
22159
22261
|
}
|
|
22160
22262
|
}
|
|
22161
22263
|
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
22162
|
-
await
|
|
22163
|
-
|
|
22164
|
-
|
|
22165
|
-
|
|
22166
|
-
|
|
22167
|
-
|
|
22168
|
-
|
|
22264
|
+
await unlessShutDown(
|
|
22265
|
+
ctx,
|
|
22266
|
+
() => runOutside(hold, async () => {
|
|
22267
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22268
|
+
if (refused.has(i)) continue;
|
|
22269
|
+
try {
|
|
22270
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
22271
|
+
} catch (err) {
|
|
22272
|
+
refused.set(i, err);
|
|
22273
|
+
}
|
|
22169
22274
|
}
|
|
22170
|
-
}
|
|
22171
|
-
|
|
22275
|
+
})
|
|
22276
|
+
);
|
|
22172
22277
|
return refused;
|
|
22173
22278
|
}
|
|
22279
|
+
function failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused) {
|
|
22280
|
+
const error = shutDownError();
|
|
22281
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22282
|
+
failed.push({ docId: docIds[i], error: refused.has(i) ? asRemoveError(ctx, refused.get(i)) : error });
|
|
22283
|
+
}
|
|
22284
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
22285
|
+
}
|
|
22286
|
+
function removalDocIdOf(docId) {
|
|
22287
|
+
return docId;
|
|
22288
|
+
}
|
|
22174
22289
|
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
22175
22290
|
ctx.guardShutdown();
|
|
22176
22291
|
ctx.requireIndex(indexName);
|
|
@@ -22182,8 +22297,16 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
22182
22297
|
let manager = first;
|
|
22183
22298
|
try {
|
|
22184
22299
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22300
|
+
if (shutDownStarted(ctx)) {
|
|
22301
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkStart);
|
|
22302
|
+
break;
|
|
22303
|
+
}
|
|
22185
22304
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
22186
22305
|
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
22306
|
+
if (shutDownStarted(ctx)) {
|
|
22307
|
+
failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused);
|
|
22308
|
+
break;
|
|
22309
|
+
}
|
|
22187
22310
|
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
22188
22311
|
const removed = [];
|
|
22189
22312
|
const prepared = [];
|
|
@@ -22193,10 +22316,15 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
22193
22316
|
if (refused.has(i)) throw refused.get(i);
|
|
22194
22317
|
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
22195
22318
|
} catch (err) {
|
|
22196
|
-
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
22319
|
+
failed.push({ docId: docIds[i], error: asRemoveError(ctx, err) });
|
|
22197
22320
|
}
|
|
22198
22321
|
}
|
|
22199
|
-
|
|
22322
|
+
if (shutDownStarted(ctx)) {
|
|
22323
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
22324
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
22325
|
+
break;
|
|
22326
|
+
}
|
|
22327
|
+
const applies = prepared.map((removal) => applyUnlessShutDown(ctx, applyOfRemoval(ctx, indexName, removal)));
|
|
22200
22328
|
try {
|
|
22201
22329
|
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
22202
22330
|
for (let i = 0; i < prepared.length; i++) {
|
|
@@ -22217,7 +22345,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
22217
22345
|
error = rollbackError;
|
|
22218
22346
|
}
|
|
22219
22347
|
removal.releaseVectorHolds?.();
|
|
22220
|
-
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
22348
|
+
failed.push({ docId: removal.docId, error: asRemoveError(ctx, error) });
|
|
22221
22349
|
continue;
|
|
22222
22350
|
}
|
|
22223
22351
|
removal.releaseVectorHolds?.();
|
|
@@ -22227,18 +22355,19 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
22227
22355
|
} finally {
|
|
22228
22356
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
22229
22357
|
}
|
|
22230
|
-
await settleRemovals(ctx, indexName, removed, hold);
|
|
22231
|
-
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
22358
|
+
await unlessShutDown(ctx, () => settleRemovals(ctx, indexName, removed, hold));
|
|
22359
|
+
if (!shutDownStarted(ctx)) manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
22232
22360
|
if (chunkEnd < docIds.length) {
|
|
22233
22361
|
await new Promise((r) => setTimeout(r, 0));
|
|
22234
22362
|
}
|
|
22235
22363
|
}
|
|
22236
22364
|
} finally {
|
|
22237
22365
|
const ending = manager;
|
|
22238
|
-
await ending?.publication.write(() => ending.endBatchRemove());
|
|
22366
|
+
await unlessShutDown(ctx, () => ending?.publication.write(() => ending.endBatchRemove()));
|
|
22239
22367
|
}
|
|
22368
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
22240
22369
|
ctx.checkHeapPressure(indexName);
|
|
22241
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22370
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
22242
22371
|
return { succeeded, failed };
|
|
22243
22372
|
}
|
|
22244
22373
|
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
@@ -22278,8 +22407,8 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
22278
22407
|
}
|
|
22279
22408
|
return { partitionDoc };
|
|
22280
22409
|
}
|
|
22281
|
-
function asUpdateError(err) {
|
|
22282
|
-
return
|
|
22410
|
+
function asUpdateError(ctx, err) {
|
|
22411
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
22283
22412
|
}
|
|
22284
22413
|
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
22285
22414
|
for (const update of updates) {
|
|
@@ -22476,24 +22605,39 @@ async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold,
|
|
|
22476
22605
|
const { document } = requireUpdateEntry(update);
|
|
22477
22606
|
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
22478
22607
|
} catch (err) {
|
|
22479
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
22608
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
22480
22609
|
}
|
|
22481
22610
|
}
|
|
22482
|
-
|
|
22483
|
-
|
|
22484
|
-
|
|
22485
|
-
|
|
22486
|
-
|
|
22487
|
-
|
|
22488
|
-
|
|
22611
|
+
let entry;
|
|
22612
|
+
try {
|
|
22613
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
22614
|
+
const writes = await prepareDocumentWrites(
|
|
22615
|
+
ctx,
|
|
22616
|
+
indexName,
|
|
22617
|
+
preparations.map((pending) => pending.preparation),
|
|
22618
|
+
hold,
|
|
22619
|
+
false
|
|
22620
|
+
);
|
|
22621
|
+
entry = writes.entry;
|
|
22622
|
+
} catch (err) {
|
|
22623
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
22624
|
+
failAsShutDown(failed, preparations, docIdOf);
|
|
22625
|
+
return [];
|
|
22626
|
+
}
|
|
22489
22627
|
const prepared = [];
|
|
22490
|
-
for (
|
|
22628
|
+
for (let position = 0; position < preparations.length; position++) {
|
|
22491
22629
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
22630
|
+
if (shutDownStarted(ctx)) {
|
|
22631
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
22632
|
+
failAsShutDown(failed, preparations, docIdOf, position);
|
|
22633
|
+
return [];
|
|
22634
|
+
}
|
|
22635
|
+
const { docId, preparation } = preparations[position];
|
|
22492
22636
|
try {
|
|
22493
22637
|
if (preparation.failure !== void 0) throw preparation.failure;
|
|
22494
22638
|
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
22495
22639
|
} catch (err) {
|
|
22496
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
22640
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
22497
22641
|
}
|
|
22498
22642
|
}
|
|
22499
22643
|
return prepared;
|
|
@@ -22506,17 +22650,26 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
22506
22650
|
const failed = [];
|
|
22507
22651
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
22508
22652
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22653
|
+
if (shutDownStarted(ctx)) {
|
|
22654
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkStart);
|
|
22655
|
+
break;
|
|
22656
|
+
}
|
|
22509
22657
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
22510
22658
|
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
22659
|
+
if (shutDownStarted(ctx)) {
|
|
22660
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
22661
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkEnd);
|
|
22662
|
+
break;
|
|
22663
|
+
}
|
|
22511
22664
|
try {
|
|
22512
22665
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
22513
22666
|
} catch (err) {
|
|
22514
|
-
const error = asUpdateError(err);
|
|
22667
|
+
const error = asUpdateError(ctx, err);
|
|
22515
22668
|
for (const update of prepared) failed.push({ docId: update.docId, error });
|
|
22516
22669
|
for (let i = chunkEnd; i < updates.length; i++) failed.push({ docId: reportedDocIdOf(updates[i]), error });
|
|
22517
22670
|
break;
|
|
22518
22671
|
}
|
|
22519
|
-
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
22672
|
+
const applies = prepared.map((update) => applyUnlessShutDown(ctx, applyOfUpdate(ctx, indexName, update)));
|
|
22520
22673
|
let failures;
|
|
22521
22674
|
try {
|
|
22522
22675
|
failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
@@ -22547,7 +22700,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
22547
22700
|
const update = prepared[i];
|
|
22548
22701
|
const failure = failures[i];
|
|
22549
22702
|
if (failure !== null) {
|
|
22550
|
-
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
22703
|
+
failed.push({ docId: update.docId, error: asUpdateError(ctx, failure.error) });
|
|
22551
22704
|
continue;
|
|
22552
22705
|
}
|
|
22553
22706
|
if (!update.buffered) settled.push(update);
|
|
@@ -22556,18 +22709,19 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
22556
22709
|
}
|
|
22557
22710
|
succeeded.push(update.docId);
|
|
22558
22711
|
}
|
|
22559
|
-
await settleUpdates(ctx, indexName, settled, hold);
|
|
22712
|
+
await unlessShutDown(ctx, () => settleUpdates(ctx, indexName, settled, hold));
|
|
22560
22713
|
if (chunkEnd < updates.length) {
|
|
22561
|
-
ctx.checkHeapPressure(indexName);
|
|
22714
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
22562
22715
|
await new Promise((r) => setTimeout(r, 0));
|
|
22563
22716
|
}
|
|
22564
22717
|
}
|
|
22718
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
22565
22719
|
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
22566
22720
|
for (const fieldPath of touchedVectorFields) {
|
|
22567
22721
|
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
22568
22722
|
}
|
|
22569
22723
|
ctx.checkHeapPressure(indexName);
|
|
22570
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22724
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
22571
22725
|
return { succeeded, failed };
|
|
22572
22726
|
}
|
|
22573
22727
|
|
|
@@ -22613,12 +22767,17 @@ async function withReservation(grouping, indexName, document, write) {
|
|
|
22613
22767
|
reservation.release();
|
|
22614
22768
|
}
|
|
22615
22769
|
}
|
|
22616
|
-
async function writeInGroups(grouping, indexName, items, write) {
|
|
22770
|
+
async function writeInGroups(ctx, grouping, indexName, items, write) {
|
|
22617
22771
|
const succeeded = [];
|
|
22618
22772
|
const failed = [];
|
|
22619
22773
|
await grouping.applyInGroups(indexName, {
|
|
22620
22774
|
items,
|
|
22621
22775
|
failed,
|
|
22776
|
+
refuseFrom: (start) => {
|
|
22777
|
+
if (!shutDownStarted(ctx)) return false;
|
|
22778
|
+
failAsShutDown(failed, items, docIdOf, start);
|
|
22779
|
+
return true;
|
|
22780
|
+
},
|
|
22622
22781
|
apply: async (group) => {
|
|
22623
22782
|
const result = await write(group);
|
|
22624
22783
|
for (const docId of result.succeeded) succeeded.push(docId);
|
|
@@ -22656,6 +22815,7 @@ async function insertDocumentBatch2(ctx, indexName, documents2, options, hold) {
|
|
|
22656
22815
|
}
|
|
22657
22816
|
const items = documents2.map((document) => ({ docId: providedDocId(document) ?? "", document }));
|
|
22658
22817
|
return writeInGroups(
|
|
22818
|
+
ctx,
|
|
22659
22819
|
grouping,
|
|
22660
22820
|
indexName,
|
|
22661
22821
|
items,
|
|
@@ -22688,6 +22848,7 @@ async function updateDocumentBatch2(ctx, indexName, updates, options, hold) {
|
|
|
22688
22848
|
}
|
|
22689
22849
|
noteGroupedChurn(ctx, indexName, updates.length);
|
|
22690
22850
|
return writeInGroups(
|
|
22851
|
+
ctx,
|
|
22691
22852
|
grouping,
|
|
22692
22853
|
indexName,
|
|
22693
22854
|
updates.map(updateItemOf),
|
|
@@ -22726,7 +22887,7 @@ var suggestionsCode = codeLoader(async () => {
|
|
|
22726
22887
|
const [msgpack, suggestionStore2, code7] = await Promise.all([
|
|
22727
22888
|
loadMessagePack(),
|
|
22728
22889
|
loadSuggestionStore(),
|
|
22729
|
-
import('./suggestions-leaf-
|
|
22890
|
+
import('./suggestions-leaf-JNGH2JYC.mjs')
|
|
22730
22891
|
]);
|
|
22731
22892
|
code7.bindSuggestionsHost(optionalCodeHost(), msgpack, suggestionStore2);
|
|
22732
22893
|
return code7;
|
|
@@ -24353,7 +24514,7 @@ var workerCopies = codeLoader(async () => {
|
|
|
24353
24514
|
const [segmentCodec2, segmentMerge2, code7] = await Promise.all([
|
|
24354
24515
|
loadSegmentCodec(),
|
|
24355
24516
|
loadSegmentMerge(),
|
|
24356
|
-
import('./worker-copies-
|
|
24517
|
+
import('./worker-copies-2OWLLBCV.mjs')
|
|
24357
24518
|
]);
|
|
24358
24519
|
code7.bindWorkerCopiesHost(optionalCodeHost(), segmentCodec2, segmentMerge2);
|
|
24359
24520
|
return code7;
|
|
@@ -24578,7 +24739,7 @@ var loaded2 = null;
|
|
|
24578
24739
|
var loading3 = null;
|
|
24579
24740
|
function importOptionalReads() {
|
|
24580
24741
|
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
24581
|
-
loading3 ??= import('./optional-reads-
|
|
24742
|
+
loading3 ??= import('./optional-reads-TN3PT6M4.mjs').then(
|
|
24582
24743
|
(code7) => {
|
|
24583
24744
|
code7.bindOptionalReadsHost(optionalCodeHost());
|
|
24584
24745
|
loaded2 = code7;
|
|
@@ -24925,6 +25086,7 @@ async function applyInGroups(projection, indexName, write) {
|
|
|
24925
25086
|
);
|
|
24926
25087
|
let start = 0;
|
|
24927
25088
|
while (start < bounded.length) {
|
|
25089
|
+
if (write.refuseFrom?.(start) === true) return null;
|
|
24928
25090
|
let group;
|
|
24929
25091
|
try {
|
|
24930
25092
|
group = projection.takeGroup(indexName, bounds, start, mostDocuments, countExactly);
|
|
@@ -26483,7 +26645,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
26483
26645
|
|
|
26484
26646
|
// src/engine/durability-config-code.ts
|
|
26485
26647
|
var durabilityConfig = codeLoader(
|
|
26486
|
-
() => import('./durability-config-
|
|
26648
|
+
() => import('./durability-config-HJETGZVB.mjs').then((code7) => {
|
|
26487
26649
|
code7.bindDurabilityConfigHost(optionalCodeHost());
|
|
26488
26650
|
return code7;
|
|
26489
26651
|
}),
|
|
@@ -27081,7 +27243,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
27081
27243
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
27082
27244
|
}
|
|
27083
27245
|
const durabilityWiring = durabilityTier === null ? null : (await Promise.all([loadStorageCode(), loadOptionalWrites()]))[0];
|
|
27084
|
-
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
27246
|
+
const filesystemDurability = durabilityTier?.kind === "wal" && durabilityTier.indexedDB === void 0;
|
|
27085
27247
|
const executor = createDirectExecutor({
|
|
27086
27248
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
27087
27249
|
vectorStorage: filesystemDurability ? "disk" : "memory",
|
|
@@ -27359,8 +27521,13 @@ async function shutdownEngine(core2) {
|
|
|
27359
27521
|
await core2.analysisRebuild.shutdown();
|
|
27360
27522
|
await core2.querySuggestionHooks?.shutdown();
|
|
27361
27523
|
core2.indexState.dispose();
|
|
27524
|
+
let durabilityFailure = null;
|
|
27362
27525
|
if (durability) {
|
|
27363
|
-
|
|
27526
|
+
try {
|
|
27527
|
+
await durability.manager.shutdown();
|
|
27528
|
+
} catch (error) {
|
|
27529
|
+
durabilityFailure = { error };
|
|
27530
|
+
}
|
|
27364
27531
|
}
|
|
27365
27532
|
const vectorDisposals = [];
|
|
27366
27533
|
for (const [name] of indexRegistry) {
|
|
@@ -27394,6 +27561,7 @@ async function shutdownEngine(core2) {
|
|
|
27394
27561
|
await Promise.all(vectorDisposals);
|
|
27395
27562
|
eventHandlers.clear();
|
|
27396
27563
|
indexRegistry.clear();
|
|
27564
|
+
if (durabilityFailure !== null) throw durabilityFailure.error;
|
|
27397
27565
|
}
|
|
27398
27566
|
|
|
27399
27567
|
// src/core/keyword-score/layout.ts
|