@delali/narsil 0.3.1-next.183 → 0.3.1-next.184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-ZZUXMJIK.mjs → chunk-CPSHS46N.mjs} +129 -165
- package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
- package/dist/browser/{optional-writes-IQFTXANH.mjs → optional-writes-ZTBHYQMN.mjs} +619 -314
- package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
- package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
- package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
- package/dist/browser/{storage-leaf-2CO6SQ7R.mjs → storage-leaf-4XMCITJ6.mjs} +27 -9
- package/dist/{chunk-PK3BGGHO.mjs → chunk-2KCCQK4R.mjs} +1 -1
- package/dist/{chunk-JVR5KMO4.mjs → chunk-2YWWHMVC.mjs} +2 -2
- package/dist/{chunk-GGG5R4CU.mjs → chunk-3D4IMFSE.mjs} +2 -2
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/{react-native/chunks/chunk-G4YYO5TC.mjs → chunk-4JBTALUC.mjs} +1 -1
- package/dist/{chunk-PXEGKQHS.mjs → chunk-5BUEBIXR.mjs} +2 -2
- package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
- package/dist/{chunk-NPXORGMK.mjs → chunk-7WXGUTWK.mjs} +3 -3
- package/dist/{chunk-2DK74OQL.mjs → chunk-A3OAFUZI.mjs} +1 -1
- package/dist/{chunk-632ABEIR.mjs → chunk-BVZWWMOU.mjs} +1 -1
- package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
- package/dist/{chunk-F2UOTRY7.mjs → chunk-CODGHJVW.mjs} +4 -4
- package/dist/{chunk-6ICOWB6I.mjs → chunk-CYCVPTTR.mjs} +1 -1
- package/dist/{chunk-WUWSUJ2X.mjs → chunk-DU23J5JG.mjs} +5 -5
- package/dist/{chunk-C77OBNPP.mjs → chunk-EDEE7ETQ.mjs} +1 -1
- package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
- package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
- package/dist/{chunk-EGWBZSHK.mjs → chunk-FCQMZDRI.mjs} +2 -2
- package/dist/{chunk-MCAJOLVK.mjs → chunk-G3XRHAJA.mjs} +2 -2
- package/dist/{chunk-FIXUVFUC.mjs → chunk-GGIW6INA.mjs} +7 -6
- package/dist/{chunk-OQANB7SJ.mjs → chunk-H4QYQ5DE.mjs} +1 -1
- package/dist/{chunk-V7F5DOSC.mjs → chunk-HQPK73EW.mjs} +3 -3
- package/dist/{chunk-CKS4QQEC.mjs → chunk-II6XF6M4.mjs} +5 -5
- package/dist/{chunk-3OEXOSWI.mjs → chunk-LR6JJVML.mjs} +4 -4
- package/dist/{chunk-TTTBE4TB.mjs → chunk-M5I64AQW.mjs} +3 -3
- package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
- package/dist/{chunk-M3H7CTIC.mjs → chunk-O337MDRJ.mjs} +5 -5
- package/dist/{chunk-C3EINJNJ.mjs → chunk-SEM3PBRH.mjs} +1 -1
- package/dist/{chunk-CWXR7OP4.mjs → chunk-SL53DHVE.mjs} +804 -527
- package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
- package/dist/{chunk-SA7U67XF.mjs → chunk-URVH6BLO.mjs} +2 -2
- package/dist/{chunk-T3YI4KN6.mjs → chunk-VYSWVQCE.mjs} +1 -1
- package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
- package/dist/{chunk-ZVZ2V2PZ.mjs → chunk-WSIU6YXQ.mjs} +39 -20
- package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
- package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
- package/dist/{direct-executor-JXYIL4OQ.mjs → direct-executor-37IH3ISS.mjs} +5 -4
- package/dist/distribution.mjs +32 -31
- package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +5 -4
- package/dist/index.react-native.mjs +4 -3
- package/dist/{long-query-thread-HPKGRYWM.mjs → long-query-thread-PU2XT527.mjs} +15 -14
- package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
- package/dist/{optional-writes-QE7QUI45.mjs → optional-writes-YIVJMQ4D.mjs} +7 -6
- package/dist/{paged-partition-archive-IK2UOY2G.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +5 -4
- package/dist/persistence/durability/checkpoint-worker.mjs +13 -12
- package/dist/{query-actions-COEFVUH3.mjs → query-actions-7EOKT3WD.mjs} +5 -4
- package/dist/query-suggestions/merge-worker.mjs +8 -7
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.mjs +6 -5
- package/dist/react-native/chunks/{chunk-DZ4FQW5K.mjs → chunk-7TPO4FLE.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-6A5XE2MA.mjs → chunk-CBU4M3EU.mjs} +803 -526
- package/dist/react-native/chunks/{chunk-DN2GVVR4.mjs → chunk-IEY4DARA.mjs} +1 -1
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
- package/dist/react-native/chunks/{chunk-UPN4NUXU.mjs → chunk-T5HF3FW3.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XNIH3WIB.mjs → chunk-ZQPSXN6C.mjs} +1 -1
- package/dist/{chunk-YWSIRKKQ.mjs → react-native/chunks/chunk-ZUFOYZ5P.mjs} +1 -1
- package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
- package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
- package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
- package/dist/react-native/chunks/{optional-writes-YZ7WTR7V.mjs → optional-writes-QETIM6BI.mjs} +5 -4
- package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
- package/dist/react-native/chunks/{restore-queue-IGKD6Q2Y.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
- package/dist/react-native/chunks/{runtime-BUCSAK4U.mjs → runtime-UOHD2XUJ.mjs} +23 -3
- package/dist/react-native/chunks/{segment-codec-leaf-JTTGDT3G.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
- package/dist/react-native/chunks/{segment-merge-leaf-D5ZFJRUB.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
- package/dist/react-native/chunks/{storage-leaf-ZKIM6IF6.mjs → storage-leaf-LNOLKWZB.mjs} +36 -17
- package/dist/react-native/chunks/{store-leaf-PHBOTT5J.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
- package/dist/react-native/chunks/{suggestions-leaf-R7MIH6Q3.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
- package/dist/react-native/chunks/{worker-copies-U35KPVJ7.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
- package/dist/react-native/partial-words.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +4 -3
- package/dist/react-native/sync.mjs +4 -3
- package/dist/react-native/synonyms.mjs +4 -3
- package/dist/{rerank-LYL3D7LU.mjs → rerank-KTV5L7MZ.mjs} +5 -4
- package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
- package/dist/{restore-queue-LF4DTVJC.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
- package/dist/{runtime-4GKUIAKV.mjs → runtime-USQYKN3I.mjs} +23 -3
- package/dist/{segment-codec-leaf-ACALKS2C.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
- package/dist/{segment-merge-leaf-SKMPASFK.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
- package/dist/server.mjs +22 -21
- package/dist/{storage-leaf-OJ55R4C5.mjs → storage-leaf-ZI3H2WJY.mjs} +26 -25
- package/dist/{store-leaf-CWHHEDRK.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
- package/dist/{suggestions-leaf-ZTU3ZTWJ.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
- package/dist/sync/filesystem.mjs +10 -9
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.mjs +5 -4
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +5 -4
- package/dist/{thread-BEGFRULV.mjs → thread-BJXE77AI.mjs} +14 -13
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/{worker-copies-373P6IYZ.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
- package/dist/workers/entry.mjs +11 -10
- package/package.json +7 -7
- package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
- package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { encodeDocument } from './chunk-KLVMK3R5.mjs';
|
|
2
|
-
import { workerResourceLimits, admittedHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal } from './chunk-
|
|
2
|
+
import { workerResourceLimits, admittedHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal } from './chunk-F4OOBPKA.mjs';
|
|
3
3
|
import { headerAt, MAP_TAGS, decode, Decoder } from './chunk-GXCDWLQI.mjs';
|
|
4
4
|
import { newSyncEpoch, isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
5
5
|
import { english } from './chunk-7T4LAYT4.mjs';
|
|
6
6
|
import { patternSearchModule, PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, matchPattern, claimsTypoPreparation, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, DEFAULT_PATTERN_WORK_CAP, bitsetSet, discardStoredValues, MAX_FILTER_DEPTH, createBitSet, installStoredFields, storedValueBytes, bitsetIterator, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIsEmpty, prepareStoredValues, fuzzyTermMatches, GrowableUint32, checkedSize, buildStoredValues, bitsetFromSet, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyPrefixOf, boundedLevenshtein, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, FILTER_OPERAND_COPY_DEPTH } from './chunk-YM2HI5N5.mjs';
|
|
7
7
|
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, isStringValuedBase, flattenSchema, schemaFieldsOf, getNestedValue, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, requireLoadedMessagePack, vectorDimensionOf, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, isWordIndexedBase, PROTOTYPE_POLLUTION_KEYS, tokenizeOptions, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, getFlatSchema, getOrCreateFieldNameIndex, MAX_NESTING_DEPTH, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-X5NHWOXT.mjs';
|
|
8
|
-
import { detectRuntime, splitWorkerBudget, resolveWorkerCount, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, countEngineThread, requirePinnedState,
|
|
9
|
-
import {
|
|
8
|
+
import { detectRuntime, splitWorkerBudget, resolveWorkerCount, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, countEngineThread, requirePinnedState, createDocIdMap, SHUTDOWN_TIMEOUT_MS, stopOfSignal, requireSearchActive, resolveWorkerEntry, missingWorkerEntry, signalOfSearch, answerFromOneState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, sharedStopOfSignal, engineThreadYoungGenerationBytes, resolveRequestThreadCount, spawnNodeWorker, bindSearchSignal, WORKER_YOUNG_GENERATION_MB, nodeThreadId, youngGenerationBytesOfThisV8 } from './chunk-6SZQETSC.mjs';
|
|
9
|
+
import { changeIndexOutsideExternalWaits, turnsAreBudgeted, turnIsSpent, yieldTurn, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, noteThreadSlotLive, threadSlotOfWorker, indexChangeOutsideExternalWaits, PROCESS_LIMIT_REFRESH_INTERVAL_MS, countWrite, threadSlotGone, noteThreadSlotGone, adoptSharedTurns, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_BLOCK_MAX_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_ATTACH_RETRY_MS, MAX_WASM_PAGES, MAX_VECTOR_ORDINALS, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_DOC_ID_TABLE_BYTES, LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, DISK_STORAGE_BLOCK_BYTES, waitOutsideCountedWrite, collectAndReturnPages } from './chunk-UPDNNOTG.mjs';
|
|
10
|
+
import { MIN_DOCUMENTS_FOR_SEGMENTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, MAX_PARTITIONS, DEFAULT_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, BATCH_CHUNK_SIZE, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_CALLS_ACROSS_INDEX_CHANGES, DEFAULT_PAGE_SIZE, CURSOR_BINDING_BUFFER_BYTES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, DEFAULT_OFFSET, MIN_SWEEP_INTERVAL_MS, MIN_RANK_CONSTANT, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, FUSION_TIE_INSERTION_LIMIT, DEFAULT_RANK_CONSTANT, DEFAULT_LINEAR_ALPHA, STATISTICS_BROADCAST_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, EXPORT_CHUNK_BYTES, DEFAULT_MAX_PARTITIONS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, ANALYSIS_CHANGE_MEMBERS, MIN_HYBRID_DEPTH, MAX_SUGGEST_SCATTER_LIMIT, MAX_SUGGEST_LIMIT, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_SUGGEST_LIMIT, DEFAULT_MAX_PER_GROUP, HEAP_PRESSURE_WARNING_FRACTION, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-3ECIRFS5.mjs';
|
|
10
11
|
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, requireSearchBackendSettings, NATIVE_NEEDS_ROOM, NATIVE_OK } from './chunk-D6FYUTFN.mjs';
|
|
11
12
|
import { PLACEMENT_ROOM_REQUESTS, NEAREST_LINK_FALLBACK_HOSTS, MIN_M, MAX_M, MAX_LAYER_CAP, INITIAL_UPPER_SLOTS, INITIAL_LIST_CAPACITY, FARTHEST_REJECTED_LINKS_KEPT, EARLIER_RELEASE_MIN_M, DEFAULT_EF_SEARCH, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, CONNECT_RESTARTS_BEFORE_ONE_PASS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD } from './chunk-ZXX5GZEX.mjs';
|
|
12
13
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS, NATIVE_CRC32_MIN_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, MAX_DOCUMENT_ENCODING_DEPTH, DOCUMENT_REPLY_ENVELOPE_DEPTH, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER } from './chunk-4PY4JYMY.mjs';
|
|
@@ -3052,11 +3053,11 @@ function captureFrozen(input, output2) {
|
|
|
3052
3053
|
spellings: captured.spellings === null ? null : copy(captured.spellings),
|
|
3053
3054
|
removalIndex: captured.removalIndex === null ? null : copy(captured.removalIndex)
|
|
3054
3055
|
};
|
|
3055
|
-
const
|
|
3056
|
-
return { raw, writable };
|
|
3056
|
+
const writable2 = copy(output2);
|
|
3057
|
+
return { raw, writable: writable2 };
|
|
3057
3058
|
}
|
|
3058
3059
|
var heldFrozen = /* @__PURE__ */ new WeakMap();
|
|
3059
|
-
function rawsOf({ raw, writable }) {
|
|
3060
|
+
function rawsOf({ raw, writable: writable2 }) {
|
|
3060
3061
|
const { terms, surfaces, spellings, removalIndex } = raw;
|
|
3061
3062
|
return [
|
|
3062
3063
|
raw.documentCount,
|
|
@@ -3081,16 +3082,16 @@ function rawsOf({ raw, writable }) {
|
|
|
3081
3082
|
removalIndex?.starts,
|
|
3082
3083
|
removalIndex?.rows,
|
|
3083
3084
|
removalIndex?.termOfSlot,
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3085
|
+
writable2.tombstones,
|
|
3086
|
+
writable2.tombstoneCount,
|
|
3087
|
+
writable2.removedDocuments,
|
|
3088
|
+
writable2.removedOccurrences,
|
|
3089
|
+
writable2.removedSurfaces,
|
|
3090
|
+
writable2.removedSpellings,
|
|
3091
|
+
writable2.pendingHeader,
|
|
3092
|
+
writable2.pendingDocuments,
|
|
3093
|
+
writable2.pendingTokens,
|
|
3094
|
+
writable2.pendingUnits
|
|
3094
3095
|
];
|
|
3095
3096
|
}
|
|
3096
3097
|
function stillHeld(raws, held) {
|
|
@@ -3106,7 +3107,7 @@ function readHeld(input, output2, captured) {
|
|
|
3106
3107
|
const raws = rawsOf(captured);
|
|
3107
3108
|
const known = heldFrozen.get(input);
|
|
3108
3109
|
if (known !== void 0 && known.output === output2 && stillHeld(raws, known)) return known;
|
|
3109
|
-
const { raw, writable } = captured;
|
|
3110
|
+
const { raw, writable: writable2 } = captured;
|
|
3110
3111
|
const regions = new Regions();
|
|
3111
3112
|
const documentCount = raw.documentCount;
|
|
3112
3113
|
requireSegment(Number.isInteger(documentCount) && documentCount >= 0 && documentCount <= 4294967295);
|
|
@@ -3134,16 +3135,16 @@ function readHeld(input, output2, captured) {
|
|
|
3134
3135
|
}
|
|
3135
3136
|
};
|
|
3136
3137
|
const state = {
|
|
3137
|
-
tombstones: regions.array(
|
|
3138
|
-
tombstoneCount: regions.array(
|
|
3139
|
-
removedDocuments: regions.array(
|
|
3140
|
-
removedOccurrences: regions.array(
|
|
3141
|
-
removedSurfaces: regions.array(
|
|
3142
|
-
removedSpellings: regions.array(
|
|
3143
|
-
pendingHeader: regions.array(
|
|
3144
|
-
pendingDocuments: regions.array(
|
|
3145
|
-
pendingTokens: regions.array(
|
|
3146
|
-
pendingUnits: regions.array(
|
|
3138
|
+
tombstones: regions.array(writable2.tombstones, "Uint32Array", Math.max(1, Math.ceil(documentCount / 32)), true),
|
|
3139
|
+
tombstoneCount: regions.array(writable2.tombstoneCount, "Uint32Array", 1, true),
|
|
3140
|
+
removedDocuments: regions.array(writable2.removedDocuments, "Uint32Array", terms.counts.length, true),
|
|
3141
|
+
removedOccurrences: regions.array(writable2.removedOccurrences, "Float64Array", terms.counts.length, true),
|
|
3142
|
+
removedSurfaces: regions.array(writable2.removedSurfaces, "Uint32Array", surfaces?.counts.length ?? 0, true),
|
|
3143
|
+
removedSpellings: regions.array(writable2.removedSpellings, "Uint32Array", spellings?.counts.length ?? 0, true),
|
|
3144
|
+
pendingHeader: regions.array(writable2.pendingHeader, "Uint32Array", PENDING_HEADER_WORDS, true),
|
|
3145
|
+
pendingDocuments: regions.array(writable2.pendingDocuments, "Uint32Array", void 0, true),
|
|
3146
|
+
pendingTokens: regions.array(writable2.pendingTokens, "Uint32Array", void 0, true),
|
|
3147
|
+
pendingUnits: regions.array(writable2.pendingUnits, "Uint16Array", void 0, true)
|
|
3147
3148
|
};
|
|
3148
3149
|
requireSegment(state.pendingTokens.length % PENDING_TOKEN_WORDS === 0);
|
|
3149
3150
|
regions.check();
|
|
@@ -6791,10 +6792,10 @@ var Regions2 = class {
|
|
|
6791
6792
|
starts = [];
|
|
6792
6793
|
ends = [];
|
|
6793
6794
|
writable = [];
|
|
6794
|
-
read(input, kind, length,
|
|
6795
|
+
read(input, kind, length, writable2 = true) {
|
|
6795
6796
|
const value = typedRegion(input, kind);
|
|
6796
6797
|
requireSegment(value !== null && (length === void 0 || value.length === length));
|
|
6797
|
-
if (
|
|
6798
|
+
if (writable2) this.writes.push(value);
|
|
6798
6799
|
else this.reads.push(value);
|
|
6799
6800
|
const byteLength = value.byteLength;
|
|
6800
6801
|
if (byteLength > 0) {
|
|
@@ -6802,19 +6803,19 @@ var Regions2 = class {
|
|
|
6802
6803
|
this.buffers.push(value.buffer);
|
|
6803
6804
|
this.starts.push(start);
|
|
6804
6805
|
this.ends.push(start + byteLength);
|
|
6805
|
-
this.writable.push(
|
|
6806
|
+
this.writable.push(writable2);
|
|
6806
6807
|
}
|
|
6807
6808
|
return value;
|
|
6808
6809
|
}
|
|
6809
6810
|
check() {
|
|
6810
|
-
const { buffers: buffers2, starts, ends, writable } = this;
|
|
6811
|
+
const { buffers: buffers2, starts, ends, writable: writable2 } = this;
|
|
6811
6812
|
for (let at = 1; at < buffers2.length; at++) {
|
|
6812
6813
|
const buffer = buffers2[at];
|
|
6813
6814
|
const start = starts[at];
|
|
6814
6815
|
const end = ends[at];
|
|
6815
|
-
const writes =
|
|
6816
|
+
const writes = writable2[at];
|
|
6816
6817
|
for (let previous = 0; previous < at; previous++)
|
|
6817
|
-
if (buffers2[previous] === buffer && (writes ||
|
|
6818
|
+
if (buffers2[previous] === buffer && (writes || writable2[previous]) && start < ends[previous] && starts[previous] < end)
|
|
6818
6819
|
requireSegment(false);
|
|
6819
6820
|
}
|
|
6820
6821
|
}
|
|
@@ -11465,6 +11466,7 @@ function searchHostPart() {
|
|
|
11465
11466
|
listBindingOf,
|
|
11466
11467
|
MAX_HYBRID_DEPTH,
|
|
11467
11468
|
MAX_PHRASE_WORDS,
|
|
11469
|
+
MAX_CALLS_ACROSS_INDEX_CHANGES,
|
|
11468
11470
|
MAX_QUERY_PHRASES,
|
|
11469
11471
|
MAX_RERANK_TIMEOUT_MS,
|
|
11470
11472
|
MAX_RERANK_WINDOW,
|
|
@@ -12680,7 +12682,7 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
12680
12682
|
|
|
12681
12683
|
// src/core/segment-codec-code.ts
|
|
12682
12684
|
var segmentCodec = codeLoader(async () => {
|
|
12683
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-
|
|
12685
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-DO75VKEV.mjs')]);
|
|
12684
12686
|
code7.bindSegmentCodecHost(optionalCodeHost(), msgpack);
|
|
12685
12687
|
return code7;
|
|
12686
12688
|
}, "segment codec");
|
|
@@ -12688,7 +12690,7 @@ var loadSegmentCodec = segmentCodec.load;
|
|
|
12688
12690
|
|
|
12689
12691
|
// src/query-suggestions/store-code.ts
|
|
12690
12692
|
var suggestionStore = codeLoader(async () => {
|
|
12691
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-
|
|
12693
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-YIH36Y5J.mjs')]);
|
|
12692
12694
|
code7.bindSuggestionStoreHost(optionalCodeHost(), msgpack);
|
|
12693
12695
|
return code7;
|
|
12694
12696
|
}, "query suggestions store");
|
|
@@ -12747,7 +12749,7 @@ function markSimdUnavailable() {
|
|
|
12747
12749
|
|
|
12748
12750
|
// src/workers/vector-core-code.ts
|
|
12749
12751
|
var vectorCore = codeLoader(
|
|
12750
|
-
() => import('./core-leaf-
|
|
12752
|
+
() => import('./core-leaf-NEUM7PU3.mjs').then((code7) => {
|
|
12751
12753
|
code7.bindVectorCoreHost(optionalCodeHost());
|
|
12752
12754
|
if (!code7.simdModuleLoaded()) markSimdUnavailable();
|
|
12753
12755
|
return code7;
|
|
@@ -12769,7 +12771,7 @@ var storageCode = codeLoader(async () => {
|
|
|
12769
12771
|
loadSegmentMerge(),
|
|
12770
12772
|
loadSuggestionStore(),
|
|
12771
12773
|
loadVectorCore(),
|
|
12772
|
-
import('./storage-leaf-
|
|
12774
|
+
import('./storage-leaf-ZI3H2WJY.mjs')
|
|
12773
12775
|
]);
|
|
12774
12776
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
12775
12777
|
return code7;
|
|
@@ -15578,10 +15580,10 @@ function uint323(value) {
|
|
|
15578
15580
|
var MergeRegions = class _MergeRegions {
|
|
15579
15581
|
reads = [];
|
|
15580
15582
|
writes = [];
|
|
15581
|
-
read(input, kind, length,
|
|
15583
|
+
read(input, kind, length, writable2 = false) {
|
|
15582
15584
|
const value = typedRegion(input, kind);
|
|
15583
15585
|
requireSegment(value !== null && (length === void 0 || value.length === length));
|
|
15584
|
-
(
|
|
15586
|
+
(writable2 ? this.writes : this.reads).push(value);
|
|
15585
15587
|
return value;
|
|
15586
15588
|
}
|
|
15587
15589
|
check() {
|
|
@@ -15599,11 +15601,11 @@ var MergeRegions = class _MergeRegions {
|
|
|
15599
15601
|
}
|
|
15600
15602
|
}
|
|
15601
15603
|
}
|
|
15602
|
-
strings(input, stride,
|
|
15603
|
-
const counts = this.read(input.counts, "Uint32Array", void 0,
|
|
15604
|
-
const text = this.read(input.text, "Uint8Array", void 0,
|
|
15605
|
-
const offsets = this.read(input.offsets, "Uint32Array",
|
|
15606
|
-
if (!
|
|
15604
|
+
strings(input, stride, writable2 = false) {
|
|
15605
|
+
const counts = this.read(input.counts, "Uint32Array", void 0, writable2);
|
|
15606
|
+
const text = this.read(input.text, "Uint8Array", void 0, writable2);
|
|
15607
|
+
const offsets = this.read(input.offsets, "Uint32Array", writable2 ? void 0 : counts.length * stride + 1, writable2);
|
|
15608
|
+
if (!writable2) requireSegment(counts.length <= 4294967295 && text.length <= 4294967295);
|
|
15607
15609
|
return { text, offsets, counts };
|
|
15608
15610
|
}
|
|
15609
15611
|
source(input) {
|
|
@@ -15649,11 +15651,11 @@ var MergeRegions = class _MergeRegions {
|
|
|
15649
15651
|
...positions2
|
|
15650
15652
|
};
|
|
15651
15653
|
}
|
|
15652
|
-
positions(input,
|
|
15654
|
+
positions(input, writable2, count) {
|
|
15653
15655
|
requireSegment(input.positionOffsets === null === (input.positionValues === null));
|
|
15654
|
-
const positionOffsets = input.positionOffsets === null ? null : this.read(input.positionOffsets, "Uint32Array", count,
|
|
15655
|
-
const positionValues = input.positionValues === null ? null : this.read(input.positionValues, "Uint32Array", void 0,
|
|
15656
|
-
if (!
|
|
15656
|
+
const positionOffsets = input.positionOffsets === null ? null : this.read(input.positionOffsets, "Uint32Array", count, writable2);
|
|
15657
|
+
const positionValues = input.positionValues === null ? null : this.read(input.positionValues, "Uint32Array", void 0, writable2);
|
|
15658
|
+
if (!writable2) requireSegment(positionValues === null || positionValues.length <= 4294967295);
|
|
15657
15659
|
return { positionOffsets, positionValues };
|
|
15658
15660
|
}
|
|
15659
15661
|
};
|
|
@@ -21492,7 +21494,7 @@ function hasLanguage(name) {
|
|
|
21492
21494
|
|
|
21493
21495
|
// src/narsil/restore-queue-code.ts
|
|
21494
21496
|
var restoreQueue = codeLoader(async () => {
|
|
21495
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
21497
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-ES7TDJ5V.mjs')]);
|
|
21496
21498
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
21497
21499
|
return code7;
|
|
21498
21500
|
}, "restore");
|
|
@@ -21719,7 +21721,7 @@ function storedBM25Params(params) {
|
|
|
21719
21721
|
|
|
21720
21722
|
// src/workers/vector-code.ts
|
|
21721
21723
|
var vectorCode = codeLoader(async () => {
|
|
21722
|
-
const [vectorCore2, code7] = await Promise.all([loadVectorCore(), import('./module-
|
|
21724
|
+
const [vectorCore2, code7] = await Promise.all([loadVectorCore(), import('./module-Z3CL4GJ4.mjs')]);
|
|
21723
21725
|
code7.bindVectorHost(optionalCodeHost(), vectorCore2);
|
|
21724
21726
|
return code7;
|
|
21725
21727
|
}, "vector");
|
|
@@ -22057,7 +22059,7 @@ function storedArrayPositionGap(value) {
|
|
|
22057
22059
|
|
|
22058
22060
|
// src/engine/embed-code.ts
|
|
22059
22061
|
var embedCode = codeLoader(
|
|
22060
|
-
() => import('./embed-
|
|
22062
|
+
() => import('./embed-2QMYCD2Q.mjs').then((code7) => {
|
|
22061
22063
|
code7.bindEmbedHost(optionalCodeHost());
|
|
22062
22064
|
return code7;
|
|
22063
22065
|
}),
|
|
@@ -22067,7 +22069,7 @@ var loadEmbedCode = embedCode.load;
|
|
|
22067
22069
|
|
|
22068
22070
|
// src/engine/optional-writes-code.ts
|
|
22069
22071
|
var optionalWrites = codeLoader(async () => {
|
|
22070
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
22072
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-YIVJMQ4D.mjs')]);
|
|
22071
22073
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
22072
22074
|
return code7;
|
|
22073
22075
|
}, "write");
|
|
@@ -22271,21 +22273,9 @@ function requireMatchingDocId(docId, document) {
|
|
|
22271
22273
|
{ docId, documentId: embeddedDocId }
|
|
22272
22274
|
);
|
|
22273
22275
|
}
|
|
22274
|
-
function asBatchInsertError(err) {
|
|
22275
|
-
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
22276
|
-
}
|
|
22277
|
-
function createPendingAdmissions() {
|
|
22278
|
-
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
22279
|
-
}
|
|
22280
22276
|
function capsEachPartition(ctx, manager) {
|
|
22281
22277
|
return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
|
|
22282
22278
|
}
|
|
22283
|
-
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
22284
|
-
pending.total += 1;
|
|
22285
|
-
if (!capsEachPartition(ctx, manager)) return;
|
|
22286
|
-
const partitionId = manager.routePartition(docId);
|
|
22287
|
-
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
22288
|
-
}
|
|
22289
22279
|
function admitInsert(ctx, indexName, manager, docId, pending) {
|
|
22290
22280
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
22291
22281
|
if (rebalancing) {
|
|
@@ -22317,66 +22307,6 @@ function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
|
|
|
22317
22307
|
ctx.rebalanceTargetPartitionCount(indexName)
|
|
22318
22308
|
);
|
|
22319
22309
|
}
|
|
22320
|
-
function collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed) {
|
|
22321
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22322
|
-
try {
|
|
22323
|
-
validateRequiredFields(documents2[i], required);
|
|
22324
|
-
} catch (err) {
|
|
22325
|
-
chunkFailedIndexes.add(i);
|
|
22326
|
-
failed.push({ docId: providedDocId(documents2[i]) ?? "", error: asBatchInsertError(err) });
|
|
22327
|
-
}
|
|
22328
|
-
}
|
|
22329
|
-
}
|
|
22330
|
-
async function embedChunkDocuments(entry, documents2, chunkStart, chunkEnd, signal, chunkFailedIndexes, failed) {
|
|
22331
|
-
if (entry.config.embedding === void 0) return;
|
|
22332
|
-
const { assertDocumentCarriesMappedVectors, embedBatchDocumentFields } = await loadEmbedCode();
|
|
22333
|
-
if (entry.embeddingAdapter === null) {
|
|
22334
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22335
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
22336
|
-
try {
|
|
22337
|
-
assertDocumentCarriesMappedVectors(
|
|
22338
|
-
documents2[i],
|
|
22339
|
-
entry.config.embedding,
|
|
22340
|
-
entry.embeddingAdapterName
|
|
22341
|
-
);
|
|
22342
|
-
} catch (err) {
|
|
22343
|
-
chunkFailedIndexes.add(i);
|
|
22344
|
-
failed.push({
|
|
22345
|
-
docId: providedDocId(documents2[i]) ?? "",
|
|
22346
|
-
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
22347
|
-
});
|
|
22348
|
-
}
|
|
22349
|
-
}
|
|
22350
|
-
return;
|
|
22351
|
-
}
|
|
22352
|
-
const embeddableSlice = [];
|
|
22353
|
-
const embeddableOriginalIndexes = [];
|
|
22354
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22355
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
22356
|
-
embeddableSlice.push(documents2[i]);
|
|
22357
|
-
embeddableOriginalIndexes.push(i);
|
|
22358
|
-
}
|
|
22359
|
-
if (embeddableSlice.length === 0) return;
|
|
22360
|
-
try {
|
|
22361
|
-
const embedResult = await embedBatchDocumentFields(
|
|
22362
|
-
embeddableSlice,
|
|
22363
|
-
entry.config.embedding,
|
|
22364
|
-
entry.embeddingAdapter,
|
|
22365
|
-
signal
|
|
22366
|
-
);
|
|
22367
|
-
for (const [sliceIndex, error] of embedResult.failed) {
|
|
22368
|
-
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
22369
|
-
chunkFailedIndexes.add(originalIdx);
|
|
22370
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error });
|
|
22371
|
-
}
|
|
22372
|
-
} catch (err) {
|
|
22373
|
-
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
22374
|
-
for (const originalIdx of embeddableOriginalIndexes) {
|
|
22375
|
-
chunkFailedIndexes.add(originalIdx);
|
|
22376
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error: embeddingError });
|
|
22377
|
-
}
|
|
22378
|
-
}
|
|
22379
|
-
}
|
|
22380
22310
|
|
|
22381
22311
|
// src/engine/mutations/write-visibility.ts
|
|
22382
22312
|
async function awaitWriteVisibility(ctx, indexName) {
|
|
@@ -22385,33 +22315,19 @@ async function awaitWriteVisibility(ctx, indexName) {
|
|
|
22385
22315
|
}
|
|
22386
22316
|
|
|
22387
22317
|
// src/engine/mutations/insert.ts
|
|
22388
|
-
async function insertDocument(ctx, indexName, document, docId, options) {
|
|
22318
|
+
async function insertDocument(ctx, indexName, document, docId, options, hold) {
|
|
22389
22319
|
ctx.guardShutdown();
|
|
22390
|
-
|
|
22320
|
+
let entry = ctx.requireIndex(indexName);
|
|
22391
22321
|
if (docId !== void 0) requireMatchingDocId(docId, document);
|
|
22392
22322
|
const resolvedDocId = docId ?? providedDocId(document) ?? ctx.idGenerator();
|
|
22393
22323
|
validateDocId(resolvedDocId);
|
|
22394
|
-
|
|
22395
|
-
if (
|
|
22324
|
+
const beforeInsert = ctx.pluginRegistry.hasHooks("beforeInsert") ? () => ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document }) : null;
|
|
22325
|
+
if (beforeInsert !== null || entry.config.embedding !== void 0) {
|
|
22326
|
+
const { prepareDocumentWrite: prepareDocumentWrite2 } = await loadOptionalWrites();
|
|
22327
|
+
entry = await prepareDocumentWrite2(ctx, indexName, document, hold, beforeInsert);
|
|
22328
|
+
} else if (entry.config.required !== void 0) {
|
|
22396
22329
|
validateRequiredFields(document, entry.config.required);
|
|
22397
22330
|
}
|
|
22398
|
-
if (entry.config.embedding) {
|
|
22399
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
22400
|
-
if (entry.embeddingAdapter) {
|
|
22401
|
-
await embedDocumentFields(
|
|
22402
|
-
document,
|
|
22403
|
-
entry.config.embedding,
|
|
22404
|
-
entry.embeddingAdapter,
|
|
22405
|
-
ctx.abortController.signal
|
|
22406
|
-
);
|
|
22407
|
-
} else {
|
|
22408
|
-
assertDocumentCarriesMappedVectors(
|
|
22409
|
-
document,
|
|
22410
|
-
entry.config.embedding,
|
|
22411
|
-
entry.embeddingAdapterName
|
|
22412
|
-
);
|
|
22413
|
-
}
|
|
22414
|
-
}
|
|
22415
22331
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
22416
22332
|
if (sizing !== null) await sizing;
|
|
22417
22333
|
const insertManager = ctx.requireManager(indexName);
|
|
@@ -22491,16 +22407,18 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
22491
22407
|
return resolvedDocId;
|
|
22492
22408
|
}
|
|
22493
22409
|
await ctx.orchestrator.settlePublications(indexName);
|
|
22494
|
-
|
|
22495
|
-
|
|
22496
|
-
|
|
22497
|
-
|
|
22410
|
+
if (ctx.pluginRegistry.hasHooks("afterInsert")) {
|
|
22411
|
+
const runHook = async () => {
|
|
22412
|
+
try {
|
|
22413
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: resolvedDocId, document });
|
|
22414
|
+
} catch (err) {
|
|
22415
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
22416
|
+
}
|
|
22417
|
+
};
|
|
22418
|
+
await (hold === void 0 ? runHook() : hold.outside(runHook));
|
|
22498
22419
|
}
|
|
22499
|
-
|
|
22500
|
-
const
|
|
22501
|
-
if (vecIndex) {
|
|
22502
|
-
vecIndex.scheduleBuild();
|
|
22503
|
-
}
|
|
22420
|
+
if (ctx.executor.getManager(indexName) === insertManager) {
|
|
22421
|
+
for (const fieldPath of extractedVectors.keys()) insertVecIndexes.get(fieldPath)?.scheduleBuild();
|
|
22504
22422
|
}
|
|
22505
22423
|
ctx.checkWatermark(indexName, resolvedDocId);
|
|
22506
22424
|
ctx.checkHeapPressure(indexName);
|
|
@@ -22586,6 +22504,326 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
22586
22504
|
function bindOptionalWritesHost(_host, _segmentCodec) {
|
|
22587
22505
|
}
|
|
22588
22506
|
|
|
22507
|
+
// src/engine/mutations/batch-admissions.ts
|
|
22508
|
+
function createPendingAdmissions() {
|
|
22509
|
+
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
22510
|
+
}
|
|
22511
|
+
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
22512
|
+
pending.total += 1;
|
|
22513
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
22514
|
+
const partitionId = manager.routePartition(docId);
|
|
22515
|
+
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
22516
|
+
}
|
|
22517
|
+
function asBatchInsertError(err) {
|
|
22518
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
22519
|
+
}
|
|
22520
|
+
|
|
22521
|
+
// src/engine/mutations/outside-hold.ts
|
|
22522
|
+
function runOutside(hold, work) {
|
|
22523
|
+
return hold === void 0 ? work() : hold.outside(work);
|
|
22524
|
+
}
|
|
22525
|
+
|
|
22526
|
+
// src/engine/mutations/write-preparation.ts
|
|
22527
|
+
function settingsOf(entry) {
|
|
22528
|
+
return { required: entry.config.required, embedding: entry.config.embedding, adapter: entry.embeddingAdapter };
|
|
22529
|
+
}
|
|
22530
|
+
function embedsThroughAdapter(entry) {
|
|
22531
|
+
return entry.config.embedding !== void 0 && entry.embeddingAdapter !== null;
|
|
22532
|
+
}
|
|
22533
|
+
function sourcesOf(sources2) {
|
|
22534
|
+
return typeof sources2 === "string" ? [sources2] : sources2;
|
|
22535
|
+
}
|
|
22536
|
+
function sameMapping(left, right) {
|
|
22537
|
+
if (left === right) return true;
|
|
22538
|
+
const targets = Object.keys(left);
|
|
22539
|
+
if (targets.length !== Object.keys(right).length) return false;
|
|
22540
|
+
for (const target of targets) {
|
|
22541
|
+
if (!Object.hasOwn(right, target)) return false;
|
|
22542
|
+
const leftSources = sourcesOf(left[target]);
|
|
22543
|
+
const rightSources = sourcesOf(right[target]);
|
|
22544
|
+
if (leftSources.length !== rightSources.length) return false;
|
|
22545
|
+
if (leftSources.some((source2, position) => source2 !== rightSources[position])) return false;
|
|
22546
|
+
}
|
|
22547
|
+
return true;
|
|
22548
|
+
}
|
|
22549
|
+
function embeddingFits(entry, embedding) {
|
|
22550
|
+
const embeddingConfig = entry.config.embedding;
|
|
22551
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter === null) return embedding === null;
|
|
22552
|
+
return embedding !== null && embedding.adapter === entry.embeddingAdapter && sameMapping(embedding.mapping, embeddingConfig.fields);
|
|
22553
|
+
}
|
|
22554
|
+
function embeddingOfDocuments(embedding, documents2) {
|
|
22555
|
+
if (embedding === null) return null;
|
|
22556
|
+
const fields = /* @__PURE__ */ new Map();
|
|
22557
|
+
for (const document of documents2) {
|
|
22558
|
+
const embedded2 = embedding.fields.get(document);
|
|
22559
|
+
if (embedded2 !== void 0) fields.set(document, embedded2);
|
|
22560
|
+
}
|
|
22561
|
+
return { adapter: embedding.adapter, mapping: embedding.mapping, fields };
|
|
22562
|
+
}
|
|
22563
|
+
function writable(preparations) {
|
|
22564
|
+
return preparations.filter((preparation) => preparation.failure === void 0);
|
|
22565
|
+
}
|
|
22566
|
+
function failAll(preparations, failure) {
|
|
22567
|
+
for (const preparation of writable(preparations)) preparation.failure = failure;
|
|
22568
|
+
}
|
|
22569
|
+
function requireFieldsOf(required, preparations) {
|
|
22570
|
+
if (required === void 0 || required.length === 0) return;
|
|
22571
|
+
for (const preparation of writable(preparations)) {
|
|
22572
|
+
try {
|
|
22573
|
+
validateRequiredFields(preparation.document, required);
|
|
22574
|
+
} catch (err) {
|
|
22575
|
+
preparation.failure = err;
|
|
22576
|
+
}
|
|
22577
|
+
}
|
|
22578
|
+
}
|
|
22579
|
+
async function requireMappedVectors(entry, preparations) {
|
|
22580
|
+
const embeddingConfig = entry.config.embedding;
|
|
22581
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter !== null) return;
|
|
22582
|
+
const { assertDocumentCarriesMappedVectors } = await loadEmbedCode();
|
|
22583
|
+
for (const preparation of writable(preparations)) {
|
|
22584
|
+
try {
|
|
22585
|
+
assertDocumentCarriesMappedVectors(
|
|
22586
|
+
preparation.document,
|
|
22587
|
+
embeddingConfig,
|
|
22588
|
+
entry.embeddingAdapterName
|
|
22589
|
+
);
|
|
22590
|
+
} catch (err) {
|
|
22591
|
+
preparation.failure = err;
|
|
22592
|
+
}
|
|
22593
|
+
}
|
|
22594
|
+
}
|
|
22595
|
+
async function embedWith(settings, signal, preparations, earlier, together) {
|
|
22596
|
+
requireFieldsOf(settings.required, preparations);
|
|
22597
|
+
const { clearWriteEmbedding, embedDocumentsForWrite, lacksMappedVectors } = await loadEmbedCode();
|
|
22598
|
+
if (earlier !== null) clearWriteEmbedding(earlier);
|
|
22599
|
+
const { embedding: embeddingConfig, adapter } = settings;
|
|
22600
|
+
const embeddable = writable(preparations);
|
|
22601
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
22602
|
+
if (embeddingConfig === void 0 || adapter === null || embeddable.length === 0) {
|
|
22603
|
+
return { embedding: null, called: false, documentFailures };
|
|
22604
|
+
}
|
|
22605
|
+
const fields = /* @__PURE__ */ new Map();
|
|
22606
|
+
const embedding = { adapter, mapping: embeddingConfig.fields, fields };
|
|
22607
|
+
const documents2 = embeddable.map((preparation) => preparation.document);
|
|
22608
|
+
if (!lacksMappedVectors(documents2, embeddingConfig)) return { embedding, called: false, documentFailures };
|
|
22609
|
+
for (let chunkStart = 0; chunkStart < embeddable.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22610
|
+
const chunk = embeddable.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
22611
|
+
const embedded2 = await embedDocumentsForWrite(
|
|
22612
|
+
chunk.map((preparation) => preparation.document),
|
|
22613
|
+
embeddingConfig,
|
|
22614
|
+
adapter,
|
|
22615
|
+
signal,
|
|
22616
|
+
together
|
|
22617
|
+
);
|
|
22618
|
+
for (const [document, embeddedFields] of embedded2.embedding.fields) fields.set(document, embeddedFields);
|
|
22619
|
+
for (const [position, error] of embedded2.failed) chunk[position].failure = error;
|
|
22620
|
+
for (const [position, error] of embedded2.documentFailures) documentFailures.set(chunk[position], error);
|
|
22621
|
+
}
|
|
22622
|
+
return { embedding, called: true, documentFailures };
|
|
22623
|
+
}
|
|
22624
|
+
function settleDocumentFailures(entry, pass) {
|
|
22625
|
+
if (!embeddingFits(entry, pass.embedding)) return [...pass.documentFailures.keys()];
|
|
22626
|
+
for (const [preparation, failure] of pass.documentFailures) preparation.failure = failure;
|
|
22627
|
+
return [];
|
|
22628
|
+
}
|
|
22629
|
+
async function runBeforeWrites(preparations) {
|
|
22630
|
+
for (const preparation of preparations) {
|
|
22631
|
+
if (preparation.beforeWrite === null || preparation.failure !== void 0) continue;
|
|
22632
|
+
try {
|
|
22633
|
+
await preparation.beforeWrite();
|
|
22634
|
+
} catch (err) {
|
|
22635
|
+
preparation.failure = err;
|
|
22636
|
+
}
|
|
22637
|
+
}
|
|
22638
|
+
}
|
|
22639
|
+
async function embedThenRunHooks(settings, signal, preparations, earlier) {
|
|
22640
|
+
const pass = await embedWith(settings, signal, preparations, earlier, true);
|
|
22641
|
+
await runBeforeWrites(preparations.filter((preparation) => !pass.documentFailures.has(preparation)));
|
|
22642
|
+
return pass;
|
|
22643
|
+
}
|
|
22644
|
+
async function runHooksThenEmbedEach(ctx, indexName, preparations, earlier) {
|
|
22645
|
+
let settings = null;
|
|
22646
|
+
let embedding = null;
|
|
22647
|
+
let called = false;
|
|
22648
|
+
let uncleared = earlier;
|
|
22649
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
22650
|
+
for (const preparation of preparations) {
|
|
22651
|
+
await runBeforeWrites([preparation]);
|
|
22652
|
+
settings ??= settingsOf(ctx.requireIndex(indexName));
|
|
22653
|
+
const pass = await embedWith(settings, ctx.abortController.signal, [preparation], uncleared, false);
|
|
22654
|
+
uncleared = null;
|
|
22655
|
+
called ||= pass.called;
|
|
22656
|
+
for (const [failed, failure] of pass.documentFailures) documentFailures.set(failed, failure);
|
|
22657
|
+
if (pass.embedding === null) continue;
|
|
22658
|
+
if (embedding === null) embedding = pass.embedding;
|
|
22659
|
+
else for (const [document, fields] of pass.embedding.fields) embedding.fields.set(document, fields);
|
|
22660
|
+
}
|
|
22661
|
+
return { embedding, called, documentFailures };
|
|
22662
|
+
}
|
|
22663
|
+
function adapterChangesRefusal(indexName, embeddings) {
|
|
22664
|
+
return new NarsilError(
|
|
22665
|
+
ErrorCodes.WORKER_BUSY,
|
|
22666
|
+
`Changes to index "${indexName}" replaced its embedding adapter after each of ${embeddings} embeddings of this write`,
|
|
22667
|
+
{ indexName, embeddings }
|
|
22668
|
+
);
|
|
22669
|
+
}
|
|
22670
|
+
async function prepareDocumentWrites(ctx, indexName, preparations, hold, together, earlier = null) {
|
|
22671
|
+
let entry = ctx.requireIndex(indexName);
|
|
22672
|
+
let embedding = earlier;
|
|
22673
|
+
if (writable(preparations).length === 0) return { entry, embedding };
|
|
22674
|
+
const signal = ctx.abortController.signal;
|
|
22675
|
+
let embeddings = 0;
|
|
22676
|
+
let unhooked = [];
|
|
22677
|
+
const hooked = preparations.some((preparation) => preparation.beforeWrite !== null);
|
|
22678
|
+
if (hooked || embedsThroughAdapter(entry) && !embeddingFits(entry, embedding)) {
|
|
22679
|
+
if (together && !embedsThroughAdapter(entry)) {
|
|
22680
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
22681
|
+
await requireMappedVectors(entry, preparations);
|
|
22682
|
+
}
|
|
22683
|
+
const settings = settingsOf(entry);
|
|
22684
|
+
const uncleared = embedding;
|
|
22685
|
+
const pass = await runOutside(
|
|
22686
|
+
hold,
|
|
22687
|
+
() => together ? embedThenRunHooks(settings, signal, preparations, uncleared) : runHooksThenEmbedEach(ctx, indexName, preparations, uncleared)
|
|
22688
|
+
);
|
|
22689
|
+
embedding = pass.embedding;
|
|
22690
|
+
if (pass.called) embeddings += 1;
|
|
22691
|
+
entry = ctx.requireIndex(indexName);
|
|
22692
|
+
unhooked = settleDocumentFailures(entry, pass);
|
|
22693
|
+
}
|
|
22694
|
+
while (writable(preparations).length > 0 && !embeddingFits(entry, embedding)) {
|
|
22695
|
+
if (embeddings >= MAX_CALLS_ACROSS_INDEX_CHANGES) {
|
|
22696
|
+
failAll(preparations, adapterChangesRefusal(indexName, embeddings));
|
|
22697
|
+
break;
|
|
22698
|
+
}
|
|
22699
|
+
const uncleared = embedding;
|
|
22700
|
+
const settings = settingsOf(entry);
|
|
22701
|
+
const hooking = unhooked;
|
|
22702
|
+
const pass = await runOutside(hold, async () => {
|
|
22703
|
+
const embedded2 = await embedWith(settings, signal, preparations, uncleared, together);
|
|
22704
|
+
await runBeforeWrites(hooking.filter((preparation) => !embedded2.documentFailures.has(preparation)));
|
|
22705
|
+
return embedded2;
|
|
22706
|
+
});
|
|
22707
|
+
embedding = pass.embedding;
|
|
22708
|
+
if (pass.called) embeddings += 1;
|
|
22709
|
+
entry = ctx.requireIndex(indexName);
|
|
22710
|
+
const stale = settleDocumentFailures(entry, pass);
|
|
22711
|
+
unhooked = hooking.filter((preparation) => stale.includes(preparation));
|
|
22712
|
+
}
|
|
22713
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
22714
|
+
await requireMappedVectors(entry, preparations);
|
|
22715
|
+
return { entry, embedding };
|
|
22716
|
+
}
|
|
22717
|
+
async function prepareDocumentWrite(ctx, indexName, document, hold, beforeWrite) {
|
|
22718
|
+
const preparation = { document, beforeWrite, failure: void 0 };
|
|
22719
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
22720
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
22721
|
+
return entry;
|
|
22722
|
+
}
|
|
22723
|
+
|
|
22724
|
+
// src/engine/mutations/insert-batch-admission.ts
|
|
22725
|
+
function pendingInsertOf(ctx, indexName, docId, document, hooked) {
|
|
22726
|
+
const pending = {
|
|
22727
|
+
docId,
|
|
22728
|
+
preparation: { document, beforeWrite: null, failure: void 0 },
|
|
22729
|
+
refusedByHook: false
|
|
22730
|
+
};
|
|
22731
|
+
if (hooked && ctx.pluginRegistry.hasHooks("beforeInsert")) {
|
|
22732
|
+
pending.preparation.beforeWrite = async () => {
|
|
22733
|
+
try {
|
|
22734
|
+
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document });
|
|
22735
|
+
} catch (err) {
|
|
22736
|
+
pending.refusedByHook = true;
|
|
22737
|
+
throw err;
|
|
22738
|
+
}
|
|
22739
|
+
};
|
|
22740
|
+
}
|
|
22741
|
+
return pending;
|
|
22742
|
+
}
|
|
22743
|
+
async function prepareInsertDocuments(ctx, indexName, documents2, start, end, hold, hooked, failed, earlier = null) {
|
|
22744
|
+
const pending = [];
|
|
22745
|
+
for (let i = start; i < end; i++) {
|
|
22746
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
22747
|
+
if (ctx.abortController.signal.aborted) break;
|
|
22748
|
+
const document = documents2[i];
|
|
22749
|
+
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
22750
|
+
try {
|
|
22751
|
+
const sizing = requireDocumentObject(document);
|
|
22752
|
+
if (sizing !== null) await sizing;
|
|
22753
|
+
validateDocId(docId);
|
|
22754
|
+
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
22755
|
+
} catch (err) {
|
|
22756
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
22757
|
+
}
|
|
22758
|
+
}
|
|
22759
|
+
const { entry, embedding } = await prepareDocumentWrites(
|
|
22760
|
+
ctx,
|
|
22761
|
+
indexName,
|
|
22762
|
+
pending.map((item) => item.preparation),
|
|
22763
|
+
hold,
|
|
22764
|
+
true,
|
|
22765
|
+
earlier
|
|
22766
|
+
);
|
|
22767
|
+
const candidates2 = [];
|
|
22768
|
+
for (const item of pending) {
|
|
22769
|
+
const { document, failure } = item.preparation;
|
|
22770
|
+
if (failure === void 0) {
|
|
22771
|
+
candidates2.push({ docId: item.docId, document });
|
|
22772
|
+
continue;
|
|
22773
|
+
}
|
|
22774
|
+
const docId = item.refusedByHook ? item.docId : providedDocId(document) ?? "";
|
|
22775
|
+
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
22776
|
+
}
|
|
22777
|
+
return { entry, candidates: candidates2, embedding };
|
|
22778
|
+
}
|
|
22779
|
+
async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
22780
|
+
const { entry, candidates: candidates2, embedding } = await prepareInsertDocuments(
|
|
22781
|
+
ctx,
|
|
22782
|
+
indexName,
|
|
22783
|
+
documents2,
|
|
22784
|
+
0,
|
|
22785
|
+
documents2.length,
|
|
22786
|
+
hold,
|
|
22787
|
+
true,
|
|
22788
|
+
failed
|
|
22789
|
+
);
|
|
22790
|
+
const manager = ctx.requireManager(indexName);
|
|
22791
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
22792
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
22793
|
+
const strict = entry.config.strict === true;
|
|
22794
|
+
const admitted = [];
|
|
22795
|
+
const admittedIds = /* @__PURE__ */ new Set();
|
|
22796
|
+
const pending = createPendingAdmissions();
|
|
22797
|
+
for (let position = 0; position < candidates2.length; position++) {
|
|
22798
|
+
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
22799
|
+
if (ctx.abortController.signal.aborted) break;
|
|
22800
|
+
const { docId, document } = candidates2[position];
|
|
22801
|
+
try {
|
|
22802
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
22803
|
+
document,
|
|
22804
|
+
vectorFieldPaths
|
|
22805
|
+
);
|
|
22806
|
+
if (extractedVectors.size > 0) {
|
|
22807
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
22808
|
+
}
|
|
22809
|
+
admitInsert(ctx, indexName, manager, docId, pending);
|
|
22810
|
+
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
22811
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
22812
|
+
}
|
|
22813
|
+
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
22814
|
+
if (strict) {
|
|
22815
|
+
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
22816
|
+
}
|
|
22817
|
+
admittedIds.add(docId);
|
|
22818
|
+
recordPendingAdmission(ctx, pending, manager, docId);
|
|
22819
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
22820
|
+
} catch (err) {
|
|
22821
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
22822
|
+
}
|
|
22823
|
+
}
|
|
22824
|
+
return { admitted, embedding };
|
|
22825
|
+
}
|
|
22826
|
+
|
|
22589
22827
|
// src/engine/mutations/durable-rollback.ts
|
|
22590
22828
|
function holdVectorsForRollback(indexes, docId) {
|
|
22591
22829
|
const releases = [];
|
|
@@ -22821,84 +23059,29 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
22821
23059
|
return applications;
|
|
22822
23060
|
}
|
|
22823
23061
|
|
|
22824
|
-
// src/engine/mutations/insert-batch-
|
|
22825
|
-
async function
|
|
22826
|
-
|
|
22827
|
-
const manager = ctx.
|
|
22828
|
-
|
|
22829
|
-
|
|
22830
|
-
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
22831
|
-
const strict = entry.config.strict === true;
|
|
22832
|
-
const required = entry.config.required;
|
|
22833
|
-
const admitted = [];
|
|
22834
|
-
const admittedIds = /* @__PURE__ */ new Set();
|
|
22835
|
-
const pending = createPendingAdmissions();
|
|
22836
|
-
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22837
|
-
if (ctx.abortController.signal.aborted) break;
|
|
22838
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
22839
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
22840
|
-
if (required && required.length > 0) {
|
|
22841
|
-
collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
22842
|
-
}
|
|
22843
|
-
await embedChunkDocuments(
|
|
22844
|
-
entry,
|
|
22845
|
-
documents2,
|
|
22846
|
-
chunkStart,
|
|
22847
|
-
chunkEnd,
|
|
22848
|
-
ctx.abortController.signal,
|
|
22849
|
-
chunkFailedIndexes,
|
|
22850
|
-
failed
|
|
22851
|
-
);
|
|
22852
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22853
|
-
if (ctx.abortController.signal.aborted) break;
|
|
22854
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
22855
|
-
const docId = providedDocId(documents2[i]) ?? ctx.idGenerator();
|
|
23062
|
+
// src/engine/mutations/insert-batch-documents.ts
|
|
23063
|
+
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
23064
|
+
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
23065
|
+
const manager = ctx.executor.getManager(indexName);
|
|
23066
|
+
await runOutside(hold, async () => {
|
|
23067
|
+
for (const doc of inserted) {
|
|
22856
23068
|
try {
|
|
22857
|
-
|
|
22858
|
-
if (sizing !== null) await sizing;
|
|
22859
|
-
validateDocId(docId);
|
|
22860
|
-
if (hasBeforeHook) {
|
|
22861
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents2[i] });
|
|
22862
|
-
}
|
|
22863
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
22864
|
-
documents2[i],
|
|
22865
|
-
vectorFieldPaths
|
|
22866
|
-
);
|
|
22867
|
-
if (extractedVectors.size > 0) {
|
|
22868
|
-
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
22869
|
-
}
|
|
22870
|
-
admitInsert(ctx, indexName, manager, docId, pending);
|
|
22871
|
-
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
22872
|
-
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
22873
|
-
}
|
|
22874
|
-
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
22875
|
-
if (strict) {
|
|
22876
|
-
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
22877
|
-
}
|
|
22878
|
-
admittedIds.add(docId);
|
|
22879
|
-
recordPendingAdmission(ctx, pending, manager, docId);
|
|
22880
|
-
admitted.push({ docId, document: documents2[i], partitionDoc, extractedVectors });
|
|
23069
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
22881
23070
|
} catch (err) {
|
|
22882
|
-
|
|
23071
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
22883
23072
|
}
|
|
22884
23073
|
}
|
|
22885
|
-
|
|
22886
|
-
|
|
22887
|
-
}
|
|
22888
|
-
}
|
|
22889
|
-
return admitted;
|
|
23074
|
+
});
|
|
23075
|
+
return ctx.executor.getManager(indexName) !== manager;
|
|
22890
23076
|
}
|
|
22891
|
-
|
|
22892
|
-
// src/engine/mutations/insert-batch-documents.ts
|
|
22893
|
-
async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed) {
|
|
22894
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
23077
|
+
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
22895
23078
|
const succeeded = [];
|
|
22896
|
-
const buffered = /* @__PURE__ */ new Set();
|
|
22897
23079
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
22898
23080
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22899
23081
|
if (ctx.abortController.signal.aborted) break;
|
|
22900
23082
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
22901
23083
|
const applications = await applyInsertChunk(ctx, indexName, chunk, options);
|
|
23084
|
+
const inserted = [];
|
|
22902
23085
|
for (let i = 0; i < chunk.length; i++) {
|
|
22903
23086
|
const doc = chunk[i];
|
|
22904
23087
|
const application = applications[i];
|
|
@@ -22907,30 +23090,103 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
|
|
|
22907
23090
|
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
22908
23091
|
continue;
|
|
22909
23092
|
}
|
|
22910
|
-
|
|
22911
|
-
|
|
22912
|
-
succeeded.push(doc.docId);
|
|
22913
|
-
continue;
|
|
22914
|
-
}
|
|
23093
|
+
succeeded.push(doc.docId);
|
|
23094
|
+
if (application.status === "buffered") continue;
|
|
22915
23095
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
22916
23096
|
touchedVectorFields.add(fieldPath);
|
|
22917
23097
|
}
|
|
22918
|
-
|
|
22919
|
-
try {
|
|
22920
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
22921
|
-
} catch (err) {
|
|
22922
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
22923
|
-
}
|
|
22924
|
-
}
|
|
22925
|
-
succeeded.push(doc.docId);
|
|
23098
|
+
inserted.push(doc);
|
|
22926
23099
|
}
|
|
22927
23100
|
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
23101
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
23102
|
+
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
23103
|
+
}
|
|
22928
23104
|
if (applied < admitted.length) {
|
|
22929
23105
|
ctx.checkHeapPressure(indexName);
|
|
22930
23106
|
await new Promise((r) => setTimeout(r, 0));
|
|
22931
23107
|
}
|
|
22932
23108
|
}
|
|
22933
|
-
return { succeeded,
|
|
23109
|
+
return { succeeded, touchedVectorFields, unrecorded: [] };
|
|
23110
|
+
}
|
|
23111
|
+
|
|
23112
|
+
// src/engine/mutations/insert-batch-chunks.ts
|
|
23113
|
+
async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
23114
|
+
const { entry, candidates: candidates2 } = await prepareInsertDocuments(
|
|
23115
|
+
ctx,
|
|
23116
|
+
indexName,
|
|
23117
|
+
documents2,
|
|
23118
|
+
chunkStart,
|
|
23119
|
+
chunkEnd,
|
|
23120
|
+
hold,
|
|
23121
|
+
hooked,
|
|
23122
|
+
failed,
|
|
23123
|
+
earlier
|
|
23124
|
+
);
|
|
23125
|
+
const manager = ctx.requireManager(indexName);
|
|
23126
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
23127
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
23128
|
+
const admitted = [];
|
|
23129
|
+
for (const { docId, document } of candidates2) {
|
|
23130
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23131
|
+
if (ctx.abortController.signal.aborted) break;
|
|
23132
|
+
try {
|
|
23133
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
23134
|
+
document,
|
|
23135
|
+
vectorFieldPaths
|
|
23136
|
+
);
|
|
23137
|
+
if (extractedVectors.size > 0) {
|
|
23138
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
23139
|
+
}
|
|
23140
|
+
admitInsert(ctx, indexName, manager, docId);
|
|
23141
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
23142
|
+
} catch (err) {
|
|
23143
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
23144
|
+
}
|
|
23145
|
+
}
|
|
23146
|
+
return admitted;
|
|
23147
|
+
}
|
|
23148
|
+
async function insertInChunks(ctx, indexName, documents2, options, hold, hooked, failed, embedded2 = null) {
|
|
23149
|
+
const succeeded = [];
|
|
23150
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23151
|
+
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23152
|
+
if (ctx.abortController.signal.aborted) break;
|
|
23153
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
23154
|
+
const earlier = embeddingOfDocuments(embedded2, documents2.slice(chunkStart, chunkEnd));
|
|
23155
|
+
const prepared = await admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
23156
|
+
if (prepared.length > 0) {
|
|
23157
|
+
const applications = await applyInsertChunk(ctx, indexName, prepared, options);
|
|
23158
|
+
const inserted = [];
|
|
23159
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
23160
|
+
const doc = prepared[i];
|
|
23161
|
+
const application = applications[i];
|
|
23162
|
+
if (application.status === "skipped") continue;
|
|
23163
|
+
if (application.status === "failed") {
|
|
23164
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
23165
|
+
continue;
|
|
23166
|
+
}
|
|
23167
|
+
succeeded.push(doc.docId);
|
|
23168
|
+
if (application.status === "buffered") continue;
|
|
23169
|
+
for (const fieldPath of doc.extractedVectors.keys()) touchedVectorFields.add(fieldPath);
|
|
23170
|
+
inserted.push(doc);
|
|
23171
|
+
}
|
|
23172
|
+
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
23173
|
+
}
|
|
23174
|
+
if (chunkEnd < documents2.length) {
|
|
23175
|
+
ctx.checkHeapPressure(indexName);
|
|
23176
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
23177
|
+
}
|
|
23178
|
+
}
|
|
23179
|
+
return { succeeded, touchedVectorFields };
|
|
23180
|
+
}
|
|
23181
|
+
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
23182
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
23183
|
+
for (const fieldPath of inserts.touchedVectorFields) {
|
|
23184
|
+
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
23185
|
+
}
|
|
23186
|
+
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
23187
|
+
ctx.checkHeapPressure(indexName);
|
|
23188
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
23189
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22934
23190
|
}
|
|
22935
23191
|
|
|
22936
23192
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -23035,10 +23291,6 @@ function mainStoreDocument(doc, options) {
|
|
|
23035
23291
|
if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
|
|
23036
23292
|
return structuredClone(doc.partitionDoc);
|
|
23037
23293
|
}
|
|
23038
|
-
async function applyIndividually(ctx, indexName, admitted, options, failed) {
|
|
23039
|
-
const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
|
|
23040
|
-
return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
|
|
23041
|
-
}
|
|
23042
23294
|
function chunksOfWholeSegments(indexName, partitionCount, built2, frozen, memberIndexes, admitted) {
|
|
23043
23295
|
const chunks = [];
|
|
23044
23296
|
let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
@@ -23088,12 +23340,12 @@ function chunksOfWholeSegments(indexName, partitionCount, built2, frozen, member
|
|
|
23088
23340
|
if (chunk.docs.length > 0) chunks.push(chunk);
|
|
23089
23341
|
return chunks;
|
|
23090
23342
|
}
|
|
23091
|
-
async function recordMergedDocuments(ctx, indexName, partitionCount, built2, frozen, memberIndexes, admitted, options, failed) {
|
|
23092
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
23343
|
+
async function recordMergedDocuments(ctx, indexName, partitionCount, built2, frozen, memberIndexes, admitted, options, hold, failed) {
|
|
23093
23344
|
const succeededIds = /* @__PURE__ */ new Set();
|
|
23094
23345
|
const errorOf = /* @__PURE__ */ new Map();
|
|
23095
23346
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23096
23347
|
const chunks = chunksOfWholeSegments(indexName, partitionCount, built2, frozen, memberIndexes, admitted);
|
|
23348
|
+
const unrecorded = [];
|
|
23097
23349
|
for (let index = 0; index < chunks.length; index++) {
|
|
23098
23350
|
const { docs, attachmentOf } = chunks[index];
|
|
23099
23351
|
const applies = docs.map((doc) => {
|
|
@@ -23103,6 +23355,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23103
23355
|
});
|
|
23104
23356
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
23105
23357
|
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
23358
|
+
const inserted = [];
|
|
23106
23359
|
for (let i = 0; i < docs.length; i++) {
|
|
23107
23360
|
const doc = docs[i];
|
|
23108
23361
|
const failure = failures[i];
|
|
@@ -23134,15 +23387,13 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23134
23387
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
23135
23388
|
touchedVectorFields.add(fieldPath);
|
|
23136
23389
|
}
|
|
23137
|
-
|
|
23138
|
-
try {
|
|
23139
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
23140
|
-
} catch (err) {
|
|
23141
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23142
|
-
}
|
|
23143
|
-
}
|
|
23390
|
+
inserted.push(doc);
|
|
23144
23391
|
succeededIds.add(doc.docId);
|
|
23145
23392
|
}
|
|
23393
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
23394
|
+
for (const later of chunks.slice(index + 1)) unrecorded.push(...later.docs);
|
|
23395
|
+
break;
|
|
23396
|
+
}
|
|
23146
23397
|
if (index + 1 < chunks.length) {
|
|
23147
23398
|
ctx.checkHeapPressure(indexName);
|
|
23148
23399
|
await new Promise((r) => setTimeout(r, 0));
|
|
@@ -23157,7 +23408,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23157
23408
|
if (!errorOf.has(doc.docId)) continue;
|
|
23158
23409
|
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
23159
23410
|
}
|
|
23160
|
-
return { succeeded, touchedVectorFields };
|
|
23411
|
+
return { succeeded, touchedVectorFields, unrecorded };
|
|
23161
23412
|
}
|
|
23162
23413
|
function frozenSegmentOf(segment, members, options) {
|
|
23163
23414
|
if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
|
|
@@ -23168,14 +23419,15 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
23168
23419
|
segment.segmentId
|
|
23169
23420
|
);
|
|
23170
23421
|
}
|
|
23171
|
-
async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
23422
|
+
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
23172
23423
|
if (admitted.length === 0) {
|
|
23173
|
-
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set() };
|
|
23424
|
+
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] };
|
|
23174
23425
|
}
|
|
23426
|
+
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
23175
23427
|
const manager = ctx.requireManager(indexName);
|
|
23176
23428
|
const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
|
|
23177
23429
|
if (admitted.length < MIN_DOCUMENTS_FOR_SEGMENTS || workers <= 0) {
|
|
23178
|
-
return applyIndividually(
|
|
23430
|
+
return applyIndividually();
|
|
23179
23431
|
}
|
|
23180
23432
|
const docIds = admitted.map((doc) => doc.docId);
|
|
23181
23433
|
const partitionDocuments = admitted.map((doc) => doc.partitionDoc);
|
|
@@ -23188,7 +23440,7 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
23188
23440
|
built2 = null;
|
|
23189
23441
|
}
|
|
23190
23442
|
if (built2 === null) {
|
|
23191
|
-
return applyIndividually(
|
|
23443
|
+
return applyIndividually();
|
|
23192
23444
|
}
|
|
23193
23445
|
const positions2 = new Map(admitted.map((document, index) => [document.docId, index]));
|
|
23194
23446
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -23197,16 +23449,15 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
23197
23449
|
const members = [];
|
|
23198
23450
|
for (const docId of segment.docIds) {
|
|
23199
23451
|
const position = positions2.get(docId);
|
|
23200
|
-
if (position === void 0 || used.has(position))
|
|
23201
|
-
return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
23452
|
+
if (position === void 0 || used.has(position)) return applyIndividually();
|
|
23202
23453
|
used.add(position);
|
|
23203
23454
|
members.push(position);
|
|
23204
23455
|
}
|
|
23205
23456
|
memberIndexes.push(members);
|
|
23206
23457
|
}
|
|
23207
|
-
if (used.size !== admitted.length) return applyIndividually(
|
|
23458
|
+
if (used.size !== admitted.length) return applyIndividually();
|
|
23208
23459
|
if (ctx.isRebalancing(indexName)) {
|
|
23209
|
-
return applyIndividually(
|
|
23460
|
+
return applyIndividually();
|
|
23210
23461
|
}
|
|
23211
23462
|
const frozen = [];
|
|
23212
23463
|
for (let i = 0; i < built2.length; i++) {
|
|
@@ -23215,149 +23466,65 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
23215
23466
|
memberIndexes[i].map((m) => admitted[m]),
|
|
23216
23467
|
options
|
|
23217
23468
|
);
|
|
23218
|
-
if (segment === null) return applyIndividually(
|
|
23469
|
+
if (segment === null) return applyIndividually();
|
|
23219
23470
|
frozen.push(segment);
|
|
23220
23471
|
}
|
|
23221
23472
|
for (const doc of admitted) {
|
|
23222
|
-
if (manager.has(doc.docId)) return applyIndividually(
|
|
23473
|
+
if (manager.has(doc.docId)) return applyIndividually();
|
|
23223
23474
|
}
|
|
23224
|
-
return recordMergedDocuments(
|
|
23475
|
+
return recordMergedDocuments(
|
|
23476
|
+
ctx,
|
|
23477
|
+
indexName,
|
|
23478
|
+
partitionCount,
|
|
23479
|
+
built2,
|
|
23480
|
+
frozen,
|
|
23481
|
+
memberIndexes,
|
|
23482
|
+
admitted,
|
|
23483
|
+
options,
|
|
23484
|
+
hold,
|
|
23485
|
+
failed
|
|
23486
|
+
);
|
|
23225
23487
|
}
|
|
23226
|
-
async function insertBatchViaSegments(ctx, indexName, documents2, options) {
|
|
23488
|
+
async function insertBatchViaSegments(ctx, indexName, documents2, options, hold) {
|
|
23227
23489
|
const failed = [];
|
|
23228
|
-
const admitted = await admitBatchDocuments(ctx, indexName, documents2, failed);
|
|
23229
|
-
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, failed);
|
|
23230
|
-
|
|
23231
|
-
|
|
23232
|
-
|
|
23233
|
-
|
|
23234
|
-
|
|
23235
|
-
|
|
23236
|
-
await ctx
|
|
23237
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23490
|
+
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents2, failed, hold);
|
|
23491
|
+
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
23492
|
+
if (outcome.unrecorded.length > 0) {
|
|
23493
|
+
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
23494
|
+
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
23495
|
+
outcome.succeeded.push(...rest.succeeded);
|
|
23496
|
+
for (const fieldPath of rest.touchedVectorFields) outcome.touchedVectorFields.add(fieldPath);
|
|
23497
|
+
}
|
|
23498
|
+
await finishInsertBatch(ctx, indexName, outcome, options);
|
|
23238
23499
|
return { succeeded: outcome.succeeded, failed };
|
|
23239
23500
|
}
|
|
23240
23501
|
|
|
23241
23502
|
// src/engine/mutations/insert-batch.ts
|
|
23242
|
-
async function insertDocumentBatch(ctx, indexName, documents2, options) {
|
|
23503
|
+
async function insertDocumentBatch(ctx, indexName, documents2, options, hold) {
|
|
23243
23504
|
ctx.guardShutdown();
|
|
23244
|
-
|
|
23505
|
+
ctx.requireIndex(indexName);
|
|
23245
23506
|
validateBatchList(documents2, "documents");
|
|
23246
23507
|
await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents2.length);
|
|
23247
23508
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
23248
23509
|
if (documents2.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
23249
|
-
return insertBatchViaSegments(ctx, indexName, documents2, options);
|
|
23510
|
+
return insertBatchViaSegments(ctx, indexName, documents2, options, hold);
|
|
23250
23511
|
}
|
|
23251
|
-
const succeeded = [];
|
|
23252
23512
|
const failed = [];
|
|
23253
|
-
const
|
|
23254
|
-
|
|
23255
|
-
|
|
23256
|
-
const batchManager = ctx.requireManager(indexName);
|
|
23257
|
-
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
23258
|
-
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
23259
|
-
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23260
|
-
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23261
|
-
if (ctx.abortController.signal.aborted) break;
|
|
23262
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
23263
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
23264
|
-
if (required && required.length > 0) {
|
|
23265
|
-
collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
23266
|
-
}
|
|
23267
|
-
await embedChunkDocuments(
|
|
23268
|
-
entry,
|
|
23269
|
-
documents2,
|
|
23270
|
-
chunkStart,
|
|
23271
|
-
chunkEnd,
|
|
23272
|
-
ctx.abortController.signal,
|
|
23273
|
-
chunkFailedIndexes,
|
|
23274
|
-
failed
|
|
23275
|
-
);
|
|
23276
|
-
async function applyPrepared(prepared2) {
|
|
23277
|
-
const applications = await applyInsertChunk(ctx, indexName, prepared2, options);
|
|
23278
|
-
for (let i = 0; i < prepared2.length; i++) {
|
|
23279
|
-
const doc = prepared2[i];
|
|
23280
|
-
const application = applications[i];
|
|
23281
|
-
if (application.status === "skipped") continue;
|
|
23282
|
-
if (application.status === "failed") {
|
|
23283
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
23284
|
-
continue;
|
|
23285
|
-
}
|
|
23286
|
-
if (application.status === "buffered") {
|
|
23287
|
-
succeeded.push(doc.docId);
|
|
23288
|
-
continue;
|
|
23289
|
-
}
|
|
23290
|
-
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
23291
|
-
touchedVectorFields.add(fieldPath);
|
|
23292
|
-
}
|
|
23293
|
-
if (hasAfterHook) {
|
|
23294
|
-
try {
|
|
23295
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
23296
|
-
} catch (err) {
|
|
23297
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23298
|
-
}
|
|
23299
|
-
}
|
|
23300
|
-
succeeded.push(doc.docId);
|
|
23301
|
-
}
|
|
23302
|
-
}
|
|
23303
|
-
const prepared = [];
|
|
23304
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23305
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23306
|
-
if (ctx.abortController.signal.aborted) break;
|
|
23307
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
23308
|
-
const batchDocId = providedDocId(documents2[i]) ?? ctx.idGenerator();
|
|
23309
|
-
try {
|
|
23310
|
-
const sizing = requireDocumentObject(documents2[i]);
|
|
23311
|
-
if (sizing !== null) await sizing;
|
|
23312
|
-
validateDocId(batchDocId);
|
|
23313
|
-
if (hasBeforeHook) {
|
|
23314
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents2[i] });
|
|
23315
|
-
}
|
|
23316
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
23317
|
-
documents2[i],
|
|
23318
|
-
batchVectorFieldPaths
|
|
23319
|
-
);
|
|
23320
|
-
if (extractedVectors.size > 0) {
|
|
23321
|
-
validateVectorDimensions(extractedVectors, batchVecIndexes);
|
|
23322
|
-
}
|
|
23323
|
-
admitInsert(ctx, indexName, batchManager, batchDocId);
|
|
23324
|
-
prepared.push({
|
|
23325
|
-
docId: batchDocId,
|
|
23326
|
-
document: documents2[i],
|
|
23327
|
-
partitionDoc,
|
|
23328
|
-
extractedVectors
|
|
23329
|
-
});
|
|
23330
|
-
} catch (err) {
|
|
23331
|
-
failed.push({ docId: batchDocId, error: asBatchInsertError(err) });
|
|
23332
|
-
}
|
|
23333
|
-
}
|
|
23334
|
-
if (prepared.length > 0) await applyPrepared(prepared);
|
|
23335
|
-
if (chunkEnd < documents2.length) {
|
|
23336
|
-
ctx.checkHeapPressure(indexName);
|
|
23337
|
-
await new Promise((r) => setTimeout(r, 0));
|
|
23338
|
-
}
|
|
23339
|
-
}
|
|
23340
|
-
for (const fieldPath of touchedVectorFields) {
|
|
23341
|
-
const vecIndex = batchVecIndexes.get(fieldPath);
|
|
23342
|
-
if (vecIndex) {
|
|
23343
|
-
vecIndex.scheduleBuild();
|
|
23344
|
-
}
|
|
23345
|
-
}
|
|
23346
|
-
ctx.checkWatermark(indexName, succeeded);
|
|
23347
|
-
ctx.checkHeapPressure(indexName);
|
|
23348
|
-
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
23349
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23350
|
-
return { succeeded, failed };
|
|
23513
|
+
const inserts = await insertInChunks(ctx, indexName, documents2, options, hold, true, failed);
|
|
23514
|
+
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
23515
|
+
return { succeeded: inserts.succeeded, failed };
|
|
23351
23516
|
}
|
|
23352
23517
|
|
|
23353
23518
|
// src/engine/mutations/remove.ts
|
|
23354
23519
|
function asRemoveError(err) {
|
|
23355
23520
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
23356
23521
|
}
|
|
23357
|
-
|
|
23522
|
+
function validateRemoval(ctx, docId) {
|
|
23358
23523
|
ctx.guardShutdown();
|
|
23359
23524
|
validateDocId(docId);
|
|
23360
|
-
|
|
23525
|
+
}
|
|
23526
|
+
function prepareRemoval(ctx, indexName, docId) {
|
|
23527
|
+
ctx.guardShutdown();
|
|
23361
23528
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
23362
23529
|
if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
|
|
23363
23530
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
@@ -23407,20 +23574,31 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
23407
23574
|
};
|
|
23408
23575
|
return () => manager.publication.writeNow(indexChangeOutsideExternalWaits(removeFromIndex));
|
|
23409
23576
|
}
|
|
23410
|
-
async function
|
|
23577
|
+
async function settleRemovals(ctx, indexName, docIds, hold) {
|
|
23578
|
+
if (docIds.length === 0) return;
|
|
23411
23579
|
const settling = ctx.orchestrator.settlePublications(indexName, "none");
|
|
23412
23580
|
if (settling !== void 0) await settling;
|
|
23413
|
-
|
|
23414
|
-
|
|
23415
|
-
|
|
23416
|
-
|
|
23417
|
-
|
|
23418
|
-
|
|
23581
|
+
if (!ctx.pluginRegistry.hasHooks("afterRemove")) return;
|
|
23582
|
+
await runOutside(hold, async () => {
|
|
23583
|
+
for (const docId of docIds) {
|
|
23584
|
+
try {
|
|
23585
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
23586
|
+
} catch (err) {
|
|
23587
|
+
console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23588
|
+
}
|
|
23589
|
+
}
|
|
23590
|
+
});
|
|
23419
23591
|
}
|
|
23420
|
-
async function removeDocument3(ctx, indexName, docId, options) {
|
|
23592
|
+
async function removeDocument3(ctx, indexName, docId, options, hold) {
|
|
23421
23593
|
ctx.guardShutdown();
|
|
23422
23594
|
ctx.requireIndex(indexName);
|
|
23423
|
-
|
|
23595
|
+
validateRemoval(ctx, docId);
|
|
23596
|
+
if (ctx.pluginRegistry.hasHooks("beforeRemove")) {
|
|
23597
|
+
await runOutside(hold, async () => {
|
|
23598
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
|
|
23599
|
+
});
|
|
23600
|
+
}
|
|
23601
|
+
const prepared = prepareRemoval(ctx, indexName, docId);
|
|
23424
23602
|
const apply = applyOfRemoval(ctx, indexName, prepared);
|
|
23425
23603
|
if (ctx.durability) {
|
|
23426
23604
|
try {
|
|
@@ -23434,25 +23612,52 @@ async function removeDocument3(ctx, indexName, docId, options) {
|
|
|
23434
23612
|
} else {
|
|
23435
23613
|
await apply();
|
|
23436
23614
|
}
|
|
23437
|
-
if (!prepared.buffered) await
|
|
23615
|
+
if (!prepared.buffered) await settleRemovals(ctx, indexName, [docId], hold);
|
|
23438
23616
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23439
23617
|
}
|
|
23440
|
-
async function
|
|
23618
|
+
async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold) {
|
|
23619
|
+
const refused = /* @__PURE__ */ new Map();
|
|
23620
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23621
|
+
try {
|
|
23622
|
+
validateRemoval(ctx, docIds[i]);
|
|
23623
|
+
} catch (err) {
|
|
23624
|
+
refused.set(i, err);
|
|
23625
|
+
}
|
|
23626
|
+
}
|
|
23627
|
+
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
23628
|
+
await runOutside(hold, async () => {
|
|
23629
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23630
|
+
if (refused.has(i)) continue;
|
|
23631
|
+
try {
|
|
23632
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
23633
|
+
} catch (err) {
|
|
23634
|
+
refused.set(i, err);
|
|
23635
|
+
}
|
|
23636
|
+
}
|
|
23637
|
+
});
|
|
23638
|
+
return refused;
|
|
23639
|
+
}
|
|
23640
|
+
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
23441
23641
|
ctx.guardShutdown();
|
|
23442
23642
|
ctx.requireIndex(indexName);
|
|
23443
23643
|
validateBatchList(docIds, "document IDs");
|
|
23444
23644
|
const succeeded = [];
|
|
23445
23645
|
const failed = [];
|
|
23446
|
-
const
|
|
23447
|
-
await
|
|
23646
|
+
const first = ctx.requireManager(indexName);
|
|
23647
|
+
await first.publication.write(() => first.beginBatchRemove());
|
|
23648
|
+
let manager = first;
|
|
23448
23649
|
try {
|
|
23449
23650
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23450
23651
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
23652
|
+
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
23653
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
23654
|
+
const removed = [];
|
|
23451
23655
|
const prepared = [];
|
|
23452
23656
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23453
23657
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23454
23658
|
try {
|
|
23455
|
-
|
|
23659
|
+
if (refused.has(i)) throw refused.get(i);
|
|
23660
|
+
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
23456
23661
|
} catch (err) {
|
|
23457
23662
|
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
23458
23663
|
}
|
|
@@ -23482,23 +23687,33 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
23482
23687
|
continue;
|
|
23483
23688
|
}
|
|
23484
23689
|
removal.releaseVectorHolds?.();
|
|
23485
|
-
if (!removal.buffered)
|
|
23690
|
+
if (!removal.buffered) removed.push(removal.docId);
|
|
23486
23691
|
succeeded.push(removal.docId);
|
|
23487
23692
|
}
|
|
23488
23693
|
} finally {
|
|
23489
23694
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
23490
23695
|
}
|
|
23696
|
+
await settleRemovals(ctx, indexName, removed, hold);
|
|
23697
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
23491
23698
|
if (chunkEnd < docIds.length) {
|
|
23492
23699
|
await new Promise((r) => setTimeout(r, 0));
|
|
23493
23700
|
}
|
|
23494
23701
|
}
|
|
23495
23702
|
} finally {
|
|
23496
|
-
|
|
23703
|
+
const ending = manager;
|
|
23704
|
+
await ending?.publication.write(() => ending.endBatchRemove());
|
|
23497
23705
|
}
|
|
23498
23706
|
ctx.checkHeapPressure(indexName);
|
|
23499
23707
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23500
23708
|
return { succeeded, failed };
|
|
23501
23709
|
}
|
|
23710
|
+
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
23711
|
+
const current = ctx.executor.getManager(indexName);
|
|
23712
|
+
if (current === batchManager) return batchManager;
|
|
23713
|
+
await batchManager?.publication.write(() => batchManager.endBatchRemove());
|
|
23714
|
+
await current?.publication.write(() => current.beginBatchRemove());
|
|
23715
|
+
return current;
|
|
23716
|
+
}
|
|
23502
23717
|
|
|
23503
23718
|
// src/engine/mutations/update.ts
|
|
23504
23719
|
function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
@@ -23532,59 +23747,47 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
23532
23747
|
function asUpdateError(err) {
|
|
23533
23748
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
23534
23749
|
}
|
|
23535
|
-
async function
|
|
23750
|
+
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
23751
|
+
for (const update of updates) {
|
|
23752
|
+
try {
|
|
23753
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
23754
|
+
indexName,
|
|
23755
|
+
docId: update.docId,
|
|
23756
|
+
oldDocument: update.oldDocument ?? {},
|
|
23757
|
+
newDocument: update.document
|
|
23758
|
+
});
|
|
23759
|
+
} catch (err) {
|
|
23760
|
+
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23761
|
+
}
|
|
23762
|
+
}
|
|
23763
|
+
}
|
|
23764
|
+
async function settleUpdates(ctx, indexName, updates, hold) {
|
|
23765
|
+
if (updates.length === 0) return;
|
|
23536
23766
|
await ctx.orchestrator.settlePublications(indexName);
|
|
23537
|
-
|
|
23538
|
-
await
|
|
23539
|
-
indexName,
|
|
23540
|
-
docId: prepared.docId,
|
|
23541
|
-
oldDocument: prepared.oldDocument ?? {},
|
|
23542
|
-
newDocument: prepared.document
|
|
23543
|
-
});
|
|
23544
|
-
} catch (err) {
|
|
23545
|
-
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23767
|
+
if (ctx.pluginRegistry.hasHooks("afterUpdate")) {
|
|
23768
|
+
await runOutside(hold, () => runAfterUpdateHooks(ctx, indexName, updates));
|
|
23546
23769
|
}
|
|
23547
|
-
const vecIndexes = ctx.
|
|
23548
|
-
|
|
23549
|
-
|
|
23550
|
-
|
|
23770
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
23771
|
+
if (vecIndexes === void 0) return;
|
|
23772
|
+
for (const update of updates) {
|
|
23773
|
+
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
23774
|
+
if (vec !== null) vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
23775
|
+
}
|
|
23551
23776
|
}
|
|
23552
23777
|
}
|
|
23553
|
-
|
|
23778
|
+
function updatePreparationOf(ctx, indexName, docId, document) {
|
|
23554
23779
|
ctx.guardShutdown();
|
|
23555
23780
|
validateDocId(docId);
|
|
23556
23781
|
requireMatchingDocId(docId, document);
|
|
23557
|
-
|
|
23558
|
-
|
|
23559
|
-
|
|
23560
|
-
|
|
23561
|
-
|
|
23562
|
-
|
|
23563
|
-
newDocument: document
|
|
23564
|
-
});
|
|
23565
|
-
}
|
|
23566
|
-
if (entry.config.required && entry.config.required.length > 0) {
|
|
23567
|
-
validateRequiredFields(document, entry.config.required);
|
|
23568
|
-
}
|
|
23569
|
-
if (entry.config.embedding) {
|
|
23570
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
23571
|
-
if (entry.embeddingAdapter) {
|
|
23572
|
-
await embedDocumentFields(
|
|
23573
|
-
document,
|
|
23574
|
-
entry.config.embedding,
|
|
23575
|
-
entry.embeddingAdapter,
|
|
23576
|
-
ctx.abortController.signal
|
|
23577
|
-
);
|
|
23578
|
-
} else {
|
|
23579
|
-
assertDocumentCarriesMappedVectors(
|
|
23580
|
-
document,
|
|
23581
|
-
entry.config.embedding,
|
|
23582
|
-
entry.embeddingAdapterName
|
|
23583
|
-
);
|
|
23584
|
-
}
|
|
23585
|
-
}
|
|
23782
|
+
if (!ctx.pluginRegistry.hasHooks("beforeUpdate")) return { document, beforeWrite: null, failure: void 0 };
|
|
23783
|
+
const oldDocument = ctx.requireManager(indexName).get(docId) ?? {};
|
|
23784
|
+
const beforeWrite = () => ctx.pluginRegistry.runHook("beforeUpdate", { indexName, docId, oldDocument, newDocument: document });
|
|
23785
|
+
return { document, beforeWrite, failure: void 0 };
|
|
23786
|
+
}
|
|
23787
|
+
async function finishUpdate(ctx, indexName, entry, docId, document) {
|
|
23586
23788
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
23587
23789
|
if (sizing !== null) await sizing;
|
|
23790
|
+
const manager = ctx.requireManager(indexName);
|
|
23588
23791
|
const vecIndexes = manager.getVectorIndexes();
|
|
23589
23792
|
const extractedVectors = /* @__PURE__ */ new Map();
|
|
23590
23793
|
if (vecIndexes.size > 0) {
|
|
@@ -23678,10 +23881,13 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
23678
23881
|
};
|
|
23679
23882
|
return () => manager.publication.write(indexChangeOutsideExternalWaits(updateInIndex));
|
|
23680
23883
|
}
|
|
23681
|
-
async function updateDocument(ctx, indexName, docId, document, options) {
|
|
23884
|
+
async function updateDocument(ctx, indexName, docId, document, options, hold) {
|
|
23682
23885
|
ctx.guardShutdown();
|
|
23683
|
-
|
|
23684
|
-
const
|
|
23886
|
+
ctx.requireIndex(indexName);
|
|
23887
|
+
const preparation = updatePreparationOf(ctx, indexName, docId, document);
|
|
23888
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
23889
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
23890
|
+
const prepared = await finishUpdate(ctx, indexName, entry, docId, document);
|
|
23685
23891
|
const apply = applyOfUpdate(ctx, indexName, prepared);
|
|
23686
23892
|
if (ctx.durability) {
|
|
23687
23893
|
try {
|
|
@@ -23704,10 +23910,12 @@ async function updateDocument(ctx, indexName, docId, document, options) {
|
|
|
23704
23910
|
} else {
|
|
23705
23911
|
await apply();
|
|
23706
23912
|
}
|
|
23707
|
-
if (!prepared.buffered) await
|
|
23913
|
+
if (!prepared.buffered) await settleUpdates(ctx, indexName, [prepared], hold);
|
|
23708
23914
|
ctx.checkHeapPressure(indexName);
|
|
23709
23915
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23710
23916
|
}
|
|
23917
|
+
|
|
23918
|
+
// src/engine/mutations/update-batch.ts
|
|
23711
23919
|
function reportedDocIdOf(update) {
|
|
23712
23920
|
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
23713
23921
|
return typeof docId === "string" ? docId : "";
|
|
@@ -23724,29 +23932,47 @@ function requireUpdateEntry(update) {
|
|
|
23724
23932
|
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
23725
23933
|
);
|
|
23726
23934
|
}
|
|
23727
|
-
async function
|
|
23935
|
+
async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed) {
|
|
23936
|
+
const preparations = [];
|
|
23937
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23938
|
+
const update = updates[i];
|
|
23939
|
+
const docId = reportedDocIdOf(update);
|
|
23940
|
+
try {
|
|
23941
|
+
const { document } = requireUpdateEntry(update);
|
|
23942
|
+
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
23943
|
+
} catch (err) {
|
|
23944
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
23945
|
+
}
|
|
23946
|
+
}
|
|
23947
|
+
const { entry } = await prepareDocumentWrites(
|
|
23948
|
+
ctx,
|
|
23949
|
+
indexName,
|
|
23950
|
+
preparations.map((pending) => pending.preparation),
|
|
23951
|
+
hold,
|
|
23952
|
+
false
|
|
23953
|
+
);
|
|
23954
|
+
const prepared = [];
|
|
23955
|
+
for (const { docId, preparation } of preparations) {
|
|
23956
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23957
|
+
try {
|
|
23958
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
23959
|
+
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
23960
|
+
} catch (err) {
|
|
23961
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
23962
|
+
}
|
|
23963
|
+
}
|
|
23964
|
+
return prepared;
|
|
23965
|
+
}
|
|
23966
|
+
async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
23728
23967
|
ctx.guardShutdown();
|
|
23729
|
-
|
|
23968
|
+
ctx.requireIndex(indexName);
|
|
23730
23969
|
validateBatchList(updates, "updates");
|
|
23731
23970
|
const succeeded = [];
|
|
23732
23971
|
const failed = [];
|
|
23733
|
-
const updateBatchManager = ctx.requireManager(indexName);
|
|
23734
|
-
const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
|
|
23735
23972
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23736
23973
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23737
23974
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
23738
|
-
const prepared =
|
|
23739
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23740
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23741
|
-
const update = updates[i];
|
|
23742
|
-
const docId = reportedDocIdOf(update);
|
|
23743
|
-
try {
|
|
23744
|
-
const { document } = requireUpdateEntry(update);
|
|
23745
|
-
prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document));
|
|
23746
|
-
} catch (err) {
|
|
23747
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
23748
|
-
}
|
|
23749
|
-
}
|
|
23975
|
+
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
23750
23976
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
23751
23977
|
let failures;
|
|
23752
23978
|
try {
|
|
@@ -23773,6 +23999,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
23773
23999
|
} finally {
|
|
23774
24000
|
for (const update of prepared) update.releaseVectorHolds?.();
|
|
23775
24001
|
}
|
|
24002
|
+
const settled = [];
|
|
23776
24003
|
for (let i = 0; i < prepared.length; i++) {
|
|
23777
24004
|
const update = prepared[i];
|
|
23778
24005
|
const failure = failures[i];
|
|
@@ -23780,19 +24007,21 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
23780
24007
|
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
23781
24008
|
continue;
|
|
23782
24009
|
}
|
|
23783
|
-
if (!update.buffered)
|
|
24010
|
+
if (!update.buffered) settled.push(update);
|
|
23784
24011
|
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
23785
24012
|
if (vec !== null) touchedVectorFields.add(fieldPath);
|
|
23786
24013
|
}
|
|
23787
24014
|
succeeded.push(update.docId);
|
|
23788
24015
|
}
|
|
24016
|
+
await settleUpdates(ctx, indexName, settled, hold);
|
|
23789
24017
|
if (chunkEnd < updates.length) {
|
|
23790
24018
|
ctx.checkHeapPressure(indexName);
|
|
23791
24019
|
await new Promise((r) => setTimeout(r, 0));
|
|
23792
24020
|
}
|
|
23793
24021
|
}
|
|
24022
|
+
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
23794
24023
|
for (const fieldPath of touchedVectorFields) {
|
|
23795
|
-
|
|
24024
|
+
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
23796
24025
|
}
|
|
23797
24026
|
ctx.checkHeapPressure(indexName);
|
|
23798
24027
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
@@ -23852,22 +24081,22 @@ function updateItemOf(update) {
|
|
|
23852
24081
|
update
|
|
23853
24082
|
};
|
|
23854
24083
|
}
|
|
23855
|
-
async function insertDocument2(ctx, indexName, document, docId, options) {
|
|
24084
|
+
async function insertDocument2(ctx, indexName, document, docId, options, hold) {
|
|
23856
24085
|
const block = blockFor(ctx, indexName);
|
|
23857
|
-
if (block === void 0) return insertDocument(ctx, indexName, document, docId, options);
|
|
24086
|
+
if (block === void 0) return insertDocument(ctx, indexName, document, docId, options, hold);
|
|
23858
24087
|
return withReservation(
|
|
23859
24088
|
block.grouping,
|
|
23860
24089
|
indexName,
|
|
23861
24090
|
document,
|
|
23862
|
-
() => insertDocument(ctx, indexName, document, docId, options)
|
|
24091
|
+
() => insertDocument(ctx, indexName, document, docId, options, hold)
|
|
23863
24092
|
);
|
|
23864
24093
|
}
|
|
23865
|
-
async function insertDocumentBatch2(ctx, indexName, documents2, options) {
|
|
24094
|
+
async function insertDocumentBatch2(ctx, indexName, documents2, options, hold) {
|
|
23866
24095
|
const block = blockFor(ctx, indexName);
|
|
23867
|
-
if (block === void 0) return insertDocumentBatch(ctx, indexName, documents2, options);
|
|
24096
|
+
if (block === void 0) return insertDocumentBatch(ctx, indexName, documents2, options, hold);
|
|
23868
24097
|
const grouping = block.grouping;
|
|
23869
24098
|
if (grouping === void 0 || !Array.isArray(documents2) || documents2.length === 0) {
|
|
23870
|
-
return insertDocumentBatch(ctx, indexName, documents2, options);
|
|
24099
|
+
return insertDocumentBatch(ctx, indexName, documents2, options, hold);
|
|
23871
24100
|
}
|
|
23872
24101
|
const items = documents2.map((document) => ({ docId: providedDocId(document) ?? "", document }));
|
|
23873
24102
|
return writeInGroups(
|
|
@@ -23878,26 +24107,27 @@ async function insertDocumentBatch2(ctx, indexName, documents2, options) {
|
|
|
23878
24107
|
ctx,
|
|
23879
24108
|
indexName,
|
|
23880
24109
|
group.map((item) => item.document),
|
|
23881
|
-
options
|
|
24110
|
+
options,
|
|
24111
|
+
hold
|
|
23882
24112
|
)
|
|
23883
24113
|
);
|
|
23884
24114
|
}
|
|
23885
|
-
async function updateDocument2(ctx, indexName, docId, document, options) {
|
|
24115
|
+
async function updateDocument2(ctx, indexName, docId, document, options, hold) {
|
|
23886
24116
|
const block = blockFor(ctx, indexName);
|
|
23887
|
-
if (block === void 0) return updateDocument(ctx, indexName, docId, document, options);
|
|
24117
|
+
if (block === void 0) return updateDocument(ctx, indexName, docId, document, options, hold);
|
|
23888
24118
|
return withReservation(
|
|
23889
24119
|
block.grouping,
|
|
23890
24120
|
indexName,
|
|
23891
24121
|
document,
|
|
23892
|
-
() => updateDocument(ctx, indexName, docId, document, options)
|
|
24122
|
+
() => updateDocument(ctx, indexName, docId, document, options, hold)
|
|
23893
24123
|
);
|
|
23894
24124
|
}
|
|
23895
|
-
async function updateDocumentBatch2(ctx, indexName, updates, options) {
|
|
24125
|
+
async function updateDocumentBatch2(ctx, indexName, updates, options, hold) {
|
|
23896
24126
|
const block = blockFor(ctx, indexName);
|
|
23897
|
-
if (block === void 0) return updateDocumentBatch(ctx, indexName, updates, options);
|
|
24127
|
+
if (block === void 0) return updateDocumentBatch(ctx, indexName, updates, options, hold);
|
|
23898
24128
|
const grouping = block.grouping;
|
|
23899
24129
|
if (grouping === void 0 || !Array.isArray(updates) || updates.length === 0) {
|
|
23900
|
-
return updateDocumentBatch(ctx, indexName, updates, options);
|
|
24130
|
+
return updateDocumentBatch(ctx, indexName, updates, options, hold);
|
|
23901
24131
|
}
|
|
23902
24132
|
return writeInGroups(
|
|
23903
24133
|
grouping,
|
|
@@ -23907,16 +24137,17 @@ async function updateDocumentBatch2(ctx, indexName, updates, options) {
|
|
|
23907
24137
|
ctx,
|
|
23908
24138
|
indexName,
|
|
23909
24139
|
group.map((item) => item.update),
|
|
23910
|
-
options
|
|
24140
|
+
options,
|
|
24141
|
+
hold
|
|
23911
24142
|
)
|
|
23912
24143
|
);
|
|
23913
24144
|
}
|
|
23914
|
-
async function removeDocument4(ctx, indexName, docId, options) {
|
|
23915
|
-
await removeDocument3(ctx, indexName, docId, options);
|
|
24145
|
+
async function removeDocument4(ctx, indexName, docId, options, hold) {
|
|
24146
|
+
await removeDocument3(ctx, indexName, docId, options, hold);
|
|
23916
24147
|
ctx.memoryWriteBlock?.memoryReleased();
|
|
23917
24148
|
}
|
|
23918
|
-
async function removeDocumentBatch2(ctx, indexName, docIds, options) {
|
|
23919
|
-
const result = await removeDocumentBatch(ctx, indexName, docIds, options);
|
|
24149
|
+
async function removeDocumentBatch2(ctx, indexName, docIds, options, hold) {
|
|
24150
|
+
const result = await removeDocumentBatch(ctx, indexName, docIds, options, hold);
|
|
23920
24151
|
if (result.succeeded.length > 0) ctx.memoryWriteBlock?.memoryReleased();
|
|
23921
24152
|
return result;
|
|
23922
24153
|
}
|
|
@@ -25275,7 +25506,7 @@ var workerCopies = codeLoader(async () => {
|
|
|
25275
25506
|
const [segmentCodec2, segmentMerge2, code7] = await Promise.all([
|
|
25276
25507
|
loadSegmentCodec(),
|
|
25277
25508
|
loadSegmentMerge(),
|
|
25278
|
-
import('./worker-copies-
|
|
25509
|
+
import('./worker-copies-WYPRCNB7.mjs')
|
|
25279
25510
|
]);
|
|
25280
25511
|
code7.bindWorkerCopiesHost(optionalCodeHost(), segmentCodec2, segmentMerge2);
|
|
25281
25512
|
return code7;
|
|
@@ -25356,7 +25587,7 @@ async function createWorkerOrchestrator(...args) {
|
|
|
25356
25587
|
var loaded3 = null;
|
|
25357
25588
|
function loadRerankRuntime() {
|
|
25358
25589
|
if (loaded3 !== null) return Promise.resolve(loaded3);
|
|
25359
|
-
return import('./runtime-
|
|
25590
|
+
return import('./runtime-USQYKN3I.mjs').then((runtime) => {
|
|
25360
25591
|
runtime.bindRerankHost(optionalCodeHost());
|
|
25361
25592
|
loaded3 = runtime;
|
|
25362
25593
|
return runtime;
|
|
@@ -27899,6 +28130,22 @@ function clearEngineIndex(core2, indexName) {
|
|
|
27899
28130
|
});
|
|
27900
28131
|
}
|
|
27901
28132
|
|
|
28133
|
+
// src/engine/index-hold.ts
|
|
28134
|
+
async function holdIndex(indexState, indexName) {
|
|
28135
|
+
let release = await indexState.acquire(indexName);
|
|
28136
|
+
return {
|
|
28137
|
+
release: () => release(),
|
|
28138
|
+
async outside(work) {
|
|
28139
|
+
release();
|
|
28140
|
+
try {
|
|
28141
|
+
return await work();
|
|
28142
|
+
} finally {
|
|
28143
|
+
release = await indexState.acquire(indexName);
|
|
28144
|
+
}
|
|
28145
|
+
}
|
|
28146
|
+
};
|
|
28147
|
+
}
|
|
28148
|
+
|
|
27902
28149
|
// src/narsil/bm25.ts
|
|
27903
28150
|
function bm25Operations(core2) {
|
|
27904
28151
|
return {
|
|
@@ -29026,7 +29273,7 @@ async function executePreflight(request, callerContext) {
|
|
|
29026
29273
|
|
|
29027
29274
|
// src/engine/resolve-vector-text-code.ts
|
|
29028
29275
|
var resolveVectorText = codeLoader(
|
|
29029
|
-
() => import('./resolve-vector-text-
|
|
29276
|
+
() => import('./resolve-vector-text-SD6BEMKZ.mjs').then((code7) => {
|
|
29030
29277
|
code7.bindResolveVectorTextHost(optionalCodeHost());
|
|
29031
29278
|
return code7;
|
|
29032
29279
|
}),
|
|
@@ -29039,10 +29286,10 @@ async function enterRead(core2, indexName) {
|
|
|
29039
29286
|
core2.guardShutdown();
|
|
29040
29287
|
if (core2.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
29041
29288
|
}
|
|
29042
|
-
async function
|
|
29043
|
-
if (params.vector?.text === void 0) return
|
|
29044
|
-
const {
|
|
29045
|
-
return
|
|
29289
|
+
async function embeddedFor(core2, indexName, params, signal, earlier) {
|
|
29290
|
+
if (params.vector?.text === void 0) return null;
|
|
29291
|
+
const { embedQueryWithAdapterOf } = await loadResolveVectorText();
|
|
29292
|
+
return embedQueryWithAdapterOf(indexName, params, core2.requireIndex(indexName), signal, earlier);
|
|
29046
29293
|
}
|
|
29047
29294
|
function phraseReadinessOf(core2, indexName, config) {
|
|
29048
29295
|
return () => {
|
|
@@ -29067,24 +29314,17 @@ function broadcastStatsFor(core2, options) {
|
|
|
29067
29314
|
}
|
|
29068
29315
|
return (name, tokens) => invalidation.broadcastStats(name, tokens);
|
|
29069
29316
|
}
|
|
29070
|
-
|
|
29071
|
-
|
|
29072
|
-
|
|
29073
|
-
await
|
|
29074
|
-
|
|
29317
|
+
var EMBEDDING_ADAPTER_CHANGED = 0;
|
|
29318
|
+
var INDEX_REPLACED_DURING_RERANK = 1;
|
|
29319
|
+
async function queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options) {
|
|
29320
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
29321
|
+
let runner;
|
|
29075
29322
|
try {
|
|
29076
29323
|
const entry = core2.requireIndex(indexName);
|
|
29324
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return EMBEDDING_ADAPTER_CHANGED;
|
|
29077
29325
|
const manager = core2.requireManager(indexName);
|
|
29078
|
-
const scoped = scopedParams(params, options);
|
|
29079
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
29080
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
29081
|
-
}
|
|
29082
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
29083
|
-
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
29084
|
-
const resolvedParams = preparedParams.vector?.text === void 0 ? preparedParams : await resolveVectorText2(preparedParams, entry, signal);
|
|
29085
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
29086
|
-
if (loading4 !== null) await loading4;
|
|
29087
29326
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
29327
|
+
runner = rerank?.rerankRunnerFor(core2, options?.signal, hold, indexName, manager);
|
|
29088
29328
|
let analysisStale = false;
|
|
29089
29329
|
const result = await answerFromOneState(
|
|
29090
29330
|
manager,
|
|
@@ -29105,16 +29345,49 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
29105
29345
|
indexBm25: options?.indexBm25,
|
|
29106
29346
|
cursorBinding: lazyQueryBinding(params),
|
|
29107
29347
|
patternWorkCap: core2.patternWorkCap,
|
|
29108
|
-
rerank:
|
|
29348
|
+
rerank: runner,
|
|
29109
29349
|
requirePhraseReady: phraseReadinessOf(core2, indexName, entry.config)
|
|
29110
29350
|
});
|
|
29111
29351
|
},
|
|
29112
29352
|
options?.signal
|
|
29113
29353
|
);
|
|
29354
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
29114
29355
|
if (analysisStale) {
|
|
29115
29356
|
result.analysisStale = true;
|
|
29116
29357
|
}
|
|
29117
29358
|
options?.onRanked?.(manager, result);
|
|
29359
|
+
return result;
|
|
29360
|
+
} catch (error) {
|
|
29361
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
29362
|
+
throw error;
|
|
29363
|
+
} finally {
|
|
29364
|
+
hold.release();
|
|
29365
|
+
}
|
|
29366
|
+
}
|
|
29367
|
+
async function runEngineQuery(core2, indexName, params, options) {
|
|
29368
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
29369
|
+
options?.signal?.throwIfAborted();
|
|
29370
|
+
await enterRead(core2, indexName);
|
|
29371
|
+
core2.requireIndex(indexName);
|
|
29372
|
+
const scoped = scopedParams(params, options);
|
|
29373
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
29374
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
29375
|
+
}
|
|
29376
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
29377
|
+
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
29378
|
+
let embedded2 = null;
|
|
29379
|
+
let replacedReranks = 0;
|
|
29380
|
+
while (true) {
|
|
29381
|
+
embedded2 = await embeddedFor(core2, indexName, preparedParams, signal, embedded2);
|
|
29382
|
+
const resolvedParams = embedded2?.params ?? preparedParams;
|
|
29383
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
29384
|
+
if (loading4 !== null) await loading4;
|
|
29385
|
+
const result = await queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options);
|
|
29386
|
+
if (result === EMBEDDING_ADAPTER_CHANGED) continue;
|
|
29387
|
+
if (result === INDEX_REPLACED_DURING_RERANK) {
|
|
29388
|
+
rerank?.refuseAfterIndexChanges(indexName, ++replacedReranks);
|
|
29389
|
+
continue;
|
|
29390
|
+
}
|
|
29118
29391
|
if (core2.pluginRegistry.hasHooks("afterSearch")) {
|
|
29119
29392
|
const observed = { ...result, hits: [...result.hits] };
|
|
29120
29393
|
try {
|
|
@@ -29124,26 +29397,15 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
29124
29397
|
}
|
|
29125
29398
|
}
|
|
29126
29399
|
return result;
|
|
29127
|
-
} finally {
|
|
29128
|
-
release();
|
|
29129
29400
|
}
|
|
29130
29401
|
}
|
|
29131
|
-
async function
|
|
29132
|
-
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
29133
|
-
options?.signal?.throwIfAborted();
|
|
29134
|
-
await enterRead(core2, indexName);
|
|
29402
|
+
async function preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options) {
|
|
29135
29403
|
const release = await core2.indexState.acquire(indexName);
|
|
29136
29404
|
try {
|
|
29137
29405
|
const entry = core2.requireIndex(indexName);
|
|
29406
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return null;
|
|
29138
29407
|
const manager = core2.requireManager(indexName);
|
|
29139
|
-
const scoped = scopedParams(params, options);
|
|
29140
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
29141
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
29142
|
-
}
|
|
29143
|
-
const resolvedParams = scoped.vector?.text === void 0 ? scoped : await resolveVectorText2(scoped, entry, signal);
|
|
29144
29408
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
29145
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
29146
|
-
if (loading4 !== null) await loading4;
|
|
29147
29409
|
let analysisStale = false;
|
|
29148
29410
|
const result = await answerFromOneState(
|
|
29149
29411
|
manager,
|
|
@@ -29176,6 +29438,25 @@ async function runEnginePreflight(core2, indexName, params, options) {
|
|
|
29176
29438
|
release();
|
|
29177
29439
|
}
|
|
29178
29440
|
}
|
|
29441
|
+
async function runEnginePreflight(core2, indexName, params, options) {
|
|
29442
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
29443
|
+
options?.signal?.throwIfAborted();
|
|
29444
|
+
await enterRead(core2, indexName);
|
|
29445
|
+
core2.requireIndex(indexName);
|
|
29446
|
+
const scoped = scopedParams(params, options);
|
|
29447
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
29448
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
29449
|
+
}
|
|
29450
|
+
let embedded2 = null;
|
|
29451
|
+
while (true) {
|
|
29452
|
+
embedded2 = await embeddedFor(core2, indexName, scoped, signal, embedded2);
|
|
29453
|
+
const resolvedParams = embedded2?.params ?? scoped;
|
|
29454
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
29455
|
+
if (loading4 !== null) await loading4;
|
|
29456
|
+
const result = await preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options);
|
|
29457
|
+
if (result !== null) return result;
|
|
29458
|
+
}
|
|
29459
|
+
}
|
|
29179
29460
|
async function runEngineSuggest(core2, indexName, params, partitionIds) {
|
|
29180
29461
|
const { executeSuggest } = await importOptionalReads();
|
|
29181
29462
|
await enterRead(core2, indexName);
|
|
@@ -29293,11 +29574,11 @@ function createNarsilFromCore(core2, config) {
|
|
|
29293
29574
|
async function withOpenIndex2(indexName, action) {
|
|
29294
29575
|
guardShutdown();
|
|
29295
29576
|
requireIndex(indexName);
|
|
29296
|
-
const
|
|
29577
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
29297
29578
|
try {
|
|
29298
|
-
return await action();
|
|
29579
|
+
return await action(hold);
|
|
29299
29580
|
} finally {
|
|
29300
|
-
release();
|
|
29581
|
+
hold.release();
|
|
29301
29582
|
}
|
|
29302
29583
|
}
|
|
29303
29584
|
const rebalancesStarting = /* @__PURE__ */ new Map();
|
|
@@ -29375,22 +29656,22 @@ function createNarsilFromCore(core2, config) {
|
|
|
29375
29656
|
return stats;
|
|
29376
29657
|
},
|
|
29377
29658
|
insert(indexName, document, docId, options) {
|
|
29378
|
-
return withCountedWrite(indexName, () => insertDocument2(mutationCtx, indexName, document, docId, options));
|
|
29659
|
+
return withCountedWrite(indexName, (hold) => insertDocument2(mutationCtx, indexName, document, docId, options, hold));
|
|
29379
29660
|
},
|
|
29380
29661
|
insertBatch(indexName, documents2, options) {
|
|
29381
|
-
return withCountedWrite(indexName, () => insertDocumentBatch2(mutationCtx, indexName, documents2, options));
|
|
29662
|
+
return withCountedWrite(indexName, (hold) => insertDocumentBatch2(mutationCtx, indexName, documents2, options, hold));
|
|
29382
29663
|
},
|
|
29383
29664
|
remove(indexName, docId, options) {
|
|
29384
|
-
return withCountedWrite(indexName, () => removeDocument4(mutationCtx, indexName, docId, options));
|
|
29665
|
+
return withCountedWrite(indexName, (hold) => removeDocument4(mutationCtx, indexName, docId, options, hold));
|
|
29385
29666
|
},
|
|
29386
29667
|
removeBatch(indexName, docIds, options) {
|
|
29387
|
-
return withCountedWrite(indexName, () => removeDocumentBatch2(mutationCtx, indexName, docIds, options));
|
|
29668
|
+
return withCountedWrite(indexName, (hold) => removeDocumentBatch2(mutationCtx, indexName, docIds, options, hold));
|
|
29388
29669
|
},
|
|
29389
29670
|
update(indexName, docId, document, options) {
|
|
29390
|
-
return withCountedWrite(indexName, () => updateDocument2(mutationCtx, indexName, docId, document, options));
|
|
29671
|
+
return withCountedWrite(indexName, (hold) => updateDocument2(mutationCtx, indexName, docId, document, options, hold));
|
|
29391
29672
|
},
|
|
29392
29673
|
updateBatch(indexName, updates, options) {
|
|
29393
|
-
return withCountedWrite(indexName, () => updateDocumentBatch2(mutationCtx, indexName, updates, options));
|
|
29674
|
+
return withCountedWrite(indexName, (hold) => updateDocumentBatch2(mutationCtx, indexName, updates, options, hold));
|
|
29394
29675
|
},
|
|
29395
29676
|
waitForWrites(indexName) {
|
|
29396
29677
|
guardShutdown();
|
|
@@ -29593,7 +29874,7 @@ var suggestionsCode = codeLoader(async () => {
|
|
|
29593
29874
|
const [msgpack, suggestionStore2, code7] = await Promise.all([
|
|
29594
29875
|
loadMessagePack(),
|
|
29595
29876
|
loadSuggestionStore(),
|
|
29596
|
-
import('./suggestions-leaf-
|
|
29877
|
+
import('./suggestions-leaf-FP5HSTFP.mjs')
|
|
29597
29878
|
]);
|
|
29598
29879
|
code7.bindSuggestionsHost(optionalCodeHost(), msgpack, suggestionStore2);
|
|
29599
29880
|
return code7;
|
|
@@ -30663,14 +30944,11 @@ function writesHostPart() {
|
|
|
30663
30944
|
admitBufferedUpsert,
|
|
30664
30945
|
admitInsert,
|
|
30665
30946
|
arrayPositionsAreStale,
|
|
30666
|
-
asBatchInsertError,
|
|
30667
30947
|
awaitWriteVisibility,
|
|
30948
|
+
capsEachPartition,
|
|
30668
30949
|
changeIndexOutsideExternalWaits,
|
|
30669
|
-
collectRequiredFieldFailures,
|
|
30670
30950
|
createFrozenSegment,
|
|
30671
|
-
createPendingAdmissions,
|
|
30672
30951
|
deleteNestedValue,
|
|
30673
|
-
embedChunkDocuments,
|
|
30674
30952
|
emitSyncChange,
|
|
30675
30953
|
extractVectorFromDoc,
|
|
30676
30954
|
fnv1a,
|
|
@@ -30692,7 +30970,6 @@ function writesHostPart() {
|
|
|
30692
30970
|
prepareStoredSynonyms,
|
|
30693
30971
|
providedDocId,
|
|
30694
30972
|
readHeapStatistics,
|
|
30695
|
-
recordPendingAdmission,
|
|
30696
30973
|
removeDocumentVectors,
|
|
30697
30974
|
requireDocumentObject,
|
|
30698
30975
|
requireDocumentWithinSizeLimit,
|
|
@@ -30747,7 +31024,7 @@ var segmentMerge = codeLoader(async () => {
|
|
|
30747
31024
|
const [msgpack, codec, code7] = await Promise.all([
|
|
30748
31025
|
loadMessagePack(),
|
|
30749
31026
|
loadSegmentCodec(),
|
|
30750
|
-
import('./segment-merge-leaf-
|
|
31027
|
+
import('./segment-merge-leaf-JGND6LG5.mjs')
|
|
30751
31028
|
]);
|
|
30752
31029
|
code7.bindSegmentMergeHost(optionalCodeHost(), msgpack, codec);
|
|
30753
31030
|
return code7;
|
|
@@ -31363,4 +31640,4 @@ function createDirectExecutor(options) {
|
|
|
31363
31640
|
};
|
|
31364
31641
|
}
|
|
31365
31642
|
|
|
31366
|
-
export { ARRAY_POSITION_GAP, CASE_FOLD_EXPANSION_FACTOR, COMPACTION_SEGMENT_TRIGGER, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_COPY_IDLE_TIMEOUT_MS, DEFAULT_COPY_THRESHOLD, DEFAULT_RERANK_WINDOW, DISTINCT_TERM_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FULL_MEMORY_COLLECTION_INTERVAL_MS, GROUP_FORMAT, GROUP_PATTERN_SOURCE, IDLE_MERGE_DELAY_MS, LIVE_TAIL_FLUSH_DOCUMENTS, LIVE_TAIL_FREEZE_FLOOR, LOAD_BUFFER_YIELD_INTERVAL, LONG_QUERY_THREADS, MAIN_COPY_LONE_QUERY_DOCUMENTS, MAX_PENDING_REPLICATION_DOCUMENTS, MAX_RERANK_WINDOW, NORMALIZATION_EXPANSION_FACTOR, NO_REMOVED_SPELLINGS, PARTIAL_SPELLING_HEAP_BYTES, PATTERN_RUN_HEAP_BYTES, POOL_RESTART_DELAY_MAX_MS, POOL_RESTART_DELAY_MS, REPLICATION_WINDOW, RETIRED_THREADS_EXIT_WAIT_MS, SEGMENTS_PER_MERGE, SUGGESTIONS_INDEX_INFIX, SegmentRefusal, TERM_UNIT_HEAP_BYTES, TermLookup, VECTOR_COMPONENT_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VERSION, afterCurrentTurn, alreadyPresentOnWorker, applyProjection, arrayPositionsAreStale, assemblePagedFrozenSegment, availablePatternSearch, bindOptionalWritesHost, bindQuerySignal, bindRerankWindowSource, bindSegmentCodecHost, bindSegmentMergeHost, bindWorkerCopiesHost, bm25SetsValue, buildPartialWordTableFrom, buildSegmentPayloads, clampLimit, clampRowCount, clearNormalizationCache, clearQuerySignal, compareScores, configurationPayloadOf, configureNormalizationCache, countsWithoutScores, createBooleanIndex, createCompositePartition, createDirectExecutor, createEncodedDocumentSource, createEngineCore, createEngineIndex, createEnumIndex, createFrozenRemovals, createFrozenSegment, createLoadedGeoIndex, createMemoryMonitor, createNarsil, createNarsilFromCore, createNumericIndex, createPartitionIndex, createPartitionManager, createPartitionRouter, createPublicationGate, createRequestId, createSharedFrozenSegment, createWorkerExecutor, createWorkerFactory, customTokensOf, decodeEveryStoredDocument, decodeMapWithoutFields, decodePageCursor, decodeStoredDocument, decodeSurfaceTable, decodeSynonymRules, documentsOf2 as documentsOf, dropEngineIndex, eligibleIndexNames, emitEngineEvent, emitSyncChange, emitWriteEventAfter, emptyPendingRegions, encodeDocumentTableData, encodePageCursor, encodeSegmentState, endOfValue, engineCoreOf, executePreflight, executeQuery, fanOutMatchCount, fanOutQuery, fieldIndexRemapOf, fillHits, filterTestCount, fnv1a, fnv1aUint32s, freezeEncodedSegmentShared, freezeSegmentShared, generateId, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, getVectorFieldPaths, growPartitionsTo, hasLanguage, hasStopWords, hasTokenizer, holdSyncEpochChange, holdVectorsForRollback, holdsADiscardedField, holdsQuotedText, importOptionalReads, initializeCopyPublication, insertDocument, insertDocumentBatch, insertDocumentVectors, isCompositePartition, isDeterministicFailure, isSimdAvailable, isValidWorkerAction, isValidWorkerResponse, joinPatternWorkMeter, kWayMerge, lazyQueryBinding, listBindingOf, listenForSyncChanges, loadCursorPageFor, loadDigest, loadFacetMergesFor, loadGeoCode, loadGeoCodeFor, loadGroupingFor, loadGzip, loadOptionalWrites, loadPhraseCodeFor, loadPinningFor, loadPrefixFor, loadQueryCode, loadRerankRuntime, loadSchemaCodeFor, loadSortingFor, loadStorageCode, loadSuggestionsCode, loadVectorCode, lookupOfStoredSynonyms, memoryFractionsOf, mergeDocFrequencies, mergeEntries, mergeFacets, mergeFieldIndexes, mergeKeywordSources, mergePartitionStats, mergePostings, mergePostings2, mergeSegmentContents, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, normalizeSort, now, oneAtATime, oversampledFacetConfig, oversampledShardSize, oversizedConfigurationPayloadOf, partialWordSearchModule, partitionStateOf, patternHost, patternIndexOf, prepareConfiguredSynonyms, prepareDocumentText, prepareDocumentVectors, prepareRemovalText, prepareStoredSynonyms, projectionKeepsField, providedDocId, publishCopyAction, queryBindingOf, rankPage, readHeapStatistics, readMemoryFigures, readRecentMemoryFigures, readResidentMemory, rebuildSizeIndexes, reconstructSchemaFromMetadata, regionsOverlap, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocument3 as removeDocument, removeDocumentBatch, removeDocumentBatch2, removeDocumentVectors, removedSurfaceSpellings, reportIneligible, requireEncodedConfigurationWithinSizeLimit, requireKnownConfig, requireLoadedOptionalReads, requireLoadedVectorCode, requireMatchingCursor, requirePatternSearch, requireSearchBackendSettings2 as requireSearchBackendSettings, requireSurfaceFormsForPartialFields, requireWithinResultWindow, rerankWindowCapOf, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, resolveProjection, resolveRunnableConfig, returnsEveryMatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, runtimeHost, sanitizeGlobalStats, scanPostingBlocks, segmentArraysOf, selectionKeeps, setNestedValue, shareSegmentArrays, sortModesOf, sortSignatureEntry, sortSignatureOf, startEngineCore, storedArrayPositionGap, storedBM25Params, storedDocumentsOf, survivorDocumentsOf, synonymModule, synonymSearchModule, toError2 as toError, topLevelMapKeys, updateDocument, updateDocumentBatch, updateDocumentVectors, validateBM25Params, validateBM25Shape, validateEmbeddingConfig, validateMemoryFractions, validatePartitionConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, visitTokens, withOpenIndex, workerIneligibility, workersEnabledByDefault, wrapFrozenTokenTable, writePartitionDocument, yieldToEventLoop };
|
|
31643
|
+
export { ARRAY_POSITION_GAP, CASE_FOLD_EXPANSION_FACTOR, COMPACTION_SEGMENT_TRIGGER, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_COPY_IDLE_TIMEOUT_MS, DEFAULT_COPY_THRESHOLD, DEFAULT_RERANK_WINDOW, DISTINCT_TERM_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FULL_MEMORY_COLLECTION_INTERVAL_MS, GROUP_FORMAT, GROUP_PATTERN_SOURCE, IDLE_MERGE_DELAY_MS, LIVE_TAIL_FLUSH_DOCUMENTS, LIVE_TAIL_FREEZE_FLOOR, LOAD_BUFFER_YIELD_INTERVAL, LONG_QUERY_THREADS, MAIN_COPY_LONE_QUERY_DOCUMENTS, MAX_PENDING_REPLICATION_DOCUMENTS, MAX_RERANK_WINDOW, NORMALIZATION_EXPANSION_FACTOR, NO_REMOVED_SPELLINGS, PARTIAL_SPELLING_HEAP_BYTES, PATTERN_RUN_HEAP_BYTES, POOL_RESTART_DELAY_MAX_MS, POOL_RESTART_DELAY_MS, REPLICATION_WINDOW, RETIRED_THREADS_EXIT_WAIT_MS, SEGMENTS_PER_MERGE, SUGGESTIONS_INDEX_INFIX, SegmentRefusal, TERM_UNIT_HEAP_BYTES, TermLookup, VECTOR_COMPONENT_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VERSION, afterCurrentTurn, alreadyPresentOnWorker, applyProjection, arrayPositionsAreStale, assemblePagedFrozenSegment, availablePatternSearch, bindOptionalWritesHost, bindQuerySignal, bindRerankWindowSource, bindSegmentCodecHost, bindSegmentMergeHost, bindWorkerCopiesHost, bm25SetsValue, buildPartialWordTableFrom, buildSegmentPayloads, clampLimit, clampRowCount, clearNormalizationCache, clearQuerySignal, compareScores, configurationPayloadOf, configureNormalizationCache, countsWithoutScores, createBooleanIndex, createCompositePartition, createDirectExecutor, createEncodedDocumentSource, createEngineCore, createEngineIndex, createEnumIndex, createFrozenRemovals, createFrozenSegment, createLoadedGeoIndex, createMemoryMonitor, createNarsil, createNarsilFromCore, createNumericIndex, createPartitionIndex, createPartitionManager, createPartitionRouter, createPublicationGate, createRequestId, createSharedFrozenSegment, createWorkerExecutor, createWorkerFactory, customTokensOf, decodeEveryStoredDocument, decodeMapWithoutFields, decodePageCursor, decodeStoredDocument, decodeSurfaceTable, decodeSynonymRules, documentsOf2 as documentsOf, dropEngineIndex, eligibleIndexNames, emitEngineEvent, emitSyncChange, emitWriteEventAfter, emptyPendingRegions, encodeDocumentTableData, encodePageCursor, encodeSegmentState, endOfValue, engineCoreOf, executePreflight, executeQuery, fanOutMatchCount, fanOutQuery, fieldIndexRemapOf, fillHits, filterTestCount, fnv1a, fnv1aUint32s, freezeEncodedSegmentShared, freezeSegmentShared, generateId, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, getVectorFieldPaths, growPartitionsTo, hasLanguage, hasStopWords, hasTokenizer, holdSyncEpochChange, holdVectorsForRollback, holdsADiscardedField, holdsQuotedText, importOptionalReads, initializeCopyPublication, insertDocument, insertDocumentBatch, insertDocumentVectors, isCompositePartition, isDeterministicFailure, isSimdAvailable, isValidWorkerAction, isValidWorkerResponse, joinPatternWorkMeter, kWayMerge, lazyQueryBinding, listBindingOf, listenForSyncChanges, loadCursorPageFor, loadDigest, loadFacetMergesFor, loadGeoCode, loadGeoCodeFor, loadGroupingFor, loadGzip, loadOptionalWrites, loadPhraseCodeFor, loadPinningFor, loadPrefixFor, loadQueryCode, loadRerankRuntime, loadSchemaCodeFor, loadSortingFor, loadStorageCode, loadSuggestionsCode, loadVectorCode, lookupOfStoredSynonyms, memoryFractionsOf, mergeDocFrequencies, mergeEntries, mergeFacets, mergeFieldIndexes, mergeKeywordSources, mergePartitionStats, mergePostings, mergePostings2, mergeSegmentContents, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, normalizeSort, now, oneAtATime, oversampledFacetConfig, oversampledShardSize, oversizedConfigurationPayloadOf, partialWordSearchModule, partitionStateOf, patternHost, patternIndexOf, prepareConfiguredSynonyms, prepareDocumentText, prepareDocumentVectors, prepareDocumentWrite, prepareRemovalText, prepareStoredSynonyms, projectionKeepsField, providedDocId, publishCopyAction, queryBindingOf, rankPage, readHeapStatistics, readMemoryFigures, readRecentMemoryFigures, readResidentMemory, rebuildSizeIndexes, reconstructSchemaFromMetadata, regionsOverlap, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocument3 as removeDocument, removeDocumentBatch, removeDocumentBatch2, removeDocumentVectors, removedSurfaceSpellings, reportIneligible, requireEncodedConfigurationWithinSizeLimit, requireKnownConfig, requireLoadedOptionalReads, requireLoadedVectorCode, requireMatchingCursor, requirePatternSearch, requireSearchBackendSettings2 as requireSearchBackendSettings, requireSurfaceFormsForPartialFields, requireWithinResultWindow, rerankWindowCapOf, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, resolveProjection, resolveRunnableConfig, returnsEveryMatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, runtimeHost, sanitizeGlobalStats, scanPostingBlocks, segmentArraysOf, selectionKeeps, setNestedValue, shareSegmentArrays, sortModesOf, sortSignatureEntry, sortSignatureOf, startEngineCore, storedArrayPositionGap, storedBM25Params, storedDocumentsOf, survivorDocumentsOf, synonymModule, synonymSearchModule, toError2 as toError, topLevelMapKeys, updateDocument, updateDocumentBatch, updateDocumentVectors, validateBM25Params, validateBM25Shape, validateEmbeddingConfig, validateMemoryFractions, validatePartitionConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, visitTokens, withOpenIndex, workerIneligibility, workersEnabledByDefault, wrapFrozenTokenTable, writePartitionDocument, yieldToEventLoop };
|