@delali/narsil 0.3.1-next.181 → 0.3.1-next.184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-CPSHS46N.mjs} +139 -177
- package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
- package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
- package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-ZTBHYQMN.mjs} +621 -314
- package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
- package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
- package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
- package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-4XMCITJ6.mjs} +886 -259
- package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
- package/dist/{chunk-KHJPMZE2.mjs → chunk-2KCCQK4R.mjs} +1 -1
- package/dist/{chunk-2KWTQ2P7.mjs → chunk-2YWWHMVC.mjs} +2 -2
- package/dist/{chunk-Y27E2MGC.mjs → chunk-3D4IMFSE.mjs} +2 -2
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/{chunk-XFOL77WH.mjs → chunk-4JBTALUC.mjs} +1 -1
- package/dist/{chunk-ISJ35523.mjs → chunk-5BUEBIXR.mjs} +2 -2
- package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
- package/dist/{chunk-UISV46WX.mjs → chunk-7WXGUTWK.mjs} +3 -3
- package/dist/{chunk-QFJMRB7D.mjs → chunk-A3OAFUZI.mjs} +1 -1
- package/dist/{chunk-GO6TRSJ6.mjs → chunk-BVZWWMOU.mjs} +1 -1
- package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
- package/dist/{chunk-XQQKETAF.mjs → chunk-CODGHJVW.mjs} +4 -4
- package/dist/{chunk-34IOOYXA.mjs → chunk-CYCVPTTR.mjs} +2 -2
- package/dist/{chunk-26E5MO3Y.mjs → chunk-DU23J5JG.mjs} +5 -5
- package/dist/{chunk-JCWC75NH.mjs → chunk-EDEE7ETQ.mjs} +1 -1
- package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
- package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
- package/dist/{chunk-BIIROQ5Q.mjs → chunk-FCQMZDRI.mjs} +2 -2
- package/dist/{chunk-YXZDTKQS.mjs → chunk-G3XRHAJA.mjs} +2 -2
- package/dist/{chunk-XTSFRWAS.mjs → chunk-GGIW6INA.mjs} +7 -6
- package/dist/{chunk-R7PLSVK3.mjs → chunk-H4QYQ5DE.mjs} +1 -1
- package/dist/{chunk-A4XYDUVN.mjs → chunk-HQPK73EW.mjs} +3 -3
- package/dist/{chunk-YPV2L7M5.mjs → chunk-II6XF6M4.mjs} +5 -5
- package/dist/{chunk-F3FDHXCJ.mjs → chunk-LR6JJVML.mjs} +251 -50
- package/dist/{chunk-ZB5X5WAK.mjs → chunk-M5I64AQW.mjs} +11 -6
- package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
- package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
- package/dist/{chunk-XITVDRRH.mjs → chunk-O337MDRJ.mjs} +6 -6
- package/dist/{chunk-XNFBYAXI.mjs → chunk-SEM3PBRH.mjs} +1 -1
- package/dist/{chunk-2C3BXZGA.mjs → chunk-SL53DHVE.mjs} +814 -539
- package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
- package/dist/{chunk-LOQHA6QK.mjs → chunk-URVH6BLO.mjs} +2 -2
- package/dist/{chunk-JDIATDPY.mjs → chunk-VYSWVQCE.mjs} +1 -1
- package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
- package/dist/{chunk-7FC2ZILY.mjs → chunk-WSIU6YXQ.mjs} +67 -78
- package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
- package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
- package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
- package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
- package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-37IH3ISS.mjs} +5 -4
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +398 -144
- package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
- package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
- package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +5 -4
- package/dist/index.react-native.mjs +4 -3
- package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-PU2XT527.mjs} +15 -14
- package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
- package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-YIVJMQ4D.mjs} +9 -6
- package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +5 -4
- package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -16
- package/dist/{query-actions-2JOLPELC.mjs → query-actions-7EOKT3WD.mjs} +5 -4
- package/dist/query-suggestions/merge-worker.mjs +8 -7
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +6 -5
- package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-7TPO4FLE.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-CBU4M3EU.mjs} +813 -538
- package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-IEY4DARA.mjs} +1 -1
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
- package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-T5HF3FW3.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-ZQPSXN6C.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-ZUFOYZ5P.mjs} +1 -1
- package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
- package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
- package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
- package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
- package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-QETIM6BI.mjs} +7 -4
- package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
- package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
- package/dist/{runtime-4GKUIAKV.mjs → react-native/chunks/runtime-UOHD2XUJ.mjs} +23 -3
- package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
- package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
- package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-LNOLKWZB.mjs} +895 -267
- package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
- package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
- package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
- package/dist/react-native/partial-words.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +4 -3
- package/dist/react-native/sync.mjs +4 -3
- package/dist/react-native/synonyms.mjs +4 -3
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-E2HFKKGJ.mjs → rerank-KTV5L7MZ.mjs} +5 -4
- package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
- package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
- package/dist/{react-native/chunks/runtime-BUCSAK4U.mjs → runtime-USQYKN3I.mjs} +23 -3
- package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
- package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +23 -22
- package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-ZI3H2WJY.mjs} +595 -164
- package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
- package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +11 -10
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +5 -4
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +5 -4
- package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
- package/dist/{thread-OWXVP2ZA.mjs → thread-BJXE77AI.mjs} +14 -13
- package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
- package/dist/workers/entry.mjs +11 -10
- package/openapi.json +32 -0
- package/package.json +7 -7
- package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
- package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { encodeDocument } from './chunk-KLVMK3R5.mjs';
|
|
2
|
-
import { workerResourceLimits, admittedHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal } from './chunk-
|
|
2
|
+
import { workerResourceLimits, admittedHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal } from './chunk-F4OOBPKA.mjs';
|
|
3
3
|
import { headerAt, MAP_TAGS, decode, Decoder } from './chunk-GXCDWLQI.mjs';
|
|
4
4
|
import { newSyncEpoch, isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
5
5
|
import { english } from './chunk-7T4LAYT4.mjs';
|
|
6
6
|
import { patternSearchModule, PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, matchPattern, claimsTypoPreparation, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, DEFAULT_PATTERN_WORK_CAP, bitsetSet, discardStoredValues, MAX_FILTER_DEPTH, createBitSet, installStoredFields, storedValueBytes, bitsetIterator, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIsEmpty, prepareStoredValues, fuzzyTermMatches, GrowableUint32, checkedSize, buildStoredValues, bitsetFromSet, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyPrefixOf, boundedLevenshtein, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, FILTER_OPERAND_COPY_DEPTH } from './chunk-YM2HI5N5.mjs';
|
|
7
7
|
import { parsedTypeOf, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, isStringValuedBase, flattenSchema, schemaFieldsOf, getNestedValue, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, requireLoadedMessagePack, vectorDimensionOf, isSingleStringFieldType, messagePackUpperBoundBytes, extractVectorFieldsFromSchema, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, loadMessagePack, validateRequiredFields, requireDocumentWithinSizeLimit, raiseEarlierReleaseGraphSetting, isWordIndexedBase, PROTOTYPE_POLLUTION_KEYS, tokenizeOptions, isPlainObject, isWordIndexedFieldType, loadedMessagePack, validateDocument, validateDocumentStrict, textFieldsChanged, getFlatSchema, getOrCreateFieldNameIndex, MAX_NESTING_DEPTH, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, MAX_DOCUMENT_NESTING_DEPTH, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES } from './chunk-X5NHWOXT.mjs';
|
|
8
|
-
import { detectRuntime, splitWorkerBudget, resolveWorkerCount, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, countEngineThread, requirePinnedState,
|
|
9
|
-
import {
|
|
8
|
+
import { detectRuntime, splitWorkerBudget, resolveWorkerCount, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, countEngineThread, requirePinnedState, createDocIdMap, SHUTDOWN_TIMEOUT_MS, stopOfSignal, requireSearchActive, resolveWorkerEntry, missingWorkerEntry, signalOfSearch, answerFromOneState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, sharedStopOfSignal, engineThreadYoungGenerationBytes, resolveRequestThreadCount, spawnNodeWorker, bindSearchSignal, WORKER_YOUNG_GENERATION_MB, nodeThreadId, youngGenerationBytesOfThisV8 } from './chunk-6SZQETSC.mjs';
|
|
9
|
+
import { changeIndexOutsideExternalWaits, turnsAreBudgeted, turnIsSpent, yieldTurn, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, noteThreadSlotLive, threadSlotOfWorker, indexChangeOutsideExternalWaits, PROCESS_LIMIT_REFRESH_INTERVAL_MS, countWrite, threadSlotGone, noteThreadSlotGone, adoptSharedTurns, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_BLOCK_MAX_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_ATTACH_RETRY_MS, MAX_WASM_PAGES, MAX_VECTOR_ORDINALS, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_DOC_ID_TABLE_BYTES, LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, DISK_STORAGE_BLOCK_BYTES, waitOutsideCountedWrite, collectAndReturnPages } from './chunk-UPDNNOTG.mjs';
|
|
10
|
+
import { MIN_DOCUMENTS_FOR_SEGMENTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, MAX_PARTITIONS, DEFAULT_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, BATCH_CHUNK_SIZE, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_CALLS_ACROSS_INDEX_CHANGES, DEFAULT_PAGE_SIZE, CURSOR_BINDING_BUFFER_BYTES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, DEFAULT_OFFSET, MIN_SWEEP_INTERVAL_MS, MIN_RANK_CONSTANT, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, FUSION_TIE_INSERTION_LIMIT, DEFAULT_RANK_CONSTANT, DEFAULT_LINEAR_ALPHA, STATISTICS_BROADCAST_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, EXPORT_CHUNK_BYTES, DEFAULT_MAX_PARTITIONS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, ANALYSIS_CHANGE_MEMBERS, MIN_HYBRID_DEPTH, MAX_SUGGEST_SCATTER_LIMIT, MAX_SUGGEST_LIMIT, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_SUGGEST_LIMIT, DEFAULT_MAX_PER_GROUP, HEAP_PRESSURE_WARNING_FRACTION, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-3ECIRFS5.mjs';
|
|
10
11
|
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, requireSearchBackendSettings, NATIVE_NEEDS_ROOM, NATIVE_OK } from './chunk-D6FYUTFN.mjs';
|
|
11
12
|
import { PLACEMENT_ROOM_REQUESTS, NEAREST_LINK_FALLBACK_HOSTS, MIN_M, MAX_M, MAX_LAYER_CAP, INITIAL_UPPER_SLOTS, INITIAL_LIST_CAPACITY, FARTHEST_REJECTED_LINKS_KEPT, EARLIER_RELEASE_MIN_M, DEFAULT_EF_SEARCH, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, CONNECT_RESTARTS_BEFORE_ONE_PASS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD } from './chunk-ZXX5GZEX.mjs';
|
|
12
13
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS, NATIVE_CRC32_MIN_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, MAX_DOCUMENT_ENCODING_DEPTH, DOCUMENT_REPLY_ENVELOPE_DEPTH, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER } from './chunk-4PY4JYMY.mjs';
|
|
@@ -3052,11 +3053,11 @@ function captureFrozen(input, output2) {
|
|
|
3052
3053
|
spellings: captured.spellings === null ? null : copy(captured.spellings),
|
|
3053
3054
|
removalIndex: captured.removalIndex === null ? null : copy(captured.removalIndex)
|
|
3054
3055
|
};
|
|
3055
|
-
const
|
|
3056
|
-
return { raw, writable };
|
|
3056
|
+
const writable2 = copy(output2);
|
|
3057
|
+
return { raw, writable: writable2 };
|
|
3057
3058
|
}
|
|
3058
3059
|
var heldFrozen = /* @__PURE__ */ new WeakMap();
|
|
3059
|
-
function rawsOf({ raw, writable }) {
|
|
3060
|
+
function rawsOf({ raw, writable: writable2 }) {
|
|
3060
3061
|
const { terms, surfaces, spellings, removalIndex } = raw;
|
|
3061
3062
|
return [
|
|
3062
3063
|
raw.documentCount,
|
|
@@ -3081,16 +3082,16 @@ function rawsOf({ raw, writable }) {
|
|
|
3081
3082
|
removalIndex?.starts,
|
|
3082
3083
|
removalIndex?.rows,
|
|
3083
3084
|
removalIndex?.termOfSlot,
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3085
|
+
writable2.tombstones,
|
|
3086
|
+
writable2.tombstoneCount,
|
|
3087
|
+
writable2.removedDocuments,
|
|
3088
|
+
writable2.removedOccurrences,
|
|
3089
|
+
writable2.removedSurfaces,
|
|
3090
|
+
writable2.removedSpellings,
|
|
3091
|
+
writable2.pendingHeader,
|
|
3092
|
+
writable2.pendingDocuments,
|
|
3093
|
+
writable2.pendingTokens,
|
|
3094
|
+
writable2.pendingUnits
|
|
3094
3095
|
];
|
|
3095
3096
|
}
|
|
3096
3097
|
function stillHeld(raws, held) {
|
|
@@ -3106,7 +3107,7 @@ function readHeld(input, output2, captured) {
|
|
|
3106
3107
|
const raws = rawsOf(captured);
|
|
3107
3108
|
const known = heldFrozen.get(input);
|
|
3108
3109
|
if (known !== void 0 && known.output === output2 && stillHeld(raws, known)) return known;
|
|
3109
|
-
const { raw, writable } = captured;
|
|
3110
|
+
const { raw, writable: writable2 } = captured;
|
|
3110
3111
|
const regions = new Regions();
|
|
3111
3112
|
const documentCount = raw.documentCount;
|
|
3112
3113
|
requireSegment(Number.isInteger(documentCount) && documentCount >= 0 && documentCount <= 4294967295);
|
|
@@ -3134,16 +3135,16 @@ function readHeld(input, output2, captured) {
|
|
|
3134
3135
|
}
|
|
3135
3136
|
};
|
|
3136
3137
|
const state = {
|
|
3137
|
-
tombstones: regions.array(
|
|
3138
|
-
tombstoneCount: regions.array(
|
|
3139
|
-
removedDocuments: regions.array(
|
|
3140
|
-
removedOccurrences: regions.array(
|
|
3141
|
-
removedSurfaces: regions.array(
|
|
3142
|
-
removedSpellings: regions.array(
|
|
3143
|
-
pendingHeader: regions.array(
|
|
3144
|
-
pendingDocuments: regions.array(
|
|
3145
|
-
pendingTokens: regions.array(
|
|
3146
|
-
pendingUnits: regions.array(
|
|
3138
|
+
tombstones: regions.array(writable2.tombstones, "Uint32Array", Math.max(1, Math.ceil(documentCount / 32)), true),
|
|
3139
|
+
tombstoneCount: regions.array(writable2.tombstoneCount, "Uint32Array", 1, true),
|
|
3140
|
+
removedDocuments: regions.array(writable2.removedDocuments, "Uint32Array", terms.counts.length, true),
|
|
3141
|
+
removedOccurrences: regions.array(writable2.removedOccurrences, "Float64Array", terms.counts.length, true),
|
|
3142
|
+
removedSurfaces: regions.array(writable2.removedSurfaces, "Uint32Array", surfaces?.counts.length ?? 0, true),
|
|
3143
|
+
removedSpellings: regions.array(writable2.removedSpellings, "Uint32Array", spellings?.counts.length ?? 0, true),
|
|
3144
|
+
pendingHeader: regions.array(writable2.pendingHeader, "Uint32Array", PENDING_HEADER_WORDS, true),
|
|
3145
|
+
pendingDocuments: regions.array(writable2.pendingDocuments, "Uint32Array", void 0, true),
|
|
3146
|
+
pendingTokens: regions.array(writable2.pendingTokens, "Uint32Array", void 0, true),
|
|
3147
|
+
pendingUnits: regions.array(writable2.pendingUnits, "Uint16Array", void 0, true)
|
|
3147
3148
|
};
|
|
3148
3149
|
requireSegment(state.pendingTokens.length % PENDING_TOKEN_WORDS === 0);
|
|
3149
3150
|
regions.check();
|
|
@@ -6791,10 +6792,10 @@ var Regions2 = class {
|
|
|
6791
6792
|
starts = [];
|
|
6792
6793
|
ends = [];
|
|
6793
6794
|
writable = [];
|
|
6794
|
-
read(input, kind, length,
|
|
6795
|
+
read(input, kind, length, writable2 = true) {
|
|
6795
6796
|
const value = typedRegion(input, kind);
|
|
6796
6797
|
requireSegment(value !== null && (length === void 0 || value.length === length));
|
|
6797
|
-
if (
|
|
6798
|
+
if (writable2) this.writes.push(value);
|
|
6798
6799
|
else this.reads.push(value);
|
|
6799
6800
|
const byteLength = value.byteLength;
|
|
6800
6801
|
if (byteLength > 0) {
|
|
@@ -6802,19 +6803,19 @@ var Regions2 = class {
|
|
|
6802
6803
|
this.buffers.push(value.buffer);
|
|
6803
6804
|
this.starts.push(start);
|
|
6804
6805
|
this.ends.push(start + byteLength);
|
|
6805
|
-
this.writable.push(
|
|
6806
|
+
this.writable.push(writable2);
|
|
6806
6807
|
}
|
|
6807
6808
|
return value;
|
|
6808
6809
|
}
|
|
6809
6810
|
check() {
|
|
6810
|
-
const { buffers: buffers2, starts, ends, writable } = this;
|
|
6811
|
+
const { buffers: buffers2, starts, ends, writable: writable2 } = this;
|
|
6811
6812
|
for (let at = 1; at < buffers2.length; at++) {
|
|
6812
6813
|
const buffer = buffers2[at];
|
|
6813
6814
|
const start = starts[at];
|
|
6814
6815
|
const end = ends[at];
|
|
6815
|
-
const writes =
|
|
6816
|
+
const writes = writable2[at];
|
|
6816
6817
|
for (let previous = 0; previous < at; previous++)
|
|
6817
|
-
if (buffers2[previous] === buffer && (writes ||
|
|
6818
|
+
if (buffers2[previous] === buffer && (writes || writable2[previous]) && start < ends[previous] && starts[previous] < end)
|
|
6818
6819
|
requireSegment(false);
|
|
6819
6820
|
}
|
|
6820
6821
|
}
|
|
@@ -11465,6 +11466,7 @@ function searchHostPart() {
|
|
|
11465
11466
|
listBindingOf,
|
|
11466
11467
|
MAX_HYBRID_DEPTH,
|
|
11467
11468
|
MAX_PHRASE_WORDS,
|
|
11469
|
+
MAX_CALLS_ACROSS_INDEX_CHANGES,
|
|
11468
11470
|
MAX_QUERY_PHRASES,
|
|
11469
11471
|
MAX_RERANK_TIMEOUT_MS,
|
|
11470
11472
|
MAX_RERANK_WINDOW,
|
|
@@ -12680,7 +12682,7 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
12680
12682
|
|
|
12681
12683
|
// src/core/segment-codec-code.ts
|
|
12682
12684
|
var segmentCodec = codeLoader(async () => {
|
|
12683
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-
|
|
12685
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-DO75VKEV.mjs')]);
|
|
12684
12686
|
code7.bindSegmentCodecHost(optionalCodeHost(), msgpack);
|
|
12685
12687
|
return code7;
|
|
12686
12688
|
}, "segment codec");
|
|
@@ -12688,7 +12690,7 @@ var loadSegmentCodec = segmentCodec.load;
|
|
|
12688
12690
|
|
|
12689
12691
|
// src/query-suggestions/store-code.ts
|
|
12690
12692
|
var suggestionStore = codeLoader(async () => {
|
|
12691
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-
|
|
12693
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-YIH36Y5J.mjs')]);
|
|
12692
12694
|
code7.bindSuggestionStoreHost(optionalCodeHost(), msgpack);
|
|
12693
12695
|
return code7;
|
|
12694
12696
|
}, "query suggestions store");
|
|
@@ -12747,7 +12749,7 @@ function markSimdUnavailable() {
|
|
|
12747
12749
|
|
|
12748
12750
|
// src/workers/vector-core-code.ts
|
|
12749
12751
|
var vectorCore = codeLoader(
|
|
12750
|
-
() => import('./core-leaf-
|
|
12752
|
+
() => import('./core-leaf-NEUM7PU3.mjs').then((code7) => {
|
|
12751
12753
|
code7.bindVectorCoreHost(optionalCodeHost());
|
|
12752
12754
|
if (!code7.simdModuleLoaded()) markSimdUnavailable();
|
|
12753
12755
|
return code7;
|
|
@@ -12769,7 +12771,7 @@ var storageCode = codeLoader(async () => {
|
|
|
12769
12771
|
loadSegmentMerge(),
|
|
12770
12772
|
loadSuggestionStore(),
|
|
12771
12773
|
loadVectorCore(),
|
|
12772
|
-
import('./storage-leaf-
|
|
12774
|
+
import('./storage-leaf-ZI3H2WJY.mjs')
|
|
12773
12775
|
]);
|
|
12774
12776
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
12775
12777
|
return code7;
|
|
@@ -15578,10 +15580,10 @@ function uint323(value) {
|
|
|
15578
15580
|
var MergeRegions = class _MergeRegions {
|
|
15579
15581
|
reads = [];
|
|
15580
15582
|
writes = [];
|
|
15581
|
-
read(input, kind, length,
|
|
15583
|
+
read(input, kind, length, writable2 = false) {
|
|
15582
15584
|
const value = typedRegion(input, kind);
|
|
15583
15585
|
requireSegment(value !== null && (length === void 0 || value.length === length));
|
|
15584
|
-
(
|
|
15586
|
+
(writable2 ? this.writes : this.reads).push(value);
|
|
15585
15587
|
return value;
|
|
15586
15588
|
}
|
|
15587
15589
|
check() {
|
|
@@ -15599,11 +15601,11 @@ var MergeRegions = class _MergeRegions {
|
|
|
15599
15601
|
}
|
|
15600
15602
|
}
|
|
15601
15603
|
}
|
|
15602
|
-
strings(input, stride,
|
|
15603
|
-
const counts = this.read(input.counts, "Uint32Array", void 0,
|
|
15604
|
-
const text = this.read(input.text, "Uint8Array", void 0,
|
|
15605
|
-
const offsets = this.read(input.offsets, "Uint32Array",
|
|
15606
|
-
if (!
|
|
15604
|
+
strings(input, stride, writable2 = false) {
|
|
15605
|
+
const counts = this.read(input.counts, "Uint32Array", void 0, writable2);
|
|
15606
|
+
const text = this.read(input.text, "Uint8Array", void 0, writable2);
|
|
15607
|
+
const offsets = this.read(input.offsets, "Uint32Array", writable2 ? void 0 : counts.length * stride + 1, writable2);
|
|
15608
|
+
if (!writable2) requireSegment(counts.length <= 4294967295 && text.length <= 4294967295);
|
|
15607
15609
|
return { text, offsets, counts };
|
|
15608
15610
|
}
|
|
15609
15611
|
source(input) {
|
|
@@ -15649,11 +15651,11 @@ var MergeRegions = class _MergeRegions {
|
|
|
15649
15651
|
...positions2
|
|
15650
15652
|
};
|
|
15651
15653
|
}
|
|
15652
|
-
positions(input,
|
|
15654
|
+
positions(input, writable2, count) {
|
|
15653
15655
|
requireSegment(input.positionOffsets === null === (input.positionValues === null));
|
|
15654
|
-
const positionOffsets = input.positionOffsets === null ? null : this.read(input.positionOffsets, "Uint32Array", count,
|
|
15655
|
-
const positionValues = input.positionValues === null ? null : this.read(input.positionValues, "Uint32Array", void 0,
|
|
15656
|
-
if (!
|
|
15656
|
+
const positionOffsets = input.positionOffsets === null ? null : this.read(input.positionOffsets, "Uint32Array", count, writable2);
|
|
15657
|
+
const positionValues = input.positionValues === null ? null : this.read(input.positionValues, "Uint32Array", void 0, writable2);
|
|
15658
|
+
if (!writable2) requireSegment(positionValues === null || positionValues.length <= 4294967295);
|
|
15657
15659
|
return { positionOffsets, positionValues };
|
|
15658
15660
|
}
|
|
15659
15661
|
};
|
|
@@ -19104,8 +19106,7 @@ function assembleFrozenSegment(segmentId, source2, documentSource, sharedSnapsho
|
|
|
19104
19106
|
if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
|
|
19105
19107
|
return removals.spellingsOf(ordinal);
|
|
19106
19108
|
},
|
|
19107
|
-
prepareTombstone(docId, removedSpellings, analysis) {
|
|
19108
|
-
const ordinal = docStore.getInternalId(docId);
|
|
19109
|
+
prepareTombstone(docId, removedSpellings, analysis, ordinal = docStore.getInternalId(docId)) {
|
|
19109
19110
|
if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
|
|
19110
19111
|
const publishTombstone = tombstones.prepare(ordinal);
|
|
19111
19112
|
const strings4 = analysis === void 0 ? null : prepareFrozenStrings(
|
|
@@ -21493,7 +21494,7 @@ function hasLanguage(name) {
|
|
|
21493
21494
|
|
|
21494
21495
|
// src/narsil/restore-queue-code.ts
|
|
21495
21496
|
var restoreQueue = codeLoader(async () => {
|
|
21496
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
21497
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-ES7TDJ5V.mjs')]);
|
|
21497
21498
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
21498
21499
|
return code7;
|
|
21499
21500
|
}, "restore");
|
|
@@ -21720,7 +21721,7 @@ function storedBM25Params(params) {
|
|
|
21720
21721
|
|
|
21721
21722
|
// src/workers/vector-code.ts
|
|
21722
21723
|
var vectorCode = codeLoader(async () => {
|
|
21723
|
-
const [vectorCore2, code7] = await Promise.all([loadVectorCore(), import('./module-
|
|
21724
|
+
const [vectorCore2, code7] = await Promise.all([loadVectorCore(), import('./module-Z3CL4GJ4.mjs')]);
|
|
21724
21725
|
code7.bindVectorHost(optionalCodeHost(), vectorCore2);
|
|
21725
21726
|
return code7;
|
|
21726
21727
|
}, "vector");
|
|
@@ -22058,7 +22059,7 @@ function storedArrayPositionGap(value) {
|
|
|
22058
22059
|
|
|
22059
22060
|
// src/engine/embed-code.ts
|
|
22060
22061
|
var embedCode = codeLoader(
|
|
22061
|
-
() => import('./embed-
|
|
22062
|
+
() => import('./embed-2QMYCD2Q.mjs').then((code7) => {
|
|
22062
22063
|
code7.bindEmbedHost(optionalCodeHost());
|
|
22063
22064
|
return code7;
|
|
22064
22065
|
}),
|
|
@@ -22068,7 +22069,7 @@ var loadEmbedCode = embedCode.load;
|
|
|
22068
22069
|
|
|
22069
22070
|
// src/engine/optional-writes-code.ts
|
|
22070
22071
|
var optionalWrites = codeLoader(async () => {
|
|
22071
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
22072
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-YIVJMQ4D.mjs')]);
|
|
22072
22073
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
22073
22074
|
return code7;
|
|
22074
22075
|
}, "write");
|
|
@@ -22272,21 +22273,9 @@ function requireMatchingDocId(docId, document) {
|
|
|
22272
22273
|
{ docId, documentId: embeddedDocId }
|
|
22273
22274
|
);
|
|
22274
22275
|
}
|
|
22275
|
-
function asBatchInsertError(err) {
|
|
22276
|
-
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
22277
|
-
}
|
|
22278
|
-
function createPendingAdmissions() {
|
|
22279
|
-
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
22280
|
-
}
|
|
22281
22276
|
function capsEachPartition(ctx, manager) {
|
|
22282
22277
|
return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
|
|
22283
22278
|
}
|
|
22284
|
-
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
22285
|
-
pending.total += 1;
|
|
22286
|
-
if (!capsEachPartition(ctx, manager)) return;
|
|
22287
|
-
const partitionId = manager.routePartition(docId);
|
|
22288
|
-
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
22289
|
-
}
|
|
22290
22279
|
function admitInsert(ctx, indexName, manager, docId, pending) {
|
|
22291
22280
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
22292
22281
|
if (rebalancing) {
|
|
@@ -22318,66 +22307,6 @@ function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
|
|
|
22318
22307
|
ctx.rebalanceTargetPartitionCount(indexName)
|
|
22319
22308
|
);
|
|
22320
22309
|
}
|
|
22321
|
-
function collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed) {
|
|
22322
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22323
|
-
try {
|
|
22324
|
-
validateRequiredFields(documents2[i], required);
|
|
22325
|
-
} catch (err) {
|
|
22326
|
-
chunkFailedIndexes.add(i);
|
|
22327
|
-
failed.push({ docId: providedDocId(documents2[i]) ?? "", error: asBatchInsertError(err) });
|
|
22328
|
-
}
|
|
22329
|
-
}
|
|
22330
|
-
}
|
|
22331
|
-
async function embedChunkDocuments(entry, documents2, chunkStart, chunkEnd, signal, chunkFailedIndexes, failed) {
|
|
22332
|
-
if (entry.config.embedding === void 0) return;
|
|
22333
|
-
const { assertDocumentCarriesMappedVectors, embedBatchDocumentFields } = await loadEmbedCode();
|
|
22334
|
-
if (entry.embeddingAdapter === null) {
|
|
22335
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22336
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
22337
|
-
try {
|
|
22338
|
-
assertDocumentCarriesMappedVectors(
|
|
22339
|
-
documents2[i],
|
|
22340
|
-
entry.config.embedding,
|
|
22341
|
-
entry.embeddingAdapterName
|
|
22342
|
-
);
|
|
22343
|
-
} catch (err) {
|
|
22344
|
-
chunkFailedIndexes.add(i);
|
|
22345
|
-
failed.push({
|
|
22346
|
-
docId: providedDocId(documents2[i]) ?? "",
|
|
22347
|
-
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
22348
|
-
});
|
|
22349
|
-
}
|
|
22350
|
-
}
|
|
22351
|
-
return;
|
|
22352
|
-
}
|
|
22353
|
-
const embeddableSlice = [];
|
|
22354
|
-
const embeddableOriginalIndexes = [];
|
|
22355
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22356
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
22357
|
-
embeddableSlice.push(documents2[i]);
|
|
22358
|
-
embeddableOriginalIndexes.push(i);
|
|
22359
|
-
}
|
|
22360
|
-
if (embeddableSlice.length === 0) return;
|
|
22361
|
-
try {
|
|
22362
|
-
const embedResult = await embedBatchDocumentFields(
|
|
22363
|
-
embeddableSlice,
|
|
22364
|
-
entry.config.embedding,
|
|
22365
|
-
entry.embeddingAdapter,
|
|
22366
|
-
signal
|
|
22367
|
-
);
|
|
22368
|
-
for (const [sliceIndex, error] of embedResult.failed) {
|
|
22369
|
-
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
22370
|
-
chunkFailedIndexes.add(originalIdx);
|
|
22371
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error });
|
|
22372
|
-
}
|
|
22373
|
-
} catch (err) {
|
|
22374
|
-
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
22375
|
-
for (const originalIdx of embeddableOriginalIndexes) {
|
|
22376
|
-
chunkFailedIndexes.add(originalIdx);
|
|
22377
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error: embeddingError });
|
|
22378
|
-
}
|
|
22379
|
-
}
|
|
22380
|
-
}
|
|
22381
22310
|
|
|
22382
22311
|
// src/engine/mutations/write-visibility.ts
|
|
22383
22312
|
async function awaitWriteVisibility(ctx, indexName) {
|
|
@@ -22386,33 +22315,19 @@ async function awaitWriteVisibility(ctx, indexName) {
|
|
|
22386
22315
|
}
|
|
22387
22316
|
|
|
22388
22317
|
// src/engine/mutations/insert.ts
|
|
22389
|
-
async function insertDocument(ctx, indexName, document, docId, options) {
|
|
22318
|
+
async function insertDocument(ctx, indexName, document, docId, options, hold) {
|
|
22390
22319
|
ctx.guardShutdown();
|
|
22391
|
-
|
|
22320
|
+
let entry = ctx.requireIndex(indexName);
|
|
22392
22321
|
if (docId !== void 0) requireMatchingDocId(docId, document);
|
|
22393
22322
|
const resolvedDocId = docId ?? providedDocId(document) ?? ctx.idGenerator();
|
|
22394
22323
|
validateDocId(resolvedDocId);
|
|
22395
|
-
|
|
22396
|
-
if (
|
|
22324
|
+
const beforeInsert = ctx.pluginRegistry.hasHooks("beforeInsert") ? () => ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document }) : null;
|
|
22325
|
+
if (beforeInsert !== null || entry.config.embedding !== void 0) {
|
|
22326
|
+
const { prepareDocumentWrite: prepareDocumentWrite2 } = await loadOptionalWrites();
|
|
22327
|
+
entry = await prepareDocumentWrite2(ctx, indexName, document, hold, beforeInsert);
|
|
22328
|
+
} else if (entry.config.required !== void 0) {
|
|
22397
22329
|
validateRequiredFields(document, entry.config.required);
|
|
22398
22330
|
}
|
|
22399
|
-
if (entry.config.embedding) {
|
|
22400
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
22401
|
-
if (entry.embeddingAdapter) {
|
|
22402
|
-
await embedDocumentFields(
|
|
22403
|
-
document,
|
|
22404
|
-
entry.config.embedding,
|
|
22405
|
-
entry.embeddingAdapter,
|
|
22406
|
-
ctx.abortController.signal
|
|
22407
|
-
);
|
|
22408
|
-
} else {
|
|
22409
|
-
assertDocumentCarriesMappedVectors(
|
|
22410
|
-
document,
|
|
22411
|
-
entry.config.embedding,
|
|
22412
|
-
entry.embeddingAdapterName
|
|
22413
|
-
);
|
|
22414
|
-
}
|
|
22415
|
-
}
|
|
22416
22331
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
22417
22332
|
if (sizing !== null) await sizing;
|
|
22418
22333
|
const insertManager = ctx.requireManager(indexName);
|
|
@@ -22492,16 +22407,18 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
22492
22407
|
return resolvedDocId;
|
|
22493
22408
|
}
|
|
22494
22409
|
await ctx.orchestrator.settlePublications(indexName);
|
|
22495
|
-
|
|
22496
|
-
|
|
22497
|
-
|
|
22498
|
-
|
|
22410
|
+
if (ctx.pluginRegistry.hasHooks("afterInsert")) {
|
|
22411
|
+
const runHook = async () => {
|
|
22412
|
+
try {
|
|
22413
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: resolvedDocId, document });
|
|
22414
|
+
} catch (err) {
|
|
22415
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
22416
|
+
}
|
|
22417
|
+
};
|
|
22418
|
+
await (hold === void 0 ? runHook() : hold.outside(runHook));
|
|
22499
22419
|
}
|
|
22500
|
-
|
|
22501
|
-
const
|
|
22502
|
-
if (vecIndex) {
|
|
22503
|
-
vecIndex.scheduleBuild();
|
|
22504
|
-
}
|
|
22420
|
+
if (ctx.executor.getManager(indexName) === insertManager) {
|
|
22421
|
+
for (const fieldPath of extractedVectors.keys()) insertVecIndexes.get(fieldPath)?.scheduleBuild();
|
|
22505
22422
|
}
|
|
22506
22423
|
ctx.checkWatermark(indexName, resolvedDocId);
|
|
22507
22424
|
ctx.checkHeapPressure(indexName);
|
|
@@ -22587,6 +22504,326 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
22587
22504
|
function bindOptionalWritesHost(_host, _segmentCodec) {
|
|
22588
22505
|
}
|
|
22589
22506
|
|
|
22507
|
+
// src/engine/mutations/batch-admissions.ts
|
|
22508
|
+
function createPendingAdmissions() {
|
|
22509
|
+
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
22510
|
+
}
|
|
22511
|
+
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
22512
|
+
pending.total += 1;
|
|
22513
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
22514
|
+
const partitionId = manager.routePartition(docId);
|
|
22515
|
+
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
22516
|
+
}
|
|
22517
|
+
function asBatchInsertError(err) {
|
|
22518
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
22519
|
+
}
|
|
22520
|
+
|
|
22521
|
+
// src/engine/mutations/outside-hold.ts
|
|
22522
|
+
function runOutside(hold, work) {
|
|
22523
|
+
return hold === void 0 ? work() : hold.outside(work);
|
|
22524
|
+
}
|
|
22525
|
+
|
|
22526
|
+
// src/engine/mutations/write-preparation.ts
|
|
22527
|
+
function settingsOf(entry) {
|
|
22528
|
+
return { required: entry.config.required, embedding: entry.config.embedding, adapter: entry.embeddingAdapter };
|
|
22529
|
+
}
|
|
22530
|
+
function embedsThroughAdapter(entry) {
|
|
22531
|
+
return entry.config.embedding !== void 0 && entry.embeddingAdapter !== null;
|
|
22532
|
+
}
|
|
22533
|
+
function sourcesOf(sources2) {
|
|
22534
|
+
return typeof sources2 === "string" ? [sources2] : sources2;
|
|
22535
|
+
}
|
|
22536
|
+
function sameMapping(left, right) {
|
|
22537
|
+
if (left === right) return true;
|
|
22538
|
+
const targets = Object.keys(left);
|
|
22539
|
+
if (targets.length !== Object.keys(right).length) return false;
|
|
22540
|
+
for (const target of targets) {
|
|
22541
|
+
if (!Object.hasOwn(right, target)) return false;
|
|
22542
|
+
const leftSources = sourcesOf(left[target]);
|
|
22543
|
+
const rightSources = sourcesOf(right[target]);
|
|
22544
|
+
if (leftSources.length !== rightSources.length) return false;
|
|
22545
|
+
if (leftSources.some((source2, position) => source2 !== rightSources[position])) return false;
|
|
22546
|
+
}
|
|
22547
|
+
return true;
|
|
22548
|
+
}
|
|
22549
|
+
function embeddingFits(entry, embedding) {
|
|
22550
|
+
const embeddingConfig = entry.config.embedding;
|
|
22551
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter === null) return embedding === null;
|
|
22552
|
+
return embedding !== null && embedding.adapter === entry.embeddingAdapter && sameMapping(embedding.mapping, embeddingConfig.fields);
|
|
22553
|
+
}
|
|
22554
|
+
function embeddingOfDocuments(embedding, documents2) {
|
|
22555
|
+
if (embedding === null) return null;
|
|
22556
|
+
const fields = /* @__PURE__ */ new Map();
|
|
22557
|
+
for (const document of documents2) {
|
|
22558
|
+
const embedded2 = embedding.fields.get(document);
|
|
22559
|
+
if (embedded2 !== void 0) fields.set(document, embedded2);
|
|
22560
|
+
}
|
|
22561
|
+
return { adapter: embedding.adapter, mapping: embedding.mapping, fields };
|
|
22562
|
+
}
|
|
22563
|
+
function writable(preparations) {
|
|
22564
|
+
return preparations.filter((preparation) => preparation.failure === void 0);
|
|
22565
|
+
}
|
|
22566
|
+
function failAll(preparations, failure) {
|
|
22567
|
+
for (const preparation of writable(preparations)) preparation.failure = failure;
|
|
22568
|
+
}
|
|
22569
|
+
function requireFieldsOf(required, preparations) {
|
|
22570
|
+
if (required === void 0 || required.length === 0) return;
|
|
22571
|
+
for (const preparation of writable(preparations)) {
|
|
22572
|
+
try {
|
|
22573
|
+
validateRequiredFields(preparation.document, required);
|
|
22574
|
+
} catch (err) {
|
|
22575
|
+
preparation.failure = err;
|
|
22576
|
+
}
|
|
22577
|
+
}
|
|
22578
|
+
}
|
|
22579
|
+
async function requireMappedVectors(entry, preparations) {
|
|
22580
|
+
const embeddingConfig = entry.config.embedding;
|
|
22581
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter !== null) return;
|
|
22582
|
+
const { assertDocumentCarriesMappedVectors } = await loadEmbedCode();
|
|
22583
|
+
for (const preparation of writable(preparations)) {
|
|
22584
|
+
try {
|
|
22585
|
+
assertDocumentCarriesMappedVectors(
|
|
22586
|
+
preparation.document,
|
|
22587
|
+
embeddingConfig,
|
|
22588
|
+
entry.embeddingAdapterName
|
|
22589
|
+
);
|
|
22590
|
+
} catch (err) {
|
|
22591
|
+
preparation.failure = err;
|
|
22592
|
+
}
|
|
22593
|
+
}
|
|
22594
|
+
}
|
|
22595
|
+
async function embedWith(settings, signal, preparations, earlier, together) {
|
|
22596
|
+
requireFieldsOf(settings.required, preparations);
|
|
22597
|
+
const { clearWriteEmbedding, embedDocumentsForWrite, lacksMappedVectors } = await loadEmbedCode();
|
|
22598
|
+
if (earlier !== null) clearWriteEmbedding(earlier);
|
|
22599
|
+
const { embedding: embeddingConfig, adapter } = settings;
|
|
22600
|
+
const embeddable = writable(preparations);
|
|
22601
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
22602
|
+
if (embeddingConfig === void 0 || adapter === null || embeddable.length === 0) {
|
|
22603
|
+
return { embedding: null, called: false, documentFailures };
|
|
22604
|
+
}
|
|
22605
|
+
const fields = /* @__PURE__ */ new Map();
|
|
22606
|
+
const embedding = { adapter, mapping: embeddingConfig.fields, fields };
|
|
22607
|
+
const documents2 = embeddable.map((preparation) => preparation.document);
|
|
22608
|
+
if (!lacksMappedVectors(documents2, embeddingConfig)) return { embedding, called: false, documentFailures };
|
|
22609
|
+
for (let chunkStart = 0; chunkStart < embeddable.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22610
|
+
const chunk = embeddable.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
22611
|
+
const embedded2 = await embedDocumentsForWrite(
|
|
22612
|
+
chunk.map((preparation) => preparation.document),
|
|
22613
|
+
embeddingConfig,
|
|
22614
|
+
adapter,
|
|
22615
|
+
signal,
|
|
22616
|
+
together
|
|
22617
|
+
);
|
|
22618
|
+
for (const [document, embeddedFields] of embedded2.embedding.fields) fields.set(document, embeddedFields);
|
|
22619
|
+
for (const [position, error] of embedded2.failed) chunk[position].failure = error;
|
|
22620
|
+
for (const [position, error] of embedded2.documentFailures) documentFailures.set(chunk[position], error);
|
|
22621
|
+
}
|
|
22622
|
+
return { embedding, called: true, documentFailures };
|
|
22623
|
+
}
|
|
22624
|
+
function settleDocumentFailures(entry, pass) {
|
|
22625
|
+
if (!embeddingFits(entry, pass.embedding)) return [...pass.documentFailures.keys()];
|
|
22626
|
+
for (const [preparation, failure] of pass.documentFailures) preparation.failure = failure;
|
|
22627
|
+
return [];
|
|
22628
|
+
}
|
|
22629
|
+
async function runBeforeWrites(preparations) {
|
|
22630
|
+
for (const preparation of preparations) {
|
|
22631
|
+
if (preparation.beforeWrite === null || preparation.failure !== void 0) continue;
|
|
22632
|
+
try {
|
|
22633
|
+
await preparation.beforeWrite();
|
|
22634
|
+
} catch (err) {
|
|
22635
|
+
preparation.failure = err;
|
|
22636
|
+
}
|
|
22637
|
+
}
|
|
22638
|
+
}
|
|
22639
|
+
async function embedThenRunHooks(settings, signal, preparations, earlier) {
|
|
22640
|
+
const pass = await embedWith(settings, signal, preparations, earlier, true);
|
|
22641
|
+
await runBeforeWrites(preparations.filter((preparation) => !pass.documentFailures.has(preparation)));
|
|
22642
|
+
return pass;
|
|
22643
|
+
}
|
|
22644
|
+
async function runHooksThenEmbedEach(ctx, indexName, preparations, earlier) {
|
|
22645
|
+
let settings = null;
|
|
22646
|
+
let embedding = null;
|
|
22647
|
+
let called = false;
|
|
22648
|
+
let uncleared = earlier;
|
|
22649
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
22650
|
+
for (const preparation of preparations) {
|
|
22651
|
+
await runBeforeWrites([preparation]);
|
|
22652
|
+
settings ??= settingsOf(ctx.requireIndex(indexName));
|
|
22653
|
+
const pass = await embedWith(settings, ctx.abortController.signal, [preparation], uncleared, false);
|
|
22654
|
+
uncleared = null;
|
|
22655
|
+
called ||= pass.called;
|
|
22656
|
+
for (const [failed, failure] of pass.documentFailures) documentFailures.set(failed, failure);
|
|
22657
|
+
if (pass.embedding === null) continue;
|
|
22658
|
+
if (embedding === null) embedding = pass.embedding;
|
|
22659
|
+
else for (const [document, fields] of pass.embedding.fields) embedding.fields.set(document, fields);
|
|
22660
|
+
}
|
|
22661
|
+
return { embedding, called, documentFailures };
|
|
22662
|
+
}
|
|
22663
|
+
function adapterChangesRefusal(indexName, embeddings) {
|
|
22664
|
+
return new NarsilError(
|
|
22665
|
+
ErrorCodes.WORKER_BUSY,
|
|
22666
|
+
`Changes to index "${indexName}" replaced its embedding adapter after each of ${embeddings} embeddings of this write`,
|
|
22667
|
+
{ indexName, embeddings }
|
|
22668
|
+
);
|
|
22669
|
+
}
|
|
22670
|
+
async function prepareDocumentWrites(ctx, indexName, preparations, hold, together, earlier = null) {
|
|
22671
|
+
let entry = ctx.requireIndex(indexName);
|
|
22672
|
+
let embedding = earlier;
|
|
22673
|
+
if (writable(preparations).length === 0) return { entry, embedding };
|
|
22674
|
+
const signal = ctx.abortController.signal;
|
|
22675
|
+
let embeddings = 0;
|
|
22676
|
+
let unhooked = [];
|
|
22677
|
+
const hooked = preparations.some((preparation) => preparation.beforeWrite !== null);
|
|
22678
|
+
if (hooked || embedsThroughAdapter(entry) && !embeddingFits(entry, embedding)) {
|
|
22679
|
+
if (together && !embedsThroughAdapter(entry)) {
|
|
22680
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
22681
|
+
await requireMappedVectors(entry, preparations);
|
|
22682
|
+
}
|
|
22683
|
+
const settings = settingsOf(entry);
|
|
22684
|
+
const uncleared = embedding;
|
|
22685
|
+
const pass = await runOutside(
|
|
22686
|
+
hold,
|
|
22687
|
+
() => together ? embedThenRunHooks(settings, signal, preparations, uncleared) : runHooksThenEmbedEach(ctx, indexName, preparations, uncleared)
|
|
22688
|
+
);
|
|
22689
|
+
embedding = pass.embedding;
|
|
22690
|
+
if (pass.called) embeddings += 1;
|
|
22691
|
+
entry = ctx.requireIndex(indexName);
|
|
22692
|
+
unhooked = settleDocumentFailures(entry, pass);
|
|
22693
|
+
}
|
|
22694
|
+
while (writable(preparations).length > 0 && !embeddingFits(entry, embedding)) {
|
|
22695
|
+
if (embeddings >= MAX_CALLS_ACROSS_INDEX_CHANGES) {
|
|
22696
|
+
failAll(preparations, adapterChangesRefusal(indexName, embeddings));
|
|
22697
|
+
break;
|
|
22698
|
+
}
|
|
22699
|
+
const uncleared = embedding;
|
|
22700
|
+
const settings = settingsOf(entry);
|
|
22701
|
+
const hooking = unhooked;
|
|
22702
|
+
const pass = await runOutside(hold, async () => {
|
|
22703
|
+
const embedded2 = await embedWith(settings, signal, preparations, uncleared, together);
|
|
22704
|
+
await runBeforeWrites(hooking.filter((preparation) => !embedded2.documentFailures.has(preparation)));
|
|
22705
|
+
return embedded2;
|
|
22706
|
+
});
|
|
22707
|
+
embedding = pass.embedding;
|
|
22708
|
+
if (pass.called) embeddings += 1;
|
|
22709
|
+
entry = ctx.requireIndex(indexName);
|
|
22710
|
+
const stale = settleDocumentFailures(entry, pass);
|
|
22711
|
+
unhooked = hooking.filter((preparation) => stale.includes(preparation));
|
|
22712
|
+
}
|
|
22713
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
22714
|
+
await requireMappedVectors(entry, preparations);
|
|
22715
|
+
return { entry, embedding };
|
|
22716
|
+
}
|
|
22717
|
+
async function prepareDocumentWrite(ctx, indexName, document, hold, beforeWrite) {
|
|
22718
|
+
const preparation = { document, beforeWrite, failure: void 0 };
|
|
22719
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
22720
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
22721
|
+
return entry;
|
|
22722
|
+
}
|
|
22723
|
+
|
|
22724
|
+
// src/engine/mutations/insert-batch-admission.ts
|
|
22725
|
+
function pendingInsertOf(ctx, indexName, docId, document, hooked) {
|
|
22726
|
+
const pending = {
|
|
22727
|
+
docId,
|
|
22728
|
+
preparation: { document, beforeWrite: null, failure: void 0 },
|
|
22729
|
+
refusedByHook: false
|
|
22730
|
+
};
|
|
22731
|
+
if (hooked && ctx.pluginRegistry.hasHooks("beforeInsert")) {
|
|
22732
|
+
pending.preparation.beforeWrite = async () => {
|
|
22733
|
+
try {
|
|
22734
|
+
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document });
|
|
22735
|
+
} catch (err) {
|
|
22736
|
+
pending.refusedByHook = true;
|
|
22737
|
+
throw err;
|
|
22738
|
+
}
|
|
22739
|
+
};
|
|
22740
|
+
}
|
|
22741
|
+
return pending;
|
|
22742
|
+
}
|
|
22743
|
+
async function prepareInsertDocuments(ctx, indexName, documents2, start, end, hold, hooked, failed, earlier = null) {
|
|
22744
|
+
const pending = [];
|
|
22745
|
+
for (let i = start; i < end; i++) {
|
|
22746
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
22747
|
+
if (ctx.abortController.signal.aborted) break;
|
|
22748
|
+
const document = documents2[i];
|
|
22749
|
+
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
22750
|
+
try {
|
|
22751
|
+
const sizing = requireDocumentObject(document);
|
|
22752
|
+
if (sizing !== null) await sizing;
|
|
22753
|
+
validateDocId(docId);
|
|
22754
|
+
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
22755
|
+
} catch (err) {
|
|
22756
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
22757
|
+
}
|
|
22758
|
+
}
|
|
22759
|
+
const { entry, embedding } = await prepareDocumentWrites(
|
|
22760
|
+
ctx,
|
|
22761
|
+
indexName,
|
|
22762
|
+
pending.map((item) => item.preparation),
|
|
22763
|
+
hold,
|
|
22764
|
+
true,
|
|
22765
|
+
earlier
|
|
22766
|
+
);
|
|
22767
|
+
const candidates2 = [];
|
|
22768
|
+
for (const item of pending) {
|
|
22769
|
+
const { document, failure } = item.preparation;
|
|
22770
|
+
if (failure === void 0) {
|
|
22771
|
+
candidates2.push({ docId: item.docId, document });
|
|
22772
|
+
continue;
|
|
22773
|
+
}
|
|
22774
|
+
const docId = item.refusedByHook ? item.docId : providedDocId(document) ?? "";
|
|
22775
|
+
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
22776
|
+
}
|
|
22777
|
+
return { entry, candidates: candidates2, embedding };
|
|
22778
|
+
}
|
|
22779
|
+
async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
22780
|
+
const { entry, candidates: candidates2, embedding } = await prepareInsertDocuments(
|
|
22781
|
+
ctx,
|
|
22782
|
+
indexName,
|
|
22783
|
+
documents2,
|
|
22784
|
+
0,
|
|
22785
|
+
documents2.length,
|
|
22786
|
+
hold,
|
|
22787
|
+
true,
|
|
22788
|
+
failed
|
|
22789
|
+
);
|
|
22790
|
+
const manager = ctx.requireManager(indexName);
|
|
22791
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
22792
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
22793
|
+
const strict = entry.config.strict === true;
|
|
22794
|
+
const admitted = [];
|
|
22795
|
+
const admittedIds = /* @__PURE__ */ new Set();
|
|
22796
|
+
const pending = createPendingAdmissions();
|
|
22797
|
+
for (let position = 0; position < candidates2.length; position++) {
|
|
22798
|
+
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
22799
|
+
if (ctx.abortController.signal.aborted) break;
|
|
22800
|
+
const { docId, document } = candidates2[position];
|
|
22801
|
+
try {
|
|
22802
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
22803
|
+
document,
|
|
22804
|
+
vectorFieldPaths
|
|
22805
|
+
);
|
|
22806
|
+
if (extractedVectors.size > 0) {
|
|
22807
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
22808
|
+
}
|
|
22809
|
+
admitInsert(ctx, indexName, manager, docId, pending);
|
|
22810
|
+
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
22811
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
22812
|
+
}
|
|
22813
|
+
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
22814
|
+
if (strict) {
|
|
22815
|
+
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
22816
|
+
}
|
|
22817
|
+
admittedIds.add(docId);
|
|
22818
|
+
recordPendingAdmission(ctx, pending, manager, docId);
|
|
22819
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
22820
|
+
} catch (err) {
|
|
22821
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
22822
|
+
}
|
|
22823
|
+
}
|
|
22824
|
+
return { admitted, embedding };
|
|
22825
|
+
}
|
|
22826
|
+
|
|
22590
22827
|
// src/engine/mutations/durable-rollback.ts
|
|
22591
22828
|
function holdVectorsForRollback(indexes, docId) {
|
|
22592
22829
|
const releases = [];
|
|
@@ -22822,84 +23059,29 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
22822
23059
|
return applications;
|
|
22823
23060
|
}
|
|
22824
23061
|
|
|
22825
|
-
// src/engine/mutations/insert-batch-
|
|
22826
|
-
async function
|
|
22827
|
-
|
|
22828
|
-
const manager = ctx.
|
|
22829
|
-
|
|
22830
|
-
|
|
22831
|
-
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
22832
|
-
const strict = entry.config.strict === true;
|
|
22833
|
-
const required = entry.config.required;
|
|
22834
|
-
const admitted = [];
|
|
22835
|
-
const admittedIds = /* @__PURE__ */ new Set();
|
|
22836
|
-
const pending = createPendingAdmissions();
|
|
22837
|
-
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22838
|
-
if (ctx.abortController.signal.aborted) break;
|
|
22839
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
22840
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
22841
|
-
if (required && required.length > 0) {
|
|
22842
|
-
collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
22843
|
-
}
|
|
22844
|
-
await embedChunkDocuments(
|
|
22845
|
-
entry,
|
|
22846
|
-
documents2,
|
|
22847
|
-
chunkStart,
|
|
22848
|
-
chunkEnd,
|
|
22849
|
-
ctx.abortController.signal,
|
|
22850
|
-
chunkFailedIndexes,
|
|
22851
|
-
failed
|
|
22852
|
-
);
|
|
22853
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22854
|
-
if (ctx.abortController.signal.aborted) break;
|
|
22855
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
22856
|
-
const docId = providedDocId(documents2[i]) ?? ctx.idGenerator();
|
|
23062
|
+
// src/engine/mutations/insert-batch-documents.ts
|
|
23063
|
+
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
23064
|
+
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
23065
|
+
const manager = ctx.executor.getManager(indexName);
|
|
23066
|
+
await runOutside(hold, async () => {
|
|
23067
|
+
for (const doc of inserted) {
|
|
22857
23068
|
try {
|
|
22858
|
-
|
|
22859
|
-
if (sizing !== null) await sizing;
|
|
22860
|
-
validateDocId(docId);
|
|
22861
|
-
if (hasBeforeHook) {
|
|
22862
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents2[i] });
|
|
22863
|
-
}
|
|
22864
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
22865
|
-
documents2[i],
|
|
22866
|
-
vectorFieldPaths
|
|
22867
|
-
);
|
|
22868
|
-
if (extractedVectors.size > 0) {
|
|
22869
|
-
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
22870
|
-
}
|
|
22871
|
-
admitInsert(ctx, indexName, manager, docId, pending);
|
|
22872
|
-
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
22873
|
-
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
22874
|
-
}
|
|
22875
|
-
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
22876
|
-
if (strict) {
|
|
22877
|
-
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
22878
|
-
}
|
|
22879
|
-
admittedIds.add(docId);
|
|
22880
|
-
recordPendingAdmission(ctx, pending, manager, docId);
|
|
22881
|
-
admitted.push({ docId, document: documents2[i], partitionDoc, extractedVectors });
|
|
23069
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
22882
23070
|
} catch (err) {
|
|
22883
|
-
|
|
23071
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
22884
23072
|
}
|
|
22885
23073
|
}
|
|
22886
|
-
|
|
22887
|
-
|
|
22888
|
-
}
|
|
22889
|
-
}
|
|
22890
|
-
return admitted;
|
|
23074
|
+
});
|
|
23075
|
+
return ctx.executor.getManager(indexName) !== manager;
|
|
22891
23076
|
}
|
|
22892
|
-
|
|
22893
|
-
// src/engine/mutations/insert-batch-documents.ts
|
|
22894
|
-
async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed) {
|
|
22895
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
23077
|
+
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
22896
23078
|
const succeeded = [];
|
|
22897
|
-
const buffered = /* @__PURE__ */ new Set();
|
|
22898
23079
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
22899
23080
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22900
23081
|
if (ctx.abortController.signal.aborted) break;
|
|
22901
23082
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
22902
23083
|
const applications = await applyInsertChunk(ctx, indexName, chunk, options);
|
|
23084
|
+
const inserted = [];
|
|
22903
23085
|
for (let i = 0; i < chunk.length; i++) {
|
|
22904
23086
|
const doc = chunk[i];
|
|
22905
23087
|
const application = applications[i];
|
|
@@ -22908,30 +23090,103 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
|
|
|
22908
23090
|
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
22909
23091
|
continue;
|
|
22910
23092
|
}
|
|
22911
|
-
|
|
22912
|
-
|
|
22913
|
-
succeeded.push(doc.docId);
|
|
22914
|
-
continue;
|
|
22915
|
-
}
|
|
23093
|
+
succeeded.push(doc.docId);
|
|
23094
|
+
if (application.status === "buffered") continue;
|
|
22916
23095
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
22917
23096
|
touchedVectorFields.add(fieldPath);
|
|
22918
23097
|
}
|
|
22919
|
-
|
|
22920
|
-
try {
|
|
22921
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
22922
|
-
} catch (err) {
|
|
22923
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
22924
|
-
}
|
|
22925
|
-
}
|
|
22926
|
-
succeeded.push(doc.docId);
|
|
23098
|
+
inserted.push(doc);
|
|
22927
23099
|
}
|
|
22928
23100
|
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
23101
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
23102
|
+
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
23103
|
+
}
|
|
22929
23104
|
if (applied < admitted.length) {
|
|
22930
23105
|
ctx.checkHeapPressure(indexName);
|
|
22931
23106
|
await new Promise((r) => setTimeout(r, 0));
|
|
22932
23107
|
}
|
|
22933
23108
|
}
|
|
22934
|
-
return { succeeded,
|
|
23109
|
+
return { succeeded, touchedVectorFields, unrecorded: [] };
|
|
23110
|
+
}
|
|
23111
|
+
|
|
23112
|
+
// src/engine/mutations/insert-batch-chunks.ts
|
|
23113
|
+
async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
23114
|
+
const { entry, candidates: candidates2 } = await prepareInsertDocuments(
|
|
23115
|
+
ctx,
|
|
23116
|
+
indexName,
|
|
23117
|
+
documents2,
|
|
23118
|
+
chunkStart,
|
|
23119
|
+
chunkEnd,
|
|
23120
|
+
hold,
|
|
23121
|
+
hooked,
|
|
23122
|
+
failed,
|
|
23123
|
+
earlier
|
|
23124
|
+
);
|
|
23125
|
+
const manager = ctx.requireManager(indexName);
|
|
23126
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
23127
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
23128
|
+
const admitted = [];
|
|
23129
|
+
for (const { docId, document } of candidates2) {
|
|
23130
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23131
|
+
if (ctx.abortController.signal.aborted) break;
|
|
23132
|
+
try {
|
|
23133
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
23134
|
+
document,
|
|
23135
|
+
vectorFieldPaths
|
|
23136
|
+
);
|
|
23137
|
+
if (extractedVectors.size > 0) {
|
|
23138
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
23139
|
+
}
|
|
23140
|
+
admitInsert(ctx, indexName, manager, docId);
|
|
23141
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
23142
|
+
} catch (err) {
|
|
23143
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
23144
|
+
}
|
|
23145
|
+
}
|
|
23146
|
+
return admitted;
|
|
23147
|
+
}
|
|
23148
|
+
async function insertInChunks(ctx, indexName, documents2, options, hold, hooked, failed, embedded2 = null) {
|
|
23149
|
+
const succeeded = [];
|
|
23150
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23151
|
+
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23152
|
+
if (ctx.abortController.signal.aborted) break;
|
|
23153
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
23154
|
+
const earlier = embeddingOfDocuments(embedded2, documents2.slice(chunkStart, chunkEnd));
|
|
23155
|
+
const prepared = await admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
23156
|
+
if (prepared.length > 0) {
|
|
23157
|
+
const applications = await applyInsertChunk(ctx, indexName, prepared, options);
|
|
23158
|
+
const inserted = [];
|
|
23159
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
23160
|
+
const doc = prepared[i];
|
|
23161
|
+
const application = applications[i];
|
|
23162
|
+
if (application.status === "skipped") continue;
|
|
23163
|
+
if (application.status === "failed") {
|
|
23164
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
23165
|
+
continue;
|
|
23166
|
+
}
|
|
23167
|
+
succeeded.push(doc.docId);
|
|
23168
|
+
if (application.status === "buffered") continue;
|
|
23169
|
+
for (const fieldPath of doc.extractedVectors.keys()) touchedVectorFields.add(fieldPath);
|
|
23170
|
+
inserted.push(doc);
|
|
23171
|
+
}
|
|
23172
|
+
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
23173
|
+
}
|
|
23174
|
+
if (chunkEnd < documents2.length) {
|
|
23175
|
+
ctx.checkHeapPressure(indexName);
|
|
23176
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
23177
|
+
}
|
|
23178
|
+
}
|
|
23179
|
+
return { succeeded, touchedVectorFields };
|
|
23180
|
+
}
|
|
23181
|
+
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
23182
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
23183
|
+
for (const fieldPath of inserts.touchedVectorFields) {
|
|
23184
|
+
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
23185
|
+
}
|
|
23186
|
+
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
23187
|
+
ctx.checkHeapPressure(indexName);
|
|
23188
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
23189
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22935
23190
|
}
|
|
22936
23191
|
|
|
22937
23192
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -23036,10 +23291,6 @@ function mainStoreDocument(doc, options) {
|
|
|
23036
23291
|
if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
|
|
23037
23292
|
return structuredClone(doc.partitionDoc);
|
|
23038
23293
|
}
|
|
23039
|
-
async function applyIndividually(ctx, indexName, admitted, options, failed) {
|
|
23040
|
-
const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
|
|
23041
|
-
return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
|
|
23042
|
-
}
|
|
23043
23294
|
function chunksOfWholeSegments(indexName, partitionCount, built2, frozen, memberIndexes, admitted) {
|
|
23044
23295
|
const chunks = [];
|
|
23045
23296
|
let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
@@ -23089,12 +23340,12 @@ function chunksOfWholeSegments(indexName, partitionCount, built2, frozen, member
|
|
|
23089
23340
|
if (chunk.docs.length > 0) chunks.push(chunk);
|
|
23090
23341
|
return chunks;
|
|
23091
23342
|
}
|
|
23092
|
-
async function recordMergedDocuments(ctx, indexName, partitionCount, built2, frozen, memberIndexes, admitted, options, failed) {
|
|
23093
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
23343
|
+
async function recordMergedDocuments(ctx, indexName, partitionCount, built2, frozen, memberIndexes, admitted, options, hold, failed) {
|
|
23094
23344
|
const succeededIds = /* @__PURE__ */ new Set();
|
|
23095
23345
|
const errorOf = /* @__PURE__ */ new Map();
|
|
23096
23346
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23097
23347
|
const chunks = chunksOfWholeSegments(indexName, partitionCount, built2, frozen, memberIndexes, admitted);
|
|
23348
|
+
const unrecorded = [];
|
|
23098
23349
|
for (let index = 0; index < chunks.length; index++) {
|
|
23099
23350
|
const { docs, attachmentOf } = chunks[index];
|
|
23100
23351
|
const applies = docs.map((doc) => {
|
|
@@ -23104,6 +23355,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23104
23355
|
});
|
|
23105
23356
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
23106
23357
|
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
23358
|
+
const inserted = [];
|
|
23107
23359
|
for (let i = 0; i < docs.length; i++) {
|
|
23108
23360
|
const doc = docs[i];
|
|
23109
23361
|
const failure = failures[i];
|
|
@@ -23135,15 +23387,13 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23135
23387
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
23136
23388
|
touchedVectorFields.add(fieldPath);
|
|
23137
23389
|
}
|
|
23138
|
-
|
|
23139
|
-
try {
|
|
23140
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
23141
|
-
} catch (err) {
|
|
23142
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23143
|
-
}
|
|
23144
|
-
}
|
|
23390
|
+
inserted.push(doc);
|
|
23145
23391
|
succeededIds.add(doc.docId);
|
|
23146
23392
|
}
|
|
23393
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
23394
|
+
for (const later of chunks.slice(index + 1)) unrecorded.push(...later.docs);
|
|
23395
|
+
break;
|
|
23396
|
+
}
|
|
23147
23397
|
if (index + 1 < chunks.length) {
|
|
23148
23398
|
ctx.checkHeapPressure(indexName);
|
|
23149
23399
|
await new Promise((r) => setTimeout(r, 0));
|
|
@@ -23158,7 +23408,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
23158
23408
|
if (!errorOf.has(doc.docId)) continue;
|
|
23159
23409
|
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
23160
23410
|
}
|
|
23161
|
-
return { succeeded, touchedVectorFields };
|
|
23411
|
+
return { succeeded, touchedVectorFields, unrecorded };
|
|
23162
23412
|
}
|
|
23163
23413
|
function frozenSegmentOf(segment, members, options) {
|
|
23164
23414
|
if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
|
|
@@ -23169,14 +23419,15 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
23169
23419
|
segment.segmentId
|
|
23170
23420
|
);
|
|
23171
23421
|
}
|
|
23172
|
-
async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
23422
|
+
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
23173
23423
|
if (admitted.length === 0) {
|
|
23174
|
-
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set() };
|
|
23424
|
+
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] };
|
|
23175
23425
|
}
|
|
23426
|
+
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
23176
23427
|
const manager = ctx.requireManager(indexName);
|
|
23177
23428
|
const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
|
|
23178
23429
|
if (admitted.length < MIN_DOCUMENTS_FOR_SEGMENTS || workers <= 0) {
|
|
23179
|
-
return applyIndividually(
|
|
23430
|
+
return applyIndividually();
|
|
23180
23431
|
}
|
|
23181
23432
|
const docIds = admitted.map((doc) => doc.docId);
|
|
23182
23433
|
const partitionDocuments = admitted.map((doc) => doc.partitionDoc);
|
|
@@ -23189,7 +23440,7 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
23189
23440
|
built2 = null;
|
|
23190
23441
|
}
|
|
23191
23442
|
if (built2 === null) {
|
|
23192
|
-
return applyIndividually(
|
|
23443
|
+
return applyIndividually();
|
|
23193
23444
|
}
|
|
23194
23445
|
const positions2 = new Map(admitted.map((document, index) => [document.docId, index]));
|
|
23195
23446
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -23198,16 +23449,15 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
23198
23449
|
const members = [];
|
|
23199
23450
|
for (const docId of segment.docIds) {
|
|
23200
23451
|
const position = positions2.get(docId);
|
|
23201
|
-
if (position === void 0 || used.has(position))
|
|
23202
|
-
return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
23452
|
+
if (position === void 0 || used.has(position)) return applyIndividually();
|
|
23203
23453
|
used.add(position);
|
|
23204
23454
|
members.push(position);
|
|
23205
23455
|
}
|
|
23206
23456
|
memberIndexes.push(members);
|
|
23207
23457
|
}
|
|
23208
|
-
if (used.size !== admitted.length) return applyIndividually(
|
|
23458
|
+
if (used.size !== admitted.length) return applyIndividually();
|
|
23209
23459
|
if (ctx.isRebalancing(indexName)) {
|
|
23210
|
-
return applyIndividually(
|
|
23460
|
+
return applyIndividually();
|
|
23211
23461
|
}
|
|
23212
23462
|
const frozen = [];
|
|
23213
23463
|
for (let i = 0; i < built2.length; i++) {
|
|
@@ -23216,149 +23466,65 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
23216
23466
|
memberIndexes[i].map((m) => admitted[m]),
|
|
23217
23467
|
options
|
|
23218
23468
|
);
|
|
23219
|
-
if (segment === null) return applyIndividually(
|
|
23469
|
+
if (segment === null) return applyIndividually();
|
|
23220
23470
|
frozen.push(segment);
|
|
23221
23471
|
}
|
|
23222
23472
|
for (const doc of admitted) {
|
|
23223
|
-
if (manager.has(doc.docId)) return applyIndividually(
|
|
23473
|
+
if (manager.has(doc.docId)) return applyIndividually();
|
|
23224
23474
|
}
|
|
23225
|
-
return recordMergedDocuments(
|
|
23475
|
+
return recordMergedDocuments(
|
|
23476
|
+
ctx,
|
|
23477
|
+
indexName,
|
|
23478
|
+
partitionCount,
|
|
23479
|
+
built2,
|
|
23480
|
+
frozen,
|
|
23481
|
+
memberIndexes,
|
|
23482
|
+
admitted,
|
|
23483
|
+
options,
|
|
23484
|
+
hold,
|
|
23485
|
+
failed
|
|
23486
|
+
);
|
|
23226
23487
|
}
|
|
23227
|
-
async function insertBatchViaSegments(ctx, indexName, documents2, options) {
|
|
23488
|
+
async function insertBatchViaSegments(ctx, indexName, documents2, options, hold) {
|
|
23228
23489
|
const failed = [];
|
|
23229
|
-
const admitted = await admitBatchDocuments(ctx, indexName, documents2, failed);
|
|
23230
|
-
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, failed);
|
|
23231
|
-
|
|
23232
|
-
|
|
23233
|
-
|
|
23234
|
-
|
|
23235
|
-
|
|
23236
|
-
|
|
23237
|
-
await ctx
|
|
23238
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23490
|
+
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents2, failed, hold);
|
|
23491
|
+
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
23492
|
+
if (outcome.unrecorded.length > 0) {
|
|
23493
|
+
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
23494
|
+
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
23495
|
+
outcome.succeeded.push(...rest.succeeded);
|
|
23496
|
+
for (const fieldPath of rest.touchedVectorFields) outcome.touchedVectorFields.add(fieldPath);
|
|
23497
|
+
}
|
|
23498
|
+
await finishInsertBatch(ctx, indexName, outcome, options);
|
|
23239
23499
|
return { succeeded: outcome.succeeded, failed };
|
|
23240
23500
|
}
|
|
23241
23501
|
|
|
23242
23502
|
// src/engine/mutations/insert-batch.ts
|
|
23243
|
-
async function insertDocumentBatch(ctx, indexName, documents2, options) {
|
|
23503
|
+
async function insertDocumentBatch(ctx, indexName, documents2, options, hold) {
|
|
23244
23504
|
ctx.guardShutdown();
|
|
23245
|
-
|
|
23505
|
+
ctx.requireIndex(indexName);
|
|
23246
23506
|
validateBatchList(documents2, "documents");
|
|
23247
23507
|
await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents2.length);
|
|
23248
23508
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
23249
23509
|
if (documents2.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
23250
|
-
return insertBatchViaSegments(ctx, indexName, documents2, options);
|
|
23510
|
+
return insertBatchViaSegments(ctx, indexName, documents2, options, hold);
|
|
23251
23511
|
}
|
|
23252
|
-
const succeeded = [];
|
|
23253
23512
|
const failed = [];
|
|
23254
|
-
const
|
|
23255
|
-
|
|
23256
|
-
|
|
23257
|
-
const batchManager = ctx.requireManager(indexName);
|
|
23258
|
-
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
23259
|
-
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
23260
|
-
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23261
|
-
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23262
|
-
if (ctx.abortController.signal.aborted) break;
|
|
23263
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
23264
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
23265
|
-
if (required && required.length > 0) {
|
|
23266
|
-
collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
23267
|
-
}
|
|
23268
|
-
await embedChunkDocuments(
|
|
23269
|
-
entry,
|
|
23270
|
-
documents2,
|
|
23271
|
-
chunkStart,
|
|
23272
|
-
chunkEnd,
|
|
23273
|
-
ctx.abortController.signal,
|
|
23274
|
-
chunkFailedIndexes,
|
|
23275
|
-
failed
|
|
23276
|
-
);
|
|
23277
|
-
async function applyPrepared(prepared2) {
|
|
23278
|
-
const applications = await applyInsertChunk(ctx, indexName, prepared2, options);
|
|
23279
|
-
for (let i = 0; i < prepared2.length; i++) {
|
|
23280
|
-
const doc = prepared2[i];
|
|
23281
|
-
const application = applications[i];
|
|
23282
|
-
if (application.status === "skipped") continue;
|
|
23283
|
-
if (application.status === "failed") {
|
|
23284
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
23285
|
-
continue;
|
|
23286
|
-
}
|
|
23287
|
-
if (application.status === "buffered") {
|
|
23288
|
-
succeeded.push(doc.docId);
|
|
23289
|
-
continue;
|
|
23290
|
-
}
|
|
23291
|
-
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
23292
|
-
touchedVectorFields.add(fieldPath);
|
|
23293
|
-
}
|
|
23294
|
-
if (hasAfterHook) {
|
|
23295
|
-
try {
|
|
23296
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
23297
|
-
} catch (err) {
|
|
23298
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23299
|
-
}
|
|
23300
|
-
}
|
|
23301
|
-
succeeded.push(doc.docId);
|
|
23302
|
-
}
|
|
23303
|
-
}
|
|
23304
|
-
const prepared = [];
|
|
23305
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23306
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23307
|
-
if (ctx.abortController.signal.aborted) break;
|
|
23308
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
23309
|
-
const batchDocId = providedDocId(documents2[i]) ?? ctx.idGenerator();
|
|
23310
|
-
try {
|
|
23311
|
-
const sizing = requireDocumentObject(documents2[i]);
|
|
23312
|
-
if (sizing !== null) await sizing;
|
|
23313
|
-
validateDocId(batchDocId);
|
|
23314
|
-
if (hasBeforeHook) {
|
|
23315
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents2[i] });
|
|
23316
|
-
}
|
|
23317
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
23318
|
-
documents2[i],
|
|
23319
|
-
batchVectorFieldPaths
|
|
23320
|
-
);
|
|
23321
|
-
if (extractedVectors.size > 0) {
|
|
23322
|
-
validateVectorDimensions(extractedVectors, batchVecIndexes);
|
|
23323
|
-
}
|
|
23324
|
-
admitInsert(ctx, indexName, batchManager, batchDocId);
|
|
23325
|
-
prepared.push({
|
|
23326
|
-
docId: batchDocId,
|
|
23327
|
-
document: documents2[i],
|
|
23328
|
-
partitionDoc,
|
|
23329
|
-
extractedVectors
|
|
23330
|
-
});
|
|
23331
|
-
} catch (err) {
|
|
23332
|
-
failed.push({ docId: batchDocId, error: asBatchInsertError(err) });
|
|
23333
|
-
}
|
|
23334
|
-
}
|
|
23335
|
-
if (prepared.length > 0) await applyPrepared(prepared);
|
|
23336
|
-
if (chunkEnd < documents2.length) {
|
|
23337
|
-
ctx.checkHeapPressure(indexName);
|
|
23338
|
-
await new Promise((r) => setTimeout(r, 0));
|
|
23339
|
-
}
|
|
23340
|
-
}
|
|
23341
|
-
for (const fieldPath of touchedVectorFields) {
|
|
23342
|
-
const vecIndex = batchVecIndexes.get(fieldPath);
|
|
23343
|
-
if (vecIndex) {
|
|
23344
|
-
vecIndex.scheduleBuild();
|
|
23345
|
-
}
|
|
23346
|
-
}
|
|
23347
|
-
ctx.checkWatermark(indexName, succeeded);
|
|
23348
|
-
ctx.checkHeapPressure(indexName);
|
|
23349
|
-
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
23350
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23351
|
-
return { succeeded, failed };
|
|
23513
|
+
const inserts = await insertInChunks(ctx, indexName, documents2, options, hold, true, failed);
|
|
23514
|
+
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
23515
|
+
return { succeeded: inserts.succeeded, failed };
|
|
23352
23516
|
}
|
|
23353
23517
|
|
|
23354
23518
|
// src/engine/mutations/remove.ts
|
|
23355
23519
|
function asRemoveError(err) {
|
|
23356
23520
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
23357
23521
|
}
|
|
23358
|
-
|
|
23522
|
+
function validateRemoval(ctx, docId) {
|
|
23359
23523
|
ctx.guardShutdown();
|
|
23360
23524
|
validateDocId(docId);
|
|
23361
|
-
|
|
23525
|
+
}
|
|
23526
|
+
function prepareRemoval(ctx, indexName, docId) {
|
|
23527
|
+
ctx.guardShutdown();
|
|
23362
23528
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
23363
23529
|
if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
|
|
23364
23530
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
@@ -23408,20 +23574,31 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
23408
23574
|
};
|
|
23409
23575
|
return () => manager.publication.writeNow(indexChangeOutsideExternalWaits(removeFromIndex));
|
|
23410
23576
|
}
|
|
23411
|
-
async function
|
|
23577
|
+
async function settleRemovals(ctx, indexName, docIds, hold) {
|
|
23578
|
+
if (docIds.length === 0) return;
|
|
23412
23579
|
const settling = ctx.orchestrator.settlePublications(indexName, "none");
|
|
23413
23580
|
if (settling !== void 0) await settling;
|
|
23414
|
-
|
|
23415
|
-
|
|
23416
|
-
|
|
23417
|
-
|
|
23418
|
-
|
|
23419
|
-
|
|
23581
|
+
if (!ctx.pluginRegistry.hasHooks("afterRemove")) return;
|
|
23582
|
+
await runOutside(hold, async () => {
|
|
23583
|
+
for (const docId of docIds) {
|
|
23584
|
+
try {
|
|
23585
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
23586
|
+
} catch (err) {
|
|
23587
|
+
console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23588
|
+
}
|
|
23589
|
+
}
|
|
23590
|
+
});
|
|
23420
23591
|
}
|
|
23421
|
-
async function removeDocument3(ctx, indexName, docId, options) {
|
|
23592
|
+
async function removeDocument3(ctx, indexName, docId, options, hold) {
|
|
23422
23593
|
ctx.guardShutdown();
|
|
23423
23594
|
ctx.requireIndex(indexName);
|
|
23424
|
-
|
|
23595
|
+
validateRemoval(ctx, docId);
|
|
23596
|
+
if (ctx.pluginRegistry.hasHooks("beforeRemove")) {
|
|
23597
|
+
await runOutside(hold, async () => {
|
|
23598
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
|
|
23599
|
+
});
|
|
23600
|
+
}
|
|
23601
|
+
const prepared = prepareRemoval(ctx, indexName, docId);
|
|
23425
23602
|
const apply = applyOfRemoval(ctx, indexName, prepared);
|
|
23426
23603
|
if (ctx.durability) {
|
|
23427
23604
|
try {
|
|
@@ -23435,25 +23612,52 @@ async function removeDocument3(ctx, indexName, docId, options) {
|
|
|
23435
23612
|
} else {
|
|
23436
23613
|
await apply();
|
|
23437
23614
|
}
|
|
23438
|
-
if (!prepared.buffered) await
|
|
23615
|
+
if (!prepared.buffered) await settleRemovals(ctx, indexName, [docId], hold);
|
|
23439
23616
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23440
23617
|
}
|
|
23441
|
-
async function
|
|
23618
|
+
async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold) {
|
|
23619
|
+
const refused = /* @__PURE__ */ new Map();
|
|
23620
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23621
|
+
try {
|
|
23622
|
+
validateRemoval(ctx, docIds[i]);
|
|
23623
|
+
} catch (err) {
|
|
23624
|
+
refused.set(i, err);
|
|
23625
|
+
}
|
|
23626
|
+
}
|
|
23627
|
+
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
23628
|
+
await runOutside(hold, async () => {
|
|
23629
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23630
|
+
if (refused.has(i)) continue;
|
|
23631
|
+
try {
|
|
23632
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
23633
|
+
} catch (err) {
|
|
23634
|
+
refused.set(i, err);
|
|
23635
|
+
}
|
|
23636
|
+
}
|
|
23637
|
+
});
|
|
23638
|
+
return refused;
|
|
23639
|
+
}
|
|
23640
|
+
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
23442
23641
|
ctx.guardShutdown();
|
|
23443
23642
|
ctx.requireIndex(indexName);
|
|
23444
23643
|
validateBatchList(docIds, "document IDs");
|
|
23445
23644
|
const succeeded = [];
|
|
23446
23645
|
const failed = [];
|
|
23447
|
-
const
|
|
23448
|
-
await
|
|
23646
|
+
const first = ctx.requireManager(indexName);
|
|
23647
|
+
await first.publication.write(() => first.beginBatchRemove());
|
|
23648
|
+
let manager = first;
|
|
23449
23649
|
try {
|
|
23450
23650
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23451
23651
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
23652
|
+
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
23653
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
23654
|
+
const removed = [];
|
|
23452
23655
|
const prepared = [];
|
|
23453
23656
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23454
23657
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23455
23658
|
try {
|
|
23456
|
-
|
|
23659
|
+
if (refused.has(i)) throw refused.get(i);
|
|
23660
|
+
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
23457
23661
|
} catch (err) {
|
|
23458
23662
|
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
23459
23663
|
}
|
|
@@ -23483,23 +23687,33 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
23483
23687
|
continue;
|
|
23484
23688
|
}
|
|
23485
23689
|
removal.releaseVectorHolds?.();
|
|
23486
|
-
if (!removal.buffered)
|
|
23690
|
+
if (!removal.buffered) removed.push(removal.docId);
|
|
23487
23691
|
succeeded.push(removal.docId);
|
|
23488
23692
|
}
|
|
23489
23693
|
} finally {
|
|
23490
23694
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
23491
23695
|
}
|
|
23696
|
+
await settleRemovals(ctx, indexName, removed, hold);
|
|
23697
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
23492
23698
|
if (chunkEnd < docIds.length) {
|
|
23493
23699
|
await new Promise((r) => setTimeout(r, 0));
|
|
23494
23700
|
}
|
|
23495
23701
|
}
|
|
23496
23702
|
} finally {
|
|
23497
|
-
|
|
23703
|
+
const ending = manager;
|
|
23704
|
+
await ending?.publication.write(() => ending.endBatchRemove());
|
|
23498
23705
|
}
|
|
23499
23706
|
ctx.checkHeapPressure(indexName);
|
|
23500
23707
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23501
23708
|
return { succeeded, failed };
|
|
23502
23709
|
}
|
|
23710
|
+
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
23711
|
+
const current = ctx.executor.getManager(indexName);
|
|
23712
|
+
if (current === batchManager) return batchManager;
|
|
23713
|
+
await batchManager?.publication.write(() => batchManager.endBatchRemove());
|
|
23714
|
+
await current?.publication.write(() => current.beginBatchRemove());
|
|
23715
|
+
return current;
|
|
23716
|
+
}
|
|
23503
23717
|
|
|
23504
23718
|
// src/engine/mutations/update.ts
|
|
23505
23719
|
function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
@@ -23533,59 +23747,47 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
23533
23747
|
function asUpdateError(err) {
|
|
23534
23748
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
23535
23749
|
}
|
|
23536
|
-
async function
|
|
23750
|
+
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
23751
|
+
for (const update of updates) {
|
|
23752
|
+
try {
|
|
23753
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
23754
|
+
indexName,
|
|
23755
|
+
docId: update.docId,
|
|
23756
|
+
oldDocument: update.oldDocument ?? {},
|
|
23757
|
+
newDocument: update.document
|
|
23758
|
+
});
|
|
23759
|
+
} catch (err) {
|
|
23760
|
+
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23761
|
+
}
|
|
23762
|
+
}
|
|
23763
|
+
}
|
|
23764
|
+
async function settleUpdates(ctx, indexName, updates, hold) {
|
|
23765
|
+
if (updates.length === 0) return;
|
|
23537
23766
|
await ctx.orchestrator.settlePublications(indexName);
|
|
23538
|
-
|
|
23539
|
-
await
|
|
23540
|
-
indexName,
|
|
23541
|
-
docId: prepared.docId,
|
|
23542
|
-
oldDocument: prepared.oldDocument ?? {},
|
|
23543
|
-
newDocument: prepared.document
|
|
23544
|
-
});
|
|
23545
|
-
} catch (err) {
|
|
23546
|
-
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
23767
|
+
if (ctx.pluginRegistry.hasHooks("afterUpdate")) {
|
|
23768
|
+
await runOutside(hold, () => runAfterUpdateHooks(ctx, indexName, updates));
|
|
23547
23769
|
}
|
|
23548
|
-
const vecIndexes = ctx.
|
|
23549
|
-
|
|
23550
|
-
|
|
23551
|
-
|
|
23770
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
23771
|
+
if (vecIndexes === void 0) return;
|
|
23772
|
+
for (const update of updates) {
|
|
23773
|
+
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
23774
|
+
if (vec !== null) vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
23775
|
+
}
|
|
23552
23776
|
}
|
|
23553
23777
|
}
|
|
23554
|
-
|
|
23778
|
+
function updatePreparationOf(ctx, indexName, docId, document) {
|
|
23555
23779
|
ctx.guardShutdown();
|
|
23556
23780
|
validateDocId(docId);
|
|
23557
23781
|
requireMatchingDocId(docId, document);
|
|
23558
|
-
|
|
23559
|
-
|
|
23560
|
-
|
|
23561
|
-
|
|
23562
|
-
|
|
23563
|
-
|
|
23564
|
-
newDocument: document
|
|
23565
|
-
});
|
|
23566
|
-
}
|
|
23567
|
-
if (entry.config.required && entry.config.required.length > 0) {
|
|
23568
|
-
validateRequiredFields(document, entry.config.required);
|
|
23569
|
-
}
|
|
23570
|
-
if (entry.config.embedding) {
|
|
23571
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
23572
|
-
if (entry.embeddingAdapter) {
|
|
23573
|
-
await embedDocumentFields(
|
|
23574
|
-
document,
|
|
23575
|
-
entry.config.embedding,
|
|
23576
|
-
entry.embeddingAdapter,
|
|
23577
|
-
ctx.abortController.signal
|
|
23578
|
-
);
|
|
23579
|
-
} else {
|
|
23580
|
-
assertDocumentCarriesMappedVectors(
|
|
23581
|
-
document,
|
|
23582
|
-
entry.config.embedding,
|
|
23583
|
-
entry.embeddingAdapterName
|
|
23584
|
-
);
|
|
23585
|
-
}
|
|
23586
|
-
}
|
|
23782
|
+
if (!ctx.pluginRegistry.hasHooks("beforeUpdate")) return { document, beforeWrite: null, failure: void 0 };
|
|
23783
|
+
const oldDocument = ctx.requireManager(indexName).get(docId) ?? {};
|
|
23784
|
+
const beforeWrite = () => ctx.pluginRegistry.runHook("beforeUpdate", { indexName, docId, oldDocument, newDocument: document });
|
|
23785
|
+
return { document, beforeWrite, failure: void 0 };
|
|
23786
|
+
}
|
|
23787
|
+
async function finishUpdate(ctx, indexName, entry, docId, document) {
|
|
23587
23788
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
23588
23789
|
if (sizing !== null) await sizing;
|
|
23790
|
+
const manager = ctx.requireManager(indexName);
|
|
23589
23791
|
const vecIndexes = manager.getVectorIndexes();
|
|
23590
23792
|
const extractedVectors = /* @__PURE__ */ new Map();
|
|
23591
23793
|
if (vecIndexes.size > 0) {
|
|
@@ -23679,10 +23881,13 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
23679
23881
|
};
|
|
23680
23882
|
return () => manager.publication.write(indexChangeOutsideExternalWaits(updateInIndex));
|
|
23681
23883
|
}
|
|
23682
|
-
async function updateDocument(ctx, indexName, docId, document, options) {
|
|
23884
|
+
async function updateDocument(ctx, indexName, docId, document, options, hold) {
|
|
23683
23885
|
ctx.guardShutdown();
|
|
23684
|
-
|
|
23685
|
-
const
|
|
23886
|
+
ctx.requireIndex(indexName);
|
|
23887
|
+
const preparation = updatePreparationOf(ctx, indexName, docId, document);
|
|
23888
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
23889
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
23890
|
+
const prepared = await finishUpdate(ctx, indexName, entry, docId, document);
|
|
23686
23891
|
const apply = applyOfUpdate(ctx, indexName, prepared);
|
|
23687
23892
|
if (ctx.durability) {
|
|
23688
23893
|
try {
|
|
@@ -23705,10 +23910,12 @@ async function updateDocument(ctx, indexName, docId, document, options) {
|
|
|
23705
23910
|
} else {
|
|
23706
23911
|
await apply();
|
|
23707
23912
|
}
|
|
23708
|
-
if (!prepared.buffered) await
|
|
23913
|
+
if (!prepared.buffered) await settleUpdates(ctx, indexName, [prepared], hold);
|
|
23709
23914
|
ctx.checkHeapPressure(indexName);
|
|
23710
23915
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
23711
23916
|
}
|
|
23917
|
+
|
|
23918
|
+
// src/engine/mutations/update-batch.ts
|
|
23712
23919
|
function reportedDocIdOf(update) {
|
|
23713
23920
|
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
23714
23921
|
return typeof docId === "string" ? docId : "";
|
|
@@ -23725,29 +23932,47 @@ function requireUpdateEntry(update) {
|
|
|
23725
23932
|
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
23726
23933
|
);
|
|
23727
23934
|
}
|
|
23728
|
-
async function
|
|
23935
|
+
async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed) {
|
|
23936
|
+
const preparations = [];
|
|
23937
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23938
|
+
const update = updates[i];
|
|
23939
|
+
const docId = reportedDocIdOf(update);
|
|
23940
|
+
try {
|
|
23941
|
+
const { document } = requireUpdateEntry(update);
|
|
23942
|
+
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
23943
|
+
} catch (err) {
|
|
23944
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
23945
|
+
}
|
|
23946
|
+
}
|
|
23947
|
+
const { entry } = await prepareDocumentWrites(
|
|
23948
|
+
ctx,
|
|
23949
|
+
indexName,
|
|
23950
|
+
preparations.map((pending) => pending.preparation),
|
|
23951
|
+
hold,
|
|
23952
|
+
false
|
|
23953
|
+
);
|
|
23954
|
+
const prepared = [];
|
|
23955
|
+
for (const { docId, preparation } of preparations) {
|
|
23956
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23957
|
+
try {
|
|
23958
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
23959
|
+
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
23960
|
+
} catch (err) {
|
|
23961
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
23962
|
+
}
|
|
23963
|
+
}
|
|
23964
|
+
return prepared;
|
|
23965
|
+
}
|
|
23966
|
+
async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
23729
23967
|
ctx.guardShutdown();
|
|
23730
|
-
|
|
23968
|
+
ctx.requireIndex(indexName);
|
|
23731
23969
|
validateBatchList(updates, "updates");
|
|
23732
23970
|
const succeeded = [];
|
|
23733
23971
|
const failed = [];
|
|
23734
|
-
const updateBatchManager = ctx.requireManager(indexName);
|
|
23735
|
-
const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
|
|
23736
23972
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
23737
23973
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
23738
23974
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
23739
|
-
const prepared =
|
|
23740
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
23741
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
23742
|
-
const update = updates[i];
|
|
23743
|
-
const docId = reportedDocIdOf(update);
|
|
23744
|
-
try {
|
|
23745
|
-
const { document } = requireUpdateEntry(update);
|
|
23746
|
-
prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document));
|
|
23747
|
-
} catch (err) {
|
|
23748
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
23749
|
-
}
|
|
23750
|
-
}
|
|
23975
|
+
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
23751
23976
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
23752
23977
|
let failures;
|
|
23753
23978
|
try {
|
|
@@ -23774,6 +23999,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
23774
23999
|
} finally {
|
|
23775
24000
|
for (const update of prepared) update.releaseVectorHolds?.();
|
|
23776
24001
|
}
|
|
24002
|
+
const settled = [];
|
|
23777
24003
|
for (let i = 0; i < prepared.length; i++) {
|
|
23778
24004
|
const update = prepared[i];
|
|
23779
24005
|
const failure = failures[i];
|
|
@@ -23781,19 +24007,21 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
23781
24007
|
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
23782
24008
|
continue;
|
|
23783
24009
|
}
|
|
23784
|
-
if (!update.buffered)
|
|
24010
|
+
if (!update.buffered) settled.push(update);
|
|
23785
24011
|
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
23786
24012
|
if (vec !== null) touchedVectorFields.add(fieldPath);
|
|
23787
24013
|
}
|
|
23788
24014
|
succeeded.push(update.docId);
|
|
23789
24015
|
}
|
|
24016
|
+
await settleUpdates(ctx, indexName, settled, hold);
|
|
23790
24017
|
if (chunkEnd < updates.length) {
|
|
23791
24018
|
ctx.checkHeapPressure(indexName);
|
|
23792
24019
|
await new Promise((r) => setTimeout(r, 0));
|
|
23793
24020
|
}
|
|
23794
24021
|
}
|
|
24022
|
+
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
23795
24023
|
for (const fieldPath of touchedVectorFields) {
|
|
23796
|
-
|
|
24024
|
+
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
23797
24025
|
}
|
|
23798
24026
|
ctx.checkHeapPressure(indexName);
|
|
23799
24027
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
@@ -23853,22 +24081,22 @@ function updateItemOf(update) {
|
|
|
23853
24081
|
update
|
|
23854
24082
|
};
|
|
23855
24083
|
}
|
|
23856
|
-
async function insertDocument2(ctx, indexName, document, docId, options) {
|
|
24084
|
+
async function insertDocument2(ctx, indexName, document, docId, options, hold) {
|
|
23857
24085
|
const block = blockFor(ctx, indexName);
|
|
23858
|
-
if (block === void 0) return insertDocument(ctx, indexName, document, docId, options);
|
|
24086
|
+
if (block === void 0) return insertDocument(ctx, indexName, document, docId, options, hold);
|
|
23859
24087
|
return withReservation(
|
|
23860
24088
|
block.grouping,
|
|
23861
24089
|
indexName,
|
|
23862
24090
|
document,
|
|
23863
|
-
() => insertDocument(ctx, indexName, document, docId, options)
|
|
24091
|
+
() => insertDocument(ctx, indexName, document, docId, options, hold)
|
|
23864
24092
|
);
|
|
23865
24093
|
}
|
|
23866
|
-
async function insertDocumentBatch2(ctx, indexName, documents2, options) {
|
|
24094
|
+
async function insertDocumentBatch2(ctx, indexName, documents2, options, hold) {
|
|
23867
24095
|
const block = blockFor(ctx, indexName);
|
|
23868
|
-
if (block === void 0) return insertDocumentBatch(ctx, indexName, documents2, options);
|
|
24096
|
+
if (block === void 0) return insertDocumentBatch(ctx, indexName, documents2, options, hold);
|
|
23869
24097
|
const grouping = block.grouping;
|
|
23870
24098
|
if (grouping === void 0 || !Array.isArray(documents2) || documents2.length === 0) {
|
|
23871
|
-
return insertDocumentBatch(ctx, indexName, documents2, options);
|
|
24099
|
+
return insertDocumentBatch(ctx, indexName, documents2, options, hold);
|
|
23872
24100
|
}
|
|
23873
24101
|
const items = documents2.map((document) => ({ docId: providedDocId(document) ?? "", document }));
|
|
23874
24102
|
return writeInGroups(
|
|
@@ -23879,26 +24107,27 @@ async function insertDocumentBatch2(ctx, indexName, documents2, options) {
|
|
|
23879
24107
|
ctx,
|
|
23880
24108
|
indexName,
|
|
23881
24109
|
group.map((item) => item.document),
|
|
23882
|
-
options
|
|
24110
|
+
options,
|
|
24111
|
+
hold
|
|
23883
24112
|
)
|
|
23884
24113
|
);
|
|
23885
24114
|
}
|
|
23886
|
-
async function updateDocument2(ctx, indexName, docId, document, options) {
|
|
24115
|
+
async function updateDocument2(ctx, indexName, docId, document, options, hold) {
|
|
23887
24116
|
const block = blockFor(ctx, indexName);
|
|
23888
|
-
if (block === void 0) return updateDocument(ctx, indexName, docId, document, options);
|
|
24117
|
+
if (block === void 0) return updateDocument(ctx, indexName, docId, document, options, hold);
|
|
23889
24118
|
return withReservation(
|
|
23890
24119
|
block.grouping,
|
|
23891
24120
|
indexName,
|
|
23892
24121
|
document,
|
|
23893
|
-
() => updateDocument(ctx, indexName, docId, document, options)
|
|
24122
|
+
() => updateDocument(ctx, indexName, docId, document, options, hold)
|
|
23894
24123
|
);
|
|
23895
24124
|
}
|
|
23896
|
-
async function updateDocumentBatch2(ctx, indexName, updates, options) {
|
|
24125
|
+
async function updateDocumentBatch2(ctx, indexName, updates, options, hold) {
|
|
23897
24126
|
const block = blockFor(ctx, indexName);
|
|
23898
|
-
if (block === void 0) return updateDocumentBatch(ctx, indexName, updates, options);
|
|
24127
|
+
if (block === void 0) return updateDocumentBatch(ctx, indexName, updates, options, hold);
|
|
23899
24128
|
const grouping = block.grouping;
|
|
23900
24129
|
if (grouping === void 0 || !Array.isArray(updates) || updates.length === 0) {
|
|
23901
|
-
return updateDocumentBatch(ctx, indexName, updates, options);
|
|
24130
|
+
return updateDocumentBatch(ctx, indexName, updates, options, hold);
|
|
23902
24131
|
}
|
|
23903
24132
|
return writeInGroups(
|
|
23904
24133
|
grouping,
|
|
@@ -23908,16 +24137,17 @@ async function updateDocumentBatch2(ctx, indexName, updates, options) {
|
|
|
23908
24137
|
ctx,
|
|
23909
24138
|
indexName,
|
|
23910
24139
|
group.map((item) => item.update),
|
|
23911
|
-
options
|
|
24140
|
+
options,
|
|
24141
|
+
hold
|
|
23912
24142
|
)
|
|
23913
24143
|
);
|
|
23914
24144
|
}
|
|
23915
|
-
async function removeDocument4(ctx, indexName, docId, options) {
|
|
23916
|
-
await removeDocument3(ctx, indexName, docId, options);
|
|
24145
|
+
async function removeDocument4(ctx, indexName, docId, options, hold) {
|
|
24146
|
+
await removeDocument3(ctx, indexName, docId, options, hold);
|
|
23917
24147
|
ctx.memoryWriteBlock?.memoryReleased();
|
|
23918
24148
|
}
|
|
23919
|
-
async function removeDocumentBatch2(ctx, indexName, docIds, options) {
|
|
23920
|
-
const result = await removeDocumentBatch(ctx, indexName, docIds, options);
|
|
24149
|
+
async function removeDocumentBatch2(ctx, indexName, docIds, options, hold) {
|
|
24150
|
+
const result = await removeDocumentBatch(ctx, indexName, docIds, options, hold);
|
|
23921
24151
|
if (result.succeeded.length > 0) ctx.memoryWriteBlock?.memoryReleased();
|
|
23922
24152
|
return result;
|
|
23923
24153
|
}
|
|
@@ -24291,7 +24521,8 @@ var DURABILITY_CONFIG_KEYS = {
|
|
|
24291
24521
|
segmentMaxBytes: true,
|
|
24292
24522
|
checkpointIntervalMs: true,
|
|
24293
24523
|
checkpointMutationThreshold: true,
|
|
24294
|
-
compactionThreshold: true
|
|
24524
|
+
compactionThreshold: true,
|
|
24525
|
+
rewriteBytesPerCheckpoint: true
|
|
24295
24526
|
};
|
|
24296
24527
|
var WORKER_CONFIG_KEYS = {
|
|
24297
24528
|
enabled: true,
|
|
@@ -24702,7 +24933,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
24702
24933
|
|
|
24703
24934
|
// src/engine/durability-config-code.ts
|
|
24704
24935
|
var durabilityConfig = codeLoader(
|
|
24705
|
-
() => import('./durability-config-
|
|
24936
|
+
() => import('./durability-config-NWU2G2SJ.mjs').then((code7) => {
|
|
24706
24937
|
code7.bindDurabilityConfigHost(optionalCodeHost());
|
|
24707
24938
|
return code7;
|
|
24708
24939
|
}),
|
|
@@ -25275,7 +25506,7 @@ var workerCopies = codeLoader(async () => {
|
|
|
25275
25506
|
const [segmentCodec2, segmentMerge2, code7] = await Promise.all([
|
|
25276
25507
|
loadSegmentCodec(),
|
|
25277
25508
|
loadSegmentMerge(),
|
|
25278
|
-
import('./worker-copies-
|
|
25509
|
+
import('./worker-copies-WYPRCNB7.mjs')
|
|
25279
25510
|
]);
|
|
25280
25511
|
code7.bindWorkerCopiesHost(optionalCodeHost(), segmentCodec2, segmentMerge2);
|
|
25281
25512
|
return code7;
|
|
@@ -25356,7 +25587,7 @@ async function createWorkerOrchestrator(...args) {
|
|
|
25356
25587
|
var loaded3 = null;
|
|
25357
25588
|
function loadRerankRuntime() {
|
|
25358
25589
|
if (loaded3 !== null) return Promise.resolve(loaded3);
|
|
25359
|
-
return import('./runtime-
|
|
25590
|
+
return import('./runtime-USQYKN3I.mjs').then((runtime) => {
|
|
25360
25591
|
runtime.bindRerankHost(optionalCodeHost());
|
|
25361
25592
|
loaded3 = runtime;
|
|
25362
25593
|
return runtime;
|
|
@@ -25521,12 +25752,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
25521
25752
|
queueDiskAdoption: orchestrator.queueDiskAdoption,
|
|
25522
25753
|
awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
|
|
25523
25754
|
publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
|
|
25524
|
-
recordCheckpoint(indexName, documentCount, partitionCount) {
|
|
25525
|
-
const entry = indexRegistry.get(indexName);
|
|
25526
|
-
if (entry === void 0) return;
|
|
25527
|
-
entry.documentCount = documentCount;
|
|
25528
|
-
entry.partitionCount = partitionCount;
|
|
25529
|
-
},
|
|
25530
25755
|
engineCore: () => engineCore
|
|
25531
25756
|
});
|
|
25532
25757
|
const invalidationCode = config?.invalidation === void 0 ? null : durabilityWiring ?? await loadStorageCode();
|
|
@@ -27905,6 +28130,22 @@ function clearEngineIndex(core2, indexName) {
|
|
|
27905
28130
|
});
|
|
27906
28131
|
}
|
|
27907
28132
|
|
|
28133
|
+
// src/engine/index-hold.ts
|
|
28134
|
+
async function holdIndex(indexState, indexName) {
|
|
28135
|
+
let release = await indexState.acquire(indexName);
|
|
28136
|
+
return {
|
|
28137
|
+
release: () => release(),
|
|
28138
|
+
async outside(work) {
|
|
28139
|
+
release();
|
|
28140
|
+
try {
|
|
28141
|
+
return await work();
|
|
28142
|
+
} finally {
|
|
28143
|
+
release = await indexState.acquire(indexName);
|
|
28144
|
+
}
|
|
28145
|
+
}
|
|
28146
|
+
};
|
|
28147
|
+
}
|
|
28148
|
+
|
|
27908
28149
|
// src/narsil/bm25.ts
|
|
27909
28150
|
function bm25Operations(core2) {
|
|
27910
28151
|
return {
|
|
@@ -29032,7 +29273,7 @@ async function executePreflight(request, callerContext) {
|
|
|
29032
29273
|
|
|
29033
29274
|
// src/engine/resolve-vector-text-code.ts
|
|
29034
29275
|
var resolveVectorText = codeLoader(
|
|
29035
|
-
() => import('./resolve-vector-text-
|
|
29276
|
+
() => import('./resolve-vector-text-SD6BEMKZ.mjs').then((code7) => {
|
|
29036
29277
|
code7.bindResolveVectorTextHost(optionalCodeHost());
|
|
29037
29278
|
return code7;
|
|
29038
29279
|
}),
|
|
@@ -29045,10 +29286,10 @@ async function enterRead(core2, indexName) {
|
|
|
29045
29286
|
core2.guardShutdown();
|
|
29046
29287
|
if (core2.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
29047
29288
|
}
|
|
29048
|
-
async function
|
|
29049
|
-
if (params.vector?.text === void 0) return
|
|
29050
|
-
const {
|
|
29051
|
-
return
|
|
29289
|
+
async function embeddedFor(core2, indexName, params, signal, earlier) {
|
|
29290
|
+
if (params.vector?.text === void 0) return null;
|
|
29291
|
+
const { embedQueryWithAdapterOf } = await loadResolveVectorText();
|
|
29292
|
+
return embedQueryWithAdapterOf(indexName, params, core2.requireIndex(indexName), signal, earlier);
|
|
29052
29293
|
}
|
|
29053
29294
|
function phraseReadinessOf(core2, indexName, config) {
|
|
29054
29295
|
return () => {
|
|
@@ -29073,24 +29314,17 @@ function broadcastStatsFor(core2, options) {
|
|
|
29073
29314
|
}
|
|
29074
29315
|
return (name, tokens) => invalidation.broadcastStats(name, tokens);
|
|
29075
29316
|
}
|
|
29076
|
-
|
|
29077
|
-
|
|
29078
|
-
|
|
29079
|
-
await
|
|
29080
|
-
|
|
29317
|
+
var EMBEDDING_ADAPTER_CHANGED = 0;
|
|
29318
|
+
var INDEX_REPLACED_DURING_RERANK = 1;
|
|
29319
|
+
async function queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options) {
|
|
29320
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
29321
|
+
let runner;
|
|
29081
29322
|
try {
|
|
29082
29323
|
const entry = core2.requireIndex(indexName);
|
|
29324
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return EMBEDDING_ADAPTER_CHANGED;
|
|
29083
29325
|
const manager = core2.requireManager(indexName);
|
|
29084
|
-
const scoped = scopedParams(params, options);
|
|
29085
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
29086
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
29087
|
-
}
|
|
29088
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
29089
|
-
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
29090
|
-
const resolvedParams = preparedParams.vector?.text === void 0 ? preparedParams : await resolveVectorText2(preparedParams, entry, signal);
|
|
29091
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
29092
|
-
if (loading4 !== null) await loading4;
|
|
29093
29326
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
29327
|
+
runner = rerank?.rerankRunnerFor(core2, options?.signal, hold, indexName, manager);
|
|
29094
29328
|
let analysisStale = false;
|
|
29095
29329
|
const result = await answerFromOneState(
|
|
29096
29330
|
manager,
|
|
@@ -29111,16 +29345,49 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
29111
29345
|
indexBm25: options?.indexBm25,
|
|
29112
29346
|
cursorBinding: lazyQueryBinding(params),
|
|
29113
29347
|
patternWorkCap: core2.patternWorkCap,
|
|
29114
|
-
rerank:
|
|
29348
|
+
rerank: runner,
|
|
29115
29349
|
requirePhraseReady: phraseReadinessOf(core2, indexName, entry.config)
|
|
29116
29350
|
});
|
|
29117
29351
|
},
|
|
29118
29352
|
options?.signal
|
|
29119
29353
|
);
|
|
29354
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
29120
29355
|
if (analysisStale) {
|
|
29121
29356
|
result.analysisStale = true;
|
|
29122
29357
|
}
|
|
29123
29358
|
options?.onRanked?.(manager, result);
|
|
29359
|
+
return result;
|
|
29360
|
+
} catch (error) {
|
|
29361
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
29362
|
+
throw error;
|
|
29363
|
+
} finally {
|
|
29364
|
+
hold.release();
|
|
29365
|
+
}
|
|
29366
|
+
}
|
|
29367
|
+
async function runEngineQuery(core2, indexName, params, options) {
|
|
29368
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
29369
|
+
options?.signal?.throwIfAborted();
|
|
29370
|
+
await enterRead(core2, indexName);
|
|
29371
|
+
core2.requireIndex(indexName);
|
|
29372
|
+
const scoped = scopedParams(params, options);
|
|
29373
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
29374
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
29375
|
+
}
|
|
29376
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
29377
|
+
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
29378
|
+
let embedded2 = null;
|
|
29379
|
+
let replacedReranks = 0;
|
|
29380
|
+
while (true) {
|
|
29381
|
+
embedded2 = await embeddedFor(core2, indexName, preparedParams, signal, embedded2);
|
|
29382
|
+
const resolvedParams = embedded2?.params ?? preparedParams;
|
|
29383
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
29384
|
+
if (loading4 !== null) await loading4;
|
|
29385
|
+
const result = await queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options);
|
|
29386
|
+
if (result === EMBEDDING_ADAPTER_CHANGED) continue;
|
|
29387
|
+
if (result === INDEX_REPLACED_DURING_RERANK) {
|
|
29388
|
+
rerank?.refuseAfterIndexChanges(indexName, ++replacedReranks);
|
|
29389
|
+
continue;
|
|
29390
|
+
}
|
|
29124
29391
|
if (core2.pluginRegistry.hasHooks("afterSearch")) {
|
|
29125
29392
|
const observed = { ...result, hits: [...result.hits] };
|
|
29126
29393
|
try {
|
|
@@ -29130,26 +29397,15 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
29130
29397
|
}
|
|
29131
29398
|
}
|
|
29132
29399
|
return result;
|
|
29133
|
-
} finally {
|
|
29134
|
-
release();
|
|
29135
29400
|
}
|
|
29136
29401
|
}
|
|
29137
|
-
async function
|
|
29138
|
-
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
29139
|
-
options?.signal?.throwIfAborted();
|
|
29140
|
-
await enterRead(core2, indexName);
|
|
29402
|
+
async function preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options) {
|
|
29141
29403
|
const release = await core2.indexState.acquire(indexName);
|
|
29142
29404
|
try {
|
|
29143
29405
|
const entry = core2.requireIndex(indexName);
|
|
29406
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return null;
|
|
29144
29407
|
const manager = core2.requireManager(indexName);
|
|
29145
|
-
const scoped = scopedParams(params, options);
|
|
29146
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
29147
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
29148
|
-
}
|
|
29149
|
-
const resolvedParams = scoped.vector?.text === void 0 ? scoped : await resolveVectorText2(scoped, entry, signal);
|
|
29150
29408
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
29151
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
29152
|
-
if (loading4 !== null) await loading4;
|
|
29153
29409
|
let analysisStale = false;
|
|
29154
29410
|
const result = await answerFromOneState(
|
|
29155
29411
|
manager,
|
|
@@ -29182,6 +29438,25 @@ async function runEnginePreflight(core2, indexName, params, options) {
|
|
|
29182
29438
|
release();
|
|
29183
29439
|
}
|
|
29184
29440
|
}
|
|
29441
|
+
async function runEnginePreflight(core2, indexName, params, options) {
|
|
29442
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
29443
|
+
options?.signal?.throwIfAborted();
|
|
29444
|
+
await enterRead(core2, indexName);
|
|
29445
|
+
core2.requireIndex(indexName);
|
|
29446
|
+
const scoped = scopedParams(params, options);
|
|
29447
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
29448
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
29449
|
+
}
|
|
29450
|
+
let embedded2 = null;
|
|
29451
|
+
while (true) {
|
|
29452
|
+
embedded2 = await embeddedFor(core2, indexName, scoped, signal, embedded2);
|
|
29453
|
+
const resolvedParams = embedded2?.params ?? scoped;
|
|
29454
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
29455
|
+
if (loading4 !== null) await loading4;
|
|
29456
|
+
const result = await preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options);
|
|
29457
|
+
if (result !== null) return result;
|
|
29458
|
+
}
|
|
29459
|
+
}
|
|
29185
29460
|
async function runEngineSuggest(core2, indexName, params, partitionIds) {
|
|
29186
29461
|
const { executeSuggest } = await importOptionalReads();
|
|
29187
29462
|
await enterRead(core2, indexName);
|
|
@@ -29299,11 +29574,11 @@ function createNarsilFromCore(core2, config) {
|
|
|
29299
29574
|
async function withOpenIndex2(indexName, action) {
|
|
29300
29575
|
guardShutdown();
|
|
29301
29576
|
requireIndex(indexName);
|
|
29302
|
-
const
|
|
29577
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
29303
29578
|
try {
|
|
29304
|
-
return await action();
|
|
29579
|
+
return await action(hold);
|
|
29305
29580
|
} finally {
|
|
29306
|
-
release();
|
|
29581
|
+
hold.release();
|
|
29307
29582
|
}
|
|
29308
29583
|
}
|
|
29309
29584
|
const rebalancesStarting = /* @__PURE__ */ new Map();
|
|
@@ -29366,6 +29641,8 @@ function createNarsilFromCore(core2, config) {
|
|
|
29366
29641
|
documentCount: manager?.countDocuments() ?? entry.documentCount,
|
|
29367
29642
|
partitionCount: manager?.partitionCount ?? entry.partitionCount,
|
|
29368
29643
|
estimatedMemoryBytes: manager?.estimateMemoryBytes() ?? 0,
|
|
29644
|
+
savedBytes: entry.storedSizes?.savedBytes,
|
|
29645
|
+
logBytes: entry.storedSizes?.logBytes,
|
|
29369
29646
|
language: entry.language.name,
|
|
29370
29647
|
schema: entry.config.schema,
|
|
29371
29648
|
heldSearchStates: manager?.heldSearchStates ?? 0
|
|
@@ -29373,26 +29650,28 @@ function createNarsilFromCore(core2, config) {
|
|
|
29373
29650
|
},
|
|
29374
29651
|
getPartitionStats(indexName) {
|
|
29375
29652
|
guardShutdown();
|
|
29376
|
-
requireIndex(indexName);
|
|
29377
|
-
|
|
29653
|
+
const entry = requireIndex(indexName);
|
|
29654
|
+
const stats = executor.getManager(indexName)?.getPartitionStats() ?? [];
|
|
29655
|
+
entry.storedSizes?.addTo(stats);
|
|
29656
|
+
return stats;
|
|
29378
29657
|
},
|
|
29379
29658
|
insert(indexName, document, docId, options) {
|
|
29380
|
-
return withCountedWrite(indexName, () => insertDocument2(mutationCtx, indexName, document, docId, options));
|
|
29659
|
+
return withCountedWrite(indexName, (hold) => insertDocument2(mutationCtx, indexName, document, docId, options, hold));
|
|
29381
29660
|
},
|
|
29382
29661
|
insertBatch(indexName, documents2, options) {
|
|
29383
|
-
return withCountedWrite(indexName, () => insertDocumentBatch2(mutationCtx, indexName, documents2, options));
|
|
29662
|
+
return withCountedWrite(indexName, (hold) => insertDocumentBatch2(mutationCtx, indexName, documents2, options, hold));
|
|
29384
29663
|
},
|
|
29385
29664
|
remove(indexName, docId, options) {
|
|
29386
|
-
return withCountedWrite(indexName, () => removeDocument4(mutationCtx, indexName, docId, options));
|
|
29665
|
+
return withCountedWrite(indexName, (hold) => removeDocument4(mutationCtx, indexName, docId, options, hold));
|
|
29387
29666
|
},
|
|
29388
29667
|
removeBatch(indexName, docIds, options) {
|
|
29389
|
-
return withCountedWrite(indexName, () => removeDocumentBatch2(mutationCtx, indexName, docIds, options));
|
|
29668
|
+
return withCountedWrite(indexName, (hold) => removeDocumentBatch2(mutationCtx, indexName, docIds, options, hold));
|
|
29390
29669
|
},
|
|
29391
29670
|
update(indexName, docId, document, options) {
|
|
29392
|
-
return withCountedWrite(indexName, () => updateDocument2(mutationCtx, indexName, docId, document, options));
|
|
29671
|
+
return withCountedWrite(indexName, (hold) => updateDocument2(mutationCtx, indexName, docId, document, options, hold));
|
|
29393
29672
|
},
|
|
29394
29673
|
updateBatch(indexName, updates, options) {
|
|
29395
|
-
return withCountedWrite(indexName, () => updateDocumentBatch2(mutationCtx, indexName, updates, options));
|
|
29674
|
+
return withCountedWrite(indexName, (hold) => updateDocumentBatch2(mutationCtx, indexName, updates, options, hold));
|
|
29396
29675
|
},
|
|
29397
29676
|
waitForWrites(indexName) {
|
|
29398
29677
|
guardShutdown();
|
|
@@ -29595,7 +29874,7 @@ var suggestionsCode = codeLoader(async () => {
|
|
|
29595
29874
|
const [msgpack, suggestionStore2, code7] = await Promise.all([
|
|
29596
29875
|
loadMessagePack(),
|
|
29597
29876
|
loadSuggestionStore(),
|
|
29598
|
-
import('./suggestions-leaf-
|
|
29877
|
+
import('./suggestions-leaf-FP5HSTFP.mjs')
|
|
29599
29878
|
]);
|
|
29600
29879
|
code7.bindSuggestionsHost(optionalCodeHost(), msgpack, suggestionStore2);
|
|
29601
29880
|
return code7;
|
|
@@ -30665,14 +30944,11 @@ function writesHostPart() {
|
|
|
30665
30944
|
admitBufferedUpsert,
|
|
30666
30945
|
admitInsert,
|
|
30667
30946
|
arrayPositionsAreStale,
|
|
30668
|
-
asBatchInsertError,
|
|
30669
30947
|
awaitWriteVisibility,
|
|
30948
|
+
capsEachPartition,
|
|
30670
30949
|
changeIndexOutsideExternalWaits,
|
|
30671
|
-
collectRequiredFieldFailures,
|
|
30672
30950
|
createFrozenSegment,
|
|
30673
|
-
createPendingAdmissions,
|
|
30674
30951
|
deleteNestedValue,
|
|
30675
|
-
embedChunkDocuments,
|
|
30676
30952
|
emitSyncChange,
|
|
30677
30953
|
extractVectorFromDoc,
|
|
30678
30954
|
fnv1a,
|
|
@@ -30694,7 +30970,6 @@ function writesHostPart() {
|
|
|
30694
30970
|
prepareStoredSynonyms,
|
|
30695
30971
|
providedDocId,
|
|
30696
30972
|
readHeapStatistics,
|
|
30697
|
-
recordPendingAdmission,
|
|
30698
30973
|
removeDocumentVectors,
|
|
30699
30974
|
requireDocumentObject,
|
|
30700
30975
|
requireDocumentWithinSizeLimit,
|
|
@@ -30749,7 +31024,7 @@ var segmentMerge = codeLoader(async () => {
|
|
|
30749
31024
|
const [msgpack, codec, code7] = await Promise.all([
|
|
30750
31025
|
loadMessagePack(),
|
|
30751
31026
|
loadSegmentCodec(),
|
|
30752
|
-
import('./segment-merge-leaf-
|
|
31027
|
+
import('./segment-merge-leaf-JGND6LG5.mjs')
|
|
30753
31028
|
]);
|
|
30754
31029
|
code7.bindSegmentMergeHost(optionalCodeHost(), msgpack, codec);
|
|
30755
31030
|
return code7;
|
|
@@ -31365,4 +31640,4 @@ function createDirectExecutor(options) {
|
|
|
31365
31640
|
};
|
|
31366
31641
|
}
|
|
31367
31642
|
|
|
31368
|
-
export { ARRAY_POSITION_GAP, CASE_FOLD_EXPANSION_FACTOR, COMPACTION_SEGMENT_TRIGGER, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_COPY_IDLE_TIMEOUT_MS, DEFAULT_COPY_THRESHOLD, DEFAULT_RERANK_WINDOW, DISTINCT_TERM_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FULL_MEMORY_COLLECTION_INTERVAL_MS, GROUP_FORMAT, GROUP_PATTERN_SOURCE, IDLE_MERGE_DELAY_MS, LIVE_TAIL_FLUSH_DOCUMENTS, LIVE_TAIL_FREEZE_FLOOR, LOAD_BUFFER_YIELD_INTERVAL, LONG_QUERY_THREADS, MAIN_COPY_LONE_QUERY_DOCUMENTS, MAX_PENDING_REPLICATION_DOCUMENTS, MAX_RERANK_WINDOW, NORMALIZATION_EXPANSION_FACTOR, NO_REMOVED_SPELLINGS, PARTIAL_SPELLING_HEAP_BYTES, PATTERN_RUN_HEAP_BYTES, POOL_RESTART_DELAY_MAX_MS, POOL_RESTART_DELAY_MS, REPLICATION_WINDOW, RETIRED_THREADS_EXIT_WAIT_MS, SEGMENTS_PER_MERGE, SUGGESTIONS_INDEX_INFIX, SegmentRefusal, TERM_UNIT_HEAP_BYTES, TermLookup, VECTOR_COMPONENT_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VERSION, afterCurrentTurn, alreadyPresentOnWorker, applyProjection, arrayPositionsAreStale, assemblePagedFrozenSegment, availablePatternSearch, bindOptionalWritesHost, bindQuerySignal, bindRerankWindowSource, bindSegmentCodecHost, bindSegmentMergeHost, bindWorkerCopiesHost, bm25SetsValue, buildPartialWordTableFrom, buildSegmentPayloads, clampLimit, clampRowCount, clearNormalizationCache, clearQuerySignal, compareScores, configurationPayloadOf, configureNormalizationCache, countsWithoutScores, createBooleanIndex, createCompositePartition, createDirectExecutor, createEncodedDocumentSource, createEngineCore, createEngineIndex, createEnumIndex, createFrozenRemovals, createFrozenSegment, createLoadedGeoIndex, createMemoryMonitor, createNarsil, createNarsilFromCore, createNumericIndex, createPartitionIndex, createPartitionManager, createPartitionRouter, createPublicationGate, createRequestId, createSharedFrozenSegment, createWorkerExecutor, createWorkerFactory, customTokensOf, decodeEveryStoredDocument, decodeMapWithoutFields, decodePageCursor, decodeStoredDocument, decodeSurfaceTable, decodeSynonymRules, documentsOf2 as documentsOf, dropEngineIndex, eligibleIndexNames, emitEngineEvent, emitSyncChange, emitWriteEventAfter, emptyPendingRegions, encodeDocumentTableData, encodePageCursor, encodeSegmentState, endOfValue, engineCoreOf, executePreflight, executeQuery, fanOutMatchCount, fanOutQuery, fieldIndexRemapOf, fillHits, filterTestCount, fnv1a, fnv1aUint32s, freezeEncodedSegmentShared, freezeSegmentShared, generateId, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, getVectorFieldPaths, growPartitionsTo, hasLanguage, hasStopWords, hasTokenizer, holdSyncEpochChange, holdVectorsForRollback, holdsADiscardedField, holdsQuotedText, importOptionalReads, initializeCopyPublication, insertDocument, insertDocumentBatch, insertDocumentVectors, isCompositePartition, isDeterministicFailure, isSimdAvailable, isValidWorkerAction, isValidWorkerResponse, joinPatternWorkMeter, kWayMerge, lazyQueryBinding, listBindingOf, listenForSyncChanges, loadCursorPageFor, loadDigest, loadFacetMergesFor, loadGeoCode, loadGeoCodeFor, loadGroupingFor, loadGzip, loadOptionalWrites, loadPhraseCodeFor, loadPinningFor, loadPrefixFor, loadQueryCode, loadRerankRuntime, loadSchemaCodeFor, loadSortingFor, loadStorageCode, loadSuggestionsCode, loadVectorCode, lookupOfStoredSynonyms, memoryFractionsOf, mergeDocFrequencies, mergeEntries, mergeFacets, mergeFieldIndexes, mergeKeywordSources, mergePartitionStats, mergePostings, mergePostings2, mergeSegmentContents, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, normalizeSort, now, oneAtATime, oversampledFacetConfig, oversampledShardSize, oversizedConfigurationPayloadOf, partialWordSearchModule, partitionStateOf, patternHost, patternIndexOf, prepareConfiguredSynonyms, prepareDocumentText, prepareDocumentVectors, prepareRemovalText, prepareStoredSynonyms, projectionKeepsField, providedDocId, publishCopyAction, queryBindingOf, rankPage, readHeapStatistics, readMemoryFigures, readRecentMemoryFigures, readResidentMemory, rebuildSizeIndexes, reconstructSchemaFromMetadata, regionsOverlap, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocument3 as removeDocument, removeDocumentBatch, removeDocumentBatch2, removeDocumentVectors, removedSurfaceSpellings, reportIneligible, requireEncodedConfigurationWithinSizeLimit, requireKnownConfig, requireLoadedOptionalReads, requireLoadedVectorCode, requireMatchingCursor, requirePatternSearch, requireSearchBackendSettings2 as requireSearchBackendSettings, requireSurfaceFormsForPartialFields, requireWithinResultWindow, rerankWindowCapOf, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, resolveProjection, resolveRunnableConfig, returnsEveryMatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, runtimeHost, sanitizeGlobalStats, scanPostingBlocks, segmentArraysOf, selectionKeeps, setNestedValue, shareSegmentArrays, sortModesOf, sortSignatureEntry, sortSignatureOf, startEngineCore, storedArrayPositionGap, storedBM25Params, storedDocumentsOf, survivorDocumentsOf, synonymModule, synonymSearchModule, toError2 as toError, topLevelMapKeys, updateDocument, updateDocumentBatch, updateDocumentVectors, validateBM25Params, validateBM25Shape, validateEmbeddingConfig, validateMemoryFractions, validatePartitionConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, visitTokens, withOpenIndex, workerIneligibility, workersEnabledByDefault, wrapFrozenTokenTable, writePartitionDocument, yieldToEventLoop };
|
|
31643
|
+
export { ARRAY_POSITION_GAP, CASE_FOLD_EXPANSION_FACTOR, COMPACTION_SEGMENT_TRIGGER, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_COPY_IDLE_TIMEOUT_MS, DEFAULT_COPY_THRESHOLD, DEFAULT_RERANK_WINDOW, DISTINCT_TERM_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FULL_MEMORY_COLLECTION_INTERVAL_MS, GROUP_FORMAT, GROUP_PATTERN_SOURCE, IDLE_MERGE_DELAY_MS, LIVE_TAIL_FLUSH_DOCUMENTS, LIVE_TAIL_FREEZE_FLOOR, LOAD_BUFFER_YIELD_INTERVAL, LONG_QUERY_THREADS, MAIN_COPY_LONE_QUERY_DOCUMENTS, MAX_PENDING_REPLICATION_DOCUMENTS, MAX_RERANK_WINDOW, NORMALIZATION_EXPANSION_FACTOR, NO_REMOVED_SPELLINGS, PARTIAL_SPELLING_HEAP_BYTES, PATTERN_RUN_HEAP_BYTES, POOL_RESTART_DELAY_MAX_MS, POOL_RESTART_DELAY_MS, REPLICATION_WINDOW, RETIRED_THREADS_EXIT_WAIT_MS, SEGMENTS_PER_MERGE, SUGGESTIONS_INDEX_INFIX, SegmentRefusal, TERM_UNIT_HEAP_BYTES, TermLookup, VECTOR_COMPONENT_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VERSION, afterCurrentTurn, alreadyPresentOnWorker, applyProjection, arrayPositionsAreStale, assemblePagedFrozenSegment, availablePatternSearch, bindOptionalWritesHost, bindQuerySignal, bindRerankWindowSource, bindSegmentCodecHost, bindSegmentMergeHost, bindWorkerCopiesHost, bm25SetsValue, buildPartialWordTableFrom, buildSegmentPayloads, clampLimit, clampRowCount, clearNormalizationCache, clearQuerySignal, compareScores, configurationPayloadOf, configureNormalizationCache, countsWithoutScores, createBooleanIndex, createCompositePartition, createDirectExecutor, createEncodedDocumentSource, createEngineCore, createEngineIndex, createEnumIndex, createFrozenRemovals, createFrozenSegment, createLoadedGeoIndex, createMemoryMonitor, createNarsil, createNarsilFromCore, createNumericIndex, createPartitionIndex, createPartitionManager, createPartitionRouter, createPublicationGate, createRequestId, createSharedFrozenSegment, createWorkerExecutor, createWorkerFactory, customTokensOf, decodeEveryStoredDocument, decodeMapWithoutFields, decodePageCursor, decodeStoredDocument, decodeSurfaceTable, decodeSynonymRules, documentsOf2 as documentsOf, dropEngineIndex, eligibleIndexNames, emitEngineEvent, emitSyncChange, emitWriteEventAfter, emptyPendingRegions, encodeDocumentTableData, encodePageCursor, encodeSegmentState, endOfValue, engineCoreOf, executePreflight, executeQuery, fanOutMatchCount, fanOutQuery, fieldIndexRemapOf, fillHits, filterTestCount, fnv1a, fnv1aUint32s, freezeEncodedSegmentShared, freezeSegmentShared, generateId, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, getVectorFieldPaths, growPartitionsTo, hasLanguage, hasStopWords, hasTokenizer, holdSyncEpochChange, holdVectorsForRollback, holdsADiscardedField, holdsQuotedText, importOptionalReads, initializeCopyPublication, insertDocument, insertDocumentBatch, insertDocumentVectors, isCompositePartition, isDeterministicFailure, isSimdAvailable, isValidWorkerAction, isValidWorkerResponse, joinPatternWorkMeter, kWayMerge, lazyQueryBinding, listBindingOf, listenForSyncChanges, loadCursorPageFor, loadDigest, loadFacetMergesFor, loadGeoCode, loadGeoCodeFor, loadGroupingFor, loadGzip, loadOptionalWrites, loadPhraseCodeFor, loadPinningFor, loadPrefixFor, loadQueryCode, loadRerankRuntime, loadSchemaCodeFor, loadSortingFor, loadStorageCode, loadSuggestionsCode, loadVectorCode, lookupOfStoredSynonyms, memoryFractionsOf, mergeDocFrequencies, mergeEntries, mergeFacets, mergeFieldIndexes, mergeKeywordSources, mergePartitionStats, mergePostings, mergePostings2, mergeSegmentContents, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, normalizeSort, now, oneAtATime, oversampledFacetConfig, oversampledShardSize, oversizedConfigurationPayloadOf, partialWordSearchModule, partitionStateOf, patternHost, patternIndexOf, prepareConfiguredSynonyms, prepareDocumentText, prepareDocumentVectors, prepareDocumentWrite, prepareRemovalText, prepareStoredSynonyms, projectionKeepsField, providedDocId, publishCopyAction, queryBindingOf, rankPage, readHeapStatistics, readMemoryFigures, readRecentMemoryFigures, readResidentMemory, rebuildSizeIndexes, reconstructSchemaFromMetadata, regionsOverlap, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocument3 as removeDocument, removeDocumentBatch, removeDocumentBatch2, removeDocumentVectors, removedSurfaceSpellings, reportIneligible, requireEncodedConfigurationWithinSizeLimit, requireKnownConfig, requireLoadedOptionalReads, requireLoadedVectorCode, requireMatchingCursor, requirePatternSearch, requireSearchBackendSettings2 as requireSearchBackendSettings, requireSurfaceFormsForPartialFields, requireWithinResultWindow, rerankWindowCapOf, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, resolveProjection, resolveRunnableConfig, returnsEveryMatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, runtimeHost, sanitizeGlobalStats, scanPostingBlocks, segmentArraysOf, selectionKeeps, setNestedValue, shareSegmentArrays, sortModesOf, sortSignatureEntry, sortSignatureOf, startEngineCore, storedArrayPositionGap, storedBM25Params, storedDocumentsOf, survivorDocumentsOf, synonymModule, synonymSearchModule, toError2 as toError, topLevelMapKeys, updateDocument, updateDocumentBatch, updateDocumentVectors, validateBM25Params, validateBM25Shape, validateEmbeddingConfig, validateMemoryFractions, validatePartitionConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, visitTokens, withOpenIndex, workerIneligibility, workersEnabledByDefault, wrapFrozenTokenTable, writePartitionDocument, yieldToEventLoop };
|