@delali/narsil 0.3.1-next.183 → 0.3.1-next.185
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-ZZUXMJIK.mjs → chunk-CPSHS46N.mjs} +129 -165
- package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
- package/dist/browser/{optional-writes-IQFTXANH.mjs → optional-writes-ZTBHYQMN.mjs} +619 -314
- package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
- package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
- package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
- package/dist/browser/{storage-leaf-2CO6SQ7R.mjs → storage-leaf-4XMCITJ6.mjs} +27 -9
- package/dist/{chunk-GGG5R4CU.mjs → chunk-2GONI5HL.mjs} +5 -3
- package/dist/{chunk-PK3BGGHO.mjs → chunk-2KCCQK4R.mjs} +1 -1
- package/dist/{chunk-JVR5KMO4.mjs → chunk-2YWWHMVC.mjs} +2 -2
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/{react-native/chunks/chunk-G4YYO5TC.mjs → chunk-4JBTALUC.mjs} +1 -1
- package/dist/{chunk-PXEGKQHS.mjs → chunk-5BUEBIXR.mjs} +2 -2
- package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
- package/dist/{chunk-NPXORGMK.mjs → chunk-7WXGUTWK.mjs} +3 -3
- package/dist/{chunk-2DK74OQL.mjs → chunk-A3OAFUZI.mjs} +1 -1
- package/dist/{chunk-632ABEIR.mjs → chunk-BVZWWMOU.mjs} +1 -1
- package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
- package/dist/{chunk-F2UOTRY7.mjs → chunk-CODGHJVW.mjs} +4 -4
- package/dist/{chunk-6ICOWB6I.mjs → chunk-CYCVPTTR.mjs} +1 -1
- package/dist/{chunk-WUWSUJ2X.mjs → chunk-DU23J5JG.mjs} +5 -5
- package/dist/{chunk-C77OBNPP.mjs → chunk-EDEE7ETQ.mjs} +1 -1
- package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
- package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
- package/dist/{chunk-V7F5DOSC.mjs → chunk-FBQYC5OX.mjs} +3 -3
- package/dist/{chunk-EGWBZSHK.mjs → chunk-FCQMZDRI.mjs} +2 -2
- package/dist/{chunk-MCAJOLVK.mjs → chunk-G3XRHAJA.mjs} +2 -2
- package/dist/{chunk-OQANB7SJ.mjs → chunk-H4QYQ5DE.mjs} +1 -1
- package/dist/{chunk-CKS4QQEC.mjs → chunk-II6XF6M4.mjs} +5 -5
- package/dist/{chunk-C3EINJNJ.mjs → chunk-JDG6CCXM.mjs} +1 -1
- package/dist/{chunk-3OEXOSWI.mjs → chunk-LR6JJVML.mjs} +4 -4
- package/dist/{chunk-TTTBE4TB.mjs → chunk-M5I64AQW.mjs} +3 -3
- package/dist/{chunk-FIXUVFUC.mjs → chunk-MHBG33R5.mjs} +7 -6
- package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
- package/dist/{chunk-M3H7CTIC.mjs → chunk-O337MDRJ.mjs} +5 -5
- package/dist/{chunk-CWXR7OP4.mjs → chunk-SL53DHVE.mjs} +804 -527
- package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
- package/dist/{chunk-SA7U67XF.mjs → chunk-URVH6BLO.mjs} +2 -2
- package/dist/{chunk-T3YI4KN6.mjs → chunk-VYSWVQCE.mjs} +1 -1
- package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
- package/dist/{chunk-ZVZ2V2PZ.mjs → chunk-WSIU6YXQ.mjs} +39 -20
- package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
- package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
- package/dist/{direct-executor-JXYIL4OQ.mjs → direct-executor-37IH3ISS.mjs} +5 -4
- package/dist/distribution.mjs +254 -76
- package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +5 -4
- package/dist/index.react-native.mjs +4 -3
- package/dist/{long-query-thread-HPKGRYWM.mjs → long-query-thread-DIPJBMV7.mjs} +15 -14
- package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
- package/dist/{optional-writes-QE7QUI45.mjs → optional-writes-YIVJMQ4D.mjs} +7 -6
- package/dist/{paged-partition-archive-IK2UOY2G.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +5 -4
- package/dist/persistence/durability/checkpoint-worker.mjs +13 -12
- package/dist/{query-actions-COEFVUH3.mjs → query-actions-7EOKT3WD.mjs} +5 -4
- package/dist/query-suggestions/merge-worker.mjs +8 -7
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.mjs +6 -5
- package/dist/react-native/chunks/{chunk-DZ4FQW5K.mjs → chunk-7TPO4FLE.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-6A5XE2MA.mjs → chunk-CBU4M3EU.mjs} +803 -526
- package/dist/react-native/chunks/{chunk-DN2GVVR4.mjs → chunk-IEY4DARA.mjs} +1 -1
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
- package/dist/react-native/chunks/{chunk-UPN4NUXU.mjs → chunk-T5HF3FW3.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XNIH3WIB.mjs → chunk-ZQPSXN6C.mjs} +1 -1
- package/dist/{chunk-YWSIRKKQ.mjs → react-native/chunks/chunk-ZUFOYZ5P.mjs} +1 -1
- package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
- package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
- package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
- package/dist/react-native/chunks/{optional-writes-YZ7WTR7V.mjs → optional-writes-QETIM6BI.mjs} +5 -4
- package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
- package/dist/react-native/chunks/{restore-queue-IGKD6Q2Y.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
- package/dist/react-native/chunks/{runtime-BUCSAK4U.mjs → runtime-UOHD2XUJ.mjs} +23 -3
- package/dist/react-native/chunks/{segment-codec-leaf-JTTGDT3G.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
- package/dist/react-native/chunks/{segment-merge-leaf-D5ZFJRUB.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
- package/dist/react-native/chunks/{storage-leaf-ZKIM6IF6.mjs → storage-leaf-LNOLKWZB.mjs} +36 -17
- package/dist/react-native/chunks/{store-leaf-PHBOTT5J.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
- package/dist/react-native/chunks/{suggestions-leaf-R7MIH6Q3.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
- package/dist/react-native/chunks/{worker-copies-U35KPVJ7.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
- package/dist/react-native/partial-words.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +4 -3
- package/dist/react-native/sync.mjs +4 -3
- package/dist/react-native/synonyms.mjs +4 -3
- package/dist/{rerank-LYL3D7LU.mjs → rerank-KTV5L7MZ.mjs} +5 -4
- package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
- package/dist/{restore-queue-LF4DTVJC.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
- package/dist/{runtime-4GKUIAKV.mjs → runtime-USQYKN3I.mjs} +23 -3
- package/dist/{segment-codec-leaf-ACALKS2C.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
- package/dist/{segment-merge-leaf-SKMPASFK.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
- package/dist/server.mjs +22 -21
- package/dist/{storage-leaf-OJ55R4C5.mjs → storage-leaf-ZI3H2WJY.mjs} +26 -25
- package/dist/{store-leaf-CWHHEDRK.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
- package/dist/{suggestions-leaf-ZTU3ZTWJ.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
- package/dist/sync/filesystem.mjs +10 -9
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.mjs +5 -4
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +5 -4
- package/dist/{thread-BEGFRULV.mjs → thread-NQRNW3IH.mjs} +14 -13
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/{worker-copies-373P6IYZ.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
- package/dist/workers/entry.mjs +11 -10
- package/package.json +7 -7
- package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
- package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { reservedHeapBytes, reserveHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, admittedHeapBytes } from './chunk-
|
|
2
|
-
import {
|
|
1
|
+
import { reservedHeapBytes, reserveHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, admittedHeapBytes } from './chunk-BN6FUB7N.mjs';
|
|
2
|
+
import { fnv1aUtf16, changeIndexOutsideExternalWaits, turnsAreBudgeted, turnIsSpent, yieldTurn, createDocIdMap, detectRuntime, fnv1a, indexChangeOutsideExternalWaits, splitWorkerBudget, resolveWorkerCount, yieldBackgroundTurn, backgroundTurnIsSpent, FNV_OFFSET_BASIS, DEFAULT_REQUEST_TIMEOUT_MS, countWrite, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, answerFromOneState, signalOfSearch, requirePinnedState, stopOfSignal, fnv1aContinue, sharedStopOfSignal, requireSearchActive, fnv1aContinueTaggedNumbers, FNV_PRIME, collectAndReturnPages, threadSlotOfWorker, resolveRequestThreadCount, SHUTDOWN_TIMEOUT_MS, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, threadSlotGone, spawnNodeWorker, resolveWorkerEntry, noteThreadSlotLive, noteThreadSlotGone, bindSearchSignal, adoptSharedTurns, WORKER_YOUNG_GENERATION_MB, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_BLOCK_MAX_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_ATTACH_RETRY_MS, MAX_WASM_PAGES, MAX_VECTOR_ORDINALS, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_DOC_ID_TABLE_BYTES, LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, DISK_STORAGE_BLOCK_BYTES, waitOutsideCountedWrite, nodeThreadId } from './chunk-NLEFMPY3.mjs';
|
|
3
|
+
import { MIN_DOCUMENTS_FOR_SEGMENTS, MAX_PARTITIONS, MAX_INDEX_CONFIGURATION_BYTES, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_CALLS_ACROSS_INDEX_CHANGES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MIN_SWEEP_INTERVAL_MS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, DEFAULT_PAGE_SIZE, CURSOR_BINDING_BUFFER_BYTES, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, MIN_RANK_CONSTANT, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, FUSION_TIE_INSERTION_LIMIT, DEFAULT_RANK_CONSTANT, DEFAULT_LINEAR_ALPHA, STATISTICS_BROADCAST_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, EXPORT_CHUNK_BYTES, DEFAULT_MAX_PARTITIONS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, ANALYSIS_CHANGE_MEMBERS, MIN_HYBRID_DEPTH, MAX_SUGGEST_SCATTER_LIMIT, MAX_SUGGEST_LIMIT, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_SUGGEST_LIMIT, DEFAULT_MAX_PER_GROUP } from './chunk-LX62Y4B5.mjs';
|
|
3
4
|
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, bitsetHas, bitsetClear, storedText, cachedStoredFields, storedValuesOf, bitsetSet, discardStoredValues, createBitSet, installStoredFields, storedValueBytes, DEFAULT_PATTERN_WORK_CAP, bitsetIsEmpty, prepareStoredValues, fuzzyPrefixOf, boundedLevenshtein, checkedSize, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, buildStoredValues, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH, TYPO_CLAIM_SLOTS_PER_WORD, fuzzyTermMatches, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIterator, GrowableUint32, bitsetFromSet } from './chunk-SK3DPAI4.mjs';
|
|
4
5
|
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_NEEDS_ROOM, NATIVE_OK } from './chunk-WFQ6YQZA.mjs';
|
|
5
6
|
import { reactNativeHost } from './chunk-PVMEHBQO.mjs';
|
|
@@ -2722,11 +2723,11 @@ function captureFrozen(input, output2) {
|
|
|
2722
2723
|
spellings: captured.spellings === null ? null : copy(captured.spellings),
|
|
2723
2724
|
removalIndex: captured.removalIndex === null ? null : copy(captured.removalIndex)
|
|
2724
2725
|
};
|
|
2725
|
-
const
|
|
2726
|
-
return { raw, writable };
|
|
2726
|
+
const writable2 = copy(output2);
|
|
2727
|
+
return { raw, writable: writable2 };
|
|
2727
2728
|
}
|
|
2728
2729
|
var heldFrozen = /* @__PURE__ */ new WeakMap();
|
|
2729
|
-
function rawsOf({ raw, writable }) {
|
|
2730
|
+
function rawsOf({ raw, writable: writable2 }) {
|
|
2730
2731
|
const { terms, surfaces, spellings, removalIndex } = raw;
|
|
2731
2732
|
return [
|
|
2732
2733
|
raw.documentCount,
|
|
@@ -2751,16 +2752,16 @@ function rawsOf({ raw, writable }) {
|
|
|
2751
2752
|
removalIndex?.starts,
|
|
2752
2753
|
removalIndex?.rows,
|
|
2753
2754
|
removalIndex?.termOfSlot,
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2755
|
+
writable2.tombstones,
|
|
2756
|
+
writable2.tombstoneCount,
|
|
2757
|
+
writable2.removedDocuments,
|
|
2758
|
+
writable2.removedOccurrences,
|
|
2759
|
+
writable2.removedSurfaces,
|
|
2760
|
+
writable2.removedSpellings,
|
|
2761
|
+
writable2.pendingHeader,
|
|
2762
|
+
writable2.pendingDocuments,
|
|
2763
|
+
writable2.pendingTokens,
|
|
2764
|
+
writable2.pendingUnits
|
|
2764
2765
|
];
|
|
2765
2766
|
}
|
|
2766
2767
|
function stillHeld(raws, held) {
|
|
@@ -2776,7 +2777,7 @@ function readHeld(input, output2, captured) {
|
|
|
2776
2777
|
const raws = rawsOf(captured);
|
|
2777
2778
|
const known = heldFrozen.get(input);
|
|
2778
2779
|
if (known !== void 0 && known.output === output2 && stillHeld(raws, known)) return known;
|
|
2779
|
-
const { raw, writable } = captured;
|
|
2780
|
+
const { raw, writable: writable2 } = captured;
|
|
2780
2781
|
const regions = new Regions();
|
|
2781
2782
|
const documentCount = raw.documentCount;
|
|
2782
2783
|
requireSegment(Number.isInteger(documentCount) && documentCount >= 0 && documentCount <= 4294967295);
|
|
@@ -2804,16 +2805,16 @@ function readHeld(input, output2, captured) {
|
|
|
2804
2805
|
}
|
|
2805
2806
|
};
|
|
2806
2807
|
const state = {
|
|
2807
|
-
tombstones: regions.array(
|
|
2808
|
-
tombstoneCount: regions.array(
|
|
2809
|
-
removedDocuments: regions.array(
|
|
2810
|
-
removedOccurrences: regions.array(
|
|
2811
|
-
removedSurfaces: regions.array(
|
|
2812
|
-
removedSpellings: regions.array(
|
|
2813
|
-
pendingHeader: regions.array(
|
|
2814
|
-
pendingDocuments: regions.array(
|
|
2815
|
-
pendingTokens: regions.array(
|
|
2816
|
-
pendingUnits: regions.array(
|
|
2808
|
+
tombstones: regions.array(writable2.tombstones, "Uint32Array", Math.max(1, Math.ceil(documentCount / 32)), true),
|
|
2809
|
+
tombstoneCount: regions.array(writable2.tombstoneCount, "Uint32Array", 1, true),
|
|
2810
|
+
removedDocuments: regions.array(writable2.removedDocuments, "Uint32Array", terms.counts.length, true),
|
|
2811
|
+
removedOccurrences: regions.array(writable2.removedOccurrences, "Float64Array", terms.counts.length, true),
|
|
2812
|
+
removedSurfaces: regions.array(writable2.removedSurfaces, "Uint32Array", surfaces?.counts.length ?? 0, true),
|
|
2813
|
+
removedSpellings: regions.array(writable2.removedSpellings, "Uint32Array", spellings?.counts.length ?? 0, true),
|
|
2814
|
+
pendingHeader: regions.array(writable2.pendingHeader, "Uint32Array", PENDING_HEADER_WORDS, true),
|
|
2815
|
+
pendingDocuments: regions.array(writable2.pendingDocuments, "Uint32Array", void 0, true),
|
|
2816
|
+
pendingTokens: regions.array(writable2.pendingTokens, "Uint32Array", void 0, true),
|
|
2817
|
+
pendingUnits: regions.array(writable2.pendingUnits, "Uint16Array", void 0, true)
|
|
2817
2818
|
};
|
|
2818
2819
|
requireSegment(state.pendingTokens.length % PENDING_TOKEN_WORDS === 0);
|
|
2819
2820
|
regions.check();
|
|
@@ -5486,6 +5487,7 @@ function searchHostPart() {
|
|
|
5486
5487
|
listBindingOf,
|
|
5487
5488
|
MAX_HYBRID_DEPTH,
|
|
5488
5489
|
MAX_PHRASE_WORDS,
|
|
5490
|
+
MAX_CALLS_ACROSS_INDEX_CHANGES,
|
|
5489
5491
|
MAX_QUERY_PHRASES,
|
|
5490
5492
|
MAX_RERANK_TIMEOUT_MS,
|
|
5491
5493
|
MAX_RERANK_WINDOW,
|
|
@@ -6184,10 +6186,10 @@ var Regions2 = class {
|
|
|
6184
6186
|
starts = [];
|
|
6185
6187
|
ends = [];
|
|
6186
6188
|
writable = [];
|
|
6187
|
-
read(input, kind, length,
|
|
6189
|
+
read(input, kind, length, writable2 = true) {
|
|
6188
6190
|
const value = typedRegion(input, kind);
|
|
6189
6191
|
requireSegment(value !== null && (length === void 0 || value.length === length));
|
|
6190
|
-
if (
|
|
6192
|
+
if (writable2) this.writes.push(value);
|
|
6191
6193
|
else this.reads.push(value);
|
|
6192
6194
|
const byteLength = value.byteLength;
|
|
6193
6195
|
if (byteLength > 0) {
|
|
@@ -6195,19 +6197,19 @@ var Regions2 = class {
|
|
|
6195
6197
|
this.buffers.push(value.buffer);
|
|
6196
6198
|
this.starts.push(start);
|
|
6197
6199
|
this.ends.push(start + byteLength);
|
|
6198
|
-
this.writable.push(
|
|
6200
|
+
this.writable.push(writable2);
|
|
6199
6201
|
}
|
|
6200
6202
|
return value;
|
|
6201
6203
|
}
|
|
6202
6204
|
check() {
|
|
6203
|
-
const { buffers, starts, ends, writable } = this;
|
|
6205
|
+
const { buffers, starts, ends, writable: writable2 } = this;
|
|
6204
6206
|
for (let at = 1; at < buffers.length; at++) {
|
|
6205
6207
|
const buffer = buffers[at];
|
|
6206
6208
|
const start = starts[at];
|
|
6207
6209
|
const end = ends[at];
|
|
6208
|
-
const writes =
|
|
6210
|
+
const writes = writable2[at];
|
|
6209
6211
|
for (let previous = 0; previous < at; previous++)
|
|
6210
|
-
if (buffers[previous] === buffer && (writes ||
|
|
6212
|
+
if (buffers[previous] === buffer && (writes || writable2[previous]) && start < ends[previous] && starts[previous] < end)
|
|
6211
6213
|
requireSegment(false);
|
|
6212
6214
|
}
|
|
6213
6215
|
}
|
|
@@ -8677,7 +8679,7 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
8677
8679
|
|
|
8678
8680
|
// src/core/segment-codec-code.ts
|
|
8679
8681
|
var segmentCodec = codeLoader(async () => {
|
|
8680
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-
|
|
8682
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-IE4DXUZV.mjs')]);
|
|
8681
8683
|
code7.bindSegmentCodecHost(optionalCodeHost(), msgpack);
|
|
8682
8684
|
return code7;
|
|
8683
8685
|
}, "segment codec");
|
|
@@ -8688,7 +8690,7 @@ var segmentMerge = codeLoader(async () => {
|
|
|
8688
8690
|
const [msgpack, codec, code7] = await Promise.all([
|
|
8689
8691
|
loadMessagePack(),
|
|
8690
8692
|
loadSegmentCodec(),
|
|
8691
|
-
import('./segment-merge-leaf-
|
|
8693
|
+
import('./segment-merge-leaf-62LQ5WZB.mjs')
|
|
8692
8694
|
]);
|
|
8693
8695
|
code7.bindSegmentMergeHost(optionalCodeHost(), msgpack, codec);
|
|
8694
8696
|
return code7;
|
|
@@ -8697,7 +8699,7 @@ var loadSegmentMerge = segmentMerge.load;
|
|
|
8697
8699
|
|
|
8698
8700
|
// src/query-suggestions/store-code.ts
|
|
8699
8701
|
var suggestionStore = codeLoader(async () => {
|
|
8700
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-
|
|
8702
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-Y7CBOPFZ.mjs')]);
|
|
8701
8703
|
code7.bindSuggestionStoreHost(optionalCodeHost(), msgpack);
|
|
8702
8704
|
return code7;
|
|
8703
8705
|
}, "query suggestions store");
|
|
@@ -8756,7 +8758,7 @@ function markSimdUnavailable() {
|
|
|
8756
8758
|
|
|
8757
8759
|
// src/workers/vector-core-code.ts
|
|
8758
8760
|
var vectorCore = codeLoader(
|
|
8759
|
-
() => import('./core-leaf-
|
|
8761
|
+
() => import('./core-leaf-TPBRRWVN.mjs').then((code7) => {
|
|
8760
8762
|
code7.bindVectorCoreHost(optionalCodeHost());
|
|
8761
8763
|
if (!code7.simdModuleLoaded()) markSimdUnavailable();
|
|
8762
8764
|
return code7;
|
|
@@ -8778,7 +8780,7 @@ var storageCode = codeLoader(async () => {
|
|
|
8778
8780
|
loadSegmentMerge(),
|
|
8779
8781
|
loadSuggestionStore(),
|
|
8780
8782
|
loadVectorCore(),
|
|
8781
|
-
import('./storage-leaf-
|
|
8783
|
+
import('./storage-leaf-LNOLKWZB.mjs')
|
|
8782
8784
|
]);
|
|
8783
8785
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
8784
8786
|
return code7;
|
|
@@ -13631,10 +13633,10 @@ function uint323(value) {
|
|
|
13631
13633
|
var MergeRegions = class _MergeRegions {
|
|
13632
13634
|
reads = [];
|
|
13633
13635
|
writes = [];
|
|
13634
|
-
read(input, kind, length,
|
|
13636
|
+
read(input, kind, length, writable2 = false) {
|
|
13635
13637
|
const value = typedRegion(input, kind);
|
|
13636
13638
|
requireSegment(value !== null && (length === void 0 || value.length === length));
|
|
13637
|
-
(
|
|
13639
|
+
(writable2 ? this.writes : this.reads).push(value);
|
|
13638
13640
|
return value;
|
|
13639
13641
|
}
|
|
13640
13642
|
check() {
|
|
@@ -13652,11 +13654,11 @@ var MergeRegions = class _MergeRegions {
|
|
|
13652
13654
|
}
|
|
13653
13655
|
}
|
|
13654
13656
|
}
|
|
13655
|
-
strings(input, stride,
|
|
13656
|
-
const counts = this.read(input.counts, "Uint32Array", void 0,
|
|
13657
|
-
const text = this.read(input.text, "Uint8Array", void 0,
|
|
13658
|
-
const offsets = this.read(input.offsets, "Uint32Array",
|
|
13659
|
-
if (!
|
|
13657
|
+
strings(input, stride, writable2 = false) {
|
|
13658
|
+
const counts = this.read(input.counts, "Uint32Array", void 0, writable2);
|
|
13659
|
+
const text = this.read(input.text, "Uint8Array", void 0, writable2);
|
|
13660
|
+
const offsets = this.read(input.offsets, "Uint32Array", writable2 ? void 0 : counts.length * stride + 1, writable2);
|
|
13661
|
+
if (!writable2) requireSegment(counts.length <= 4294967295 && text.length <= 4294967295);
|
|
13660
13662
|
return { text, offsets, counts };
|
|
13661
13663
|
}
|
|
13662
13664
|
source(input) {
|
|
@@ -13702,11 +13704,11 @@ var MergeRegions = class _MergeRegions {
|
|
|
13702
13704
|
...positions2
|
|
13703
13705
|
};
|
|
13704
13706
|
}
|
|
13705
|
-
positions(input,
|
|
13707
|
+
positions(input, writable2, count) {
|
|
13706
13708
|
requireSegment(input.positionOffsets === null === (input.positionValues === null));
|
|
13707
|
-
const positionOffsets = input.positionOffsets === null ? null : this.read(input.positionOffsets, "Uint32Array", count,
|
|
13708
|
-
const positionValues = input.positionValues === null ? null : this.read(input.positionValues, "Uint32Array", void 0,
|
|
13709
|
-
if (!
|
|
13709
|
+
const positionOffsets = input.positionOffsets === null ? null : this.read(input.positionOffsets, "Uint32Array", count, writable2);
|
|
13710
|
+
const positionValues = input.positionValues === null ? null : this.read(input.positionValues, "Uint32Array", void 0, writable2);
|
|
13711
|
+
if (!writable2) requireSegment(positionValues === null || positionValues.length <= 4294967295);
|
|
13710
13712
|
return { positionOffsets, positionValues };
|
|
13711
13713
|
}
|
|
13712
13714
|
};
|
|
@@ -19941,7 +19943,7 @@ function hasLanguage(name) {
|
|
|
19941
19943
|
|
|
19942
19944
|
// src/narsil/restore-queue-code.ts
|
|
19943
19945
|
var restoreQueue = codeLoader(async () => {
|
|
19944
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
19946
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-3E2MJIJS.mjs')]);
|
|
19945
19947
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
19946
19948
|
return code7;
|
|
19947
19949
|
}, "restore");
|
|
@@ -20163,7 +20165,7 @@ function storedBM25Params(params) {
|
|
|
20163
20165
|
|
|
20164
20166
|
// src/workers/vector-code.ts
|
|
20165
20167
|
var vectorCode = codeLoader(async () => {
|
|
20166
|
-
const [vectorCore2, code7] = await Promise.all([loadVectorCore(), import('./module-
|
|
20168
|
+
const [vectorCore2, code7] = await Promise.all([loadVectorCore(), import('./module-COLNJNML.mjs')]);
|
|
20167
20169
|
code7.bindVectorHost(optionalCodeHost(), vectorCore2);
|
|
20168
20170
|
return code7;
|
|
20169
20171
|
}, "vector");
|
|
@@ -20531,7 +20533,7 @@ function storedArrayPositionGap(value) {
|
|
|
20531
20533
|
|
|
20532
20534
|
// src/engine/embed-code.ts
|
|
20533
20535
|
var embedCode = codeLoader(
|
|
20534
|
-
() => import('./embed-
|
|
20536
|
+
() => import('./embed-4LBSQD7U.mjs').then((code7) => {
|
|
20535
20537
|
code7.bindEmbedHost(optionalCodeHost());
|
|
20536
20538
|
return code7;
|
|
20537
20539
|
}),
|
|
@@ -20541,7 +20543,7 @@ var loadEmbedCode = embedCode.load;
|
|
|
20541
20543
|
|
|
20542
20544
|
// src/engine/optional-writes-code.ts
|
|
20543
20545
|
var optionalWrites = codeLoader(async () => {
|
|
20544
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
20546
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-QETIM6BI.mjs')]);
|
|
20545
20547
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
20546
20548
|
return code7;
|
|
20547
20549
|
}, "write");
|
|
@@ -20745,21 +20747,9 @@ function requireMatchingDocId(docId, document) {
|
|
|
20745
20747
|
{ docId, documentId: embeddedDocId }
|
|
20746
20748
|
);
|
|
20747
20749
|
}
|
|
20748
|
-
function asBatchInsertError(err) {
|
|
20749
|
-
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
20750
|
-
}
|
|
20751
|
-
function createPendingAdmissions() {
|
|
20752
|
-
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
20753
|
-
}
|
|
20754
20750
|
function capsEachPartition(ctx, manager) {
|
|
20755
20751
|
return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
|
|
20756
20752
|
}
|
|
20757
|
-
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
20758
|
-
pending.total += 1;
|
|
20759
|
-
if (!capsEachPartition(ctx, manager)) return;
|
|
20760
|
-
const partitionId = manager.routePartition(docId);
|
|
20761
|
-
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
20762
|
-
}
|
|
20763
20753
|
function admitInsert(ctx, indexName, manager, docId, pending) {
|
|
20764
20754
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
20765
20755
|
if (rebalancing) {
|
|
@@ -20791,66 +20781,6 @@ function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
|
|
|
20791
20781
|
ctx.rebalanceTargetPartitionCount(indexName)
|
|
20792
20782
|
);
|
|
20793
20783
|
}
|
|
20794
|
-
function collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed) {
|
|
20795
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
20796
|
-
try {
|
|
20797
|
-
validateRequiredFields(documents2[i], required);
|
|
20798
|
-
} catch (err) {
|
|
20799
|
-
chunkFailedIndexes.add(i);
|
|
20800
|
-
failed.push({ docId: providedDocId(documents2[i]) ?? "", error: asBatchInsertError(err) });
|
|
20801
|
-
}
|
|
20802
|
-
}
|
|
20803
|
-
}
|
|
20804
|
-
async function embedChunkDocuments(entry, documents2, chunkStart, chunkEnd, signal, chunkFailedIndexes, failed) {
|
|
20805
|
-
if (entry.config.embedding === void 0) return;
|
|
20806
|
-
const { assertDocumentCarriesMappedVectors, embedBatchDocumentFields } = await loadEmbedCode();
|
|
20807
|
-
if (entry.embeddingAdapter === null) {
|
|
20808
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
20809
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
20810
|
-
try {
|
|
20811
|
-
assertDocumentCarriesMappedVectors(
|
|
20812
|
-
documents2[i],
|
|
20813
|
-
entry.config.embedding,
|
|
20814
|
-
entry.embeddingAdapterName
|
|
20815
|
-
);
|
|
20816
|
-
} catch (err) {
|
|
20817
|
-
chunkFailedIndexes.add(i);
|
|
20818
|
-
failed.push({
|
|
20819
|
-
docId: providedDocId(documents2[i]) ?? "",
|
|
20820
|
-
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
20821
|
-
});
|
|
20822
|
-
}
|
|
20823
|
-
}
|
|
20824
|
-
return;
|
|
20825
|
-
}
|
|
20826
|
-
const embeddableSlice = [];
|
|
20827
|
-
const embeddableOriginalIndexes = [];
|
|
20828
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
20829
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
20830
|
-
embeddableSlice.push(documents2[i]);
|
|
20831
|
-
embeddableOriginalIndexes.push(i);
|
|
20832
|
-
}
|
|
20833
|
-
if (embeddableSlice.length === 0) return;
|
|
20834
|
-
try {
|
|
20835
|
-
const embedResult = await embedBatchDocumentFields(
|
|
20836
|
-
embeddableSlice,
|
|
20837
|
-
entry.config.embedding,
|
|
20838
|
-
entry.embeddingAdapter,
|
|
20839
|
-
signal
|
|
20840
|
-
);
|
|
20841
|
-
for (const [sliceIndex, error] of embedResult.failed) {
|
|
20842
|
-
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
20843
|
-
chunkFailedIndexes.add(originalIdx);
|
|
20844
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error });
|
|
20845
|
-
}
|
|
20846
|
-
} catch (err) {
|
|
20847
|
-
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
20848
|
-
for (const originalIdx of embeddableOriginalIndexes) {
|
|
20849
|
-
chunkFailedIndexes.add(originalIdx);
|
|
20850
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error: embeddingError });
|
|
20851
|
-
}
|
|
20852
|
-
}
|
|
20853
|
-
}
|
|
20854
20784
|
|
|
20855
20785
|
// src/engine/mutations/write-visibility.ts
|
|
20856
20786
|
async function awaitWriteVisibility(ctx, indexName) {
|
|
@@ -20859,33 +20789,19 @@ async function awaitWriteVisibility(ctx, indexName) {
|
|
|
20859
20789
|
}
|
|
20860
20790
|
|
|
20861
20791
|
// src/engine/mutations/insert.ts
|
|
20862
|
-
async function insertDocument(ctx, indexName, document, docId, options) {
|
|
20792
|
+
async function insertDocument(ctx, indexName, document, docId, options, hold) {
|
|
20863
20793
|
ctx.guardShutdown();
|
|
20864
|
-
|
|
20794
|
+
let entry = ctx.requireIndex(indexName);
|
|
20865
20795
|
if (docId !== void 0) requireMatchingDocId(docId, document);
|
|
20866
20796
|
const resolvedDocId = docId ?? providedDocId(document) ?? ctx.idGenerator();
|
|
20867
20797
|
validateDocId(resolvedDocId);
|
|
20868
|
-
|
|
20869
|
-
if (
|
|
20798
|
+
const beforeInsert = ctx.pluginRegistry.hasHooks("beforeInsert") ? () => ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document }) : null;
|
|
20799
|
+
if (beforeInsert !== null || entry.config.embedding !== void 0) {
|
|
20800
|
+
const { prepareDocumentWrite: prepareDocumentWrite2 } = await loadOptionalWrites();
|
|
20801
|
+
entry = await prepareDocumentWrite2(ctx, indexName, document, hold, beforeInsert);
|
|
20802
|
+
} else if (entry.config.required !== void 0) {
|
|
20870
20803
|
validateRequiredFields(document, entry.config.required);
|
|
20871
20804
|
}
|
|
20872
|
-
if (entry.config.embedding) {
|
|
20873
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
20874
|
-
if (entry.embeddingAdapter) {
|
|
20875
|
-
await embedDocumentFields(
|
|
20876
|
-
document,
|
|
20877
|
-
entry.config.embedding,
|
|
20878
|
-
entry.embeddingAdapter,
|
|
20879
|
-
ctx.abortController.signal
|
|
20880
|
-
);
|
|
20881
|
-
} else {
|
|
20882
|
-
assertDocumentCarriesMappedVectors(
|
|
20883
|
-
document,
|
|
20884
|
-
entry.config.embedding,
|
|
20885
|
-
entry.embeddingAdapterName
|
|
20886
|
-
);
|
|
20887
|
-
}
|
|
20888
|
-
}
|
|
20889
20805
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
20890
20806
|
if (sizing !== null) await sizing;
|
|
20891
20807
|
const insertManager = ctx.requireManager(indexName);
|
|
@@ -20965,16 +20881,18 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
20965
20881
|
return resolvedDocId;
|
|
20966
20882
|
}
|
|
20967
20883
|
await ctx.orchestrator.settlePublications(indexName);
|
|
20968
|
-
|
|
20969
|
-
|
|
20970
|
-
|
|
20971
|
-
|
|
20884
|
+
if (ctx.pluginRegistry.hasHooks("afterInsert")) {
|
|
20885
|
+
const runHook = async () => {
|
|
20886
|
+
try {
|
|
20887
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: resolvedDocId, document });
|
|
20888
|
+
} catch (err) {
|
|
20889
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
20890
|
+
}
|
|
20891
|
+
};
|
|
20892
|
+
await (hold === void 0 ? runHook() : hold.outside(runHook));
|
|
20972
20893
|
}
|
|
20973
|
-
|
|
20974
|
-
const
|
|
20975
|
-
if (vecIndex) {
|
|
20976
|
-
vecIndex.scheduleBuild();
|
|
20977
|
-
}
|
|
20894
|
+
if (ctx.executor.getManager(indexName) === insertManager) {
|
|
20895
|
+
for (const fieldPath of extractedVectors.keys()) insertVecIndexes.get(fieldPath)?.scheduleBuild();
|
|
20978
20896
|
}
|
|
20979
20897
|
ctx.checkWatermark(indexName, resolvedDocId);
|
|
20980
20898
|
ctx.checkHeapPressure(indexName);
|
|
@@ -21089,6 +21007,326 @@ async function oneAtATime(core2, indexName, action) {
|
|
|
21089
21007
|
function bindOptionalWritesHost(_host, _segmentCodec) {
|
|
21090
21008
|
}
|
|
21091
21009
|
|
|
21010
|
+
// src/engine/mutations/batch-admissions.ts
|
|
21011
|
+
function createPendingAdmissions() {
|
|
21012
|
+
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
21013
|
+
}
|
|
21014
|
+
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
21015
|
+
pending.total += 1;
|
|
21016
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
21017
|
+
const partitionId = manager.routePartition(docId);
|
|
21018
|
+
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
21019
|
+
}
|
|
21020
|
+
function asBatchInsertError(err) {
|
|
21021
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
21022
|
+
}
|
|
21023
|
+
|
|
21024
|
+
// src/engine/mutations/outside-hold.ts
|
|
21025
|
+
function runOutside(hold, work) {
|
|
21026
|
+
return hold === void 0 ? work() : hold.outside(work);
|
|
21027
|
+
}
|
|
21028
|
+
|
|
21029
|
+
// src/engine/mutations/write-preparation.ts
|
|
21030
|
+
function settingsOf(entry) {
|
|
21031
|
+
return { required: entry.config.required, embedding: entry.config.embedding, adapter: entry.embeddingAdapter };
|
|
21032
|
+
}
|
|
21033
|
+
function embedsThroughAdapter(entry) {
|
|
21034
|
+
return entry.config.embedding !== void 0 && entry.embeddingAdapter !== null;
|
|
21035
|
+
}
|
|
21036
|
+
function sourcesOf(sources2) {
|
|
21037
|
+
return typeof sources2 === "string" ? [sources2] : sources2;
|
|
21038
|
+
}
|
|
21039
|
+
function sameMapping(left, right) {
|
|
21040
|
+
if (left === right) return true;
|
|
21041
|
+
const targets = Object.keys(left);
|
|
21042
|
+
if (targets.length !== Object.keys(right).length) return false;
|
|
21043
|
+
for (const target of targets) {
|
|
21044
|
+
if (!Object.hasOwn(right, target)) return false;
|
|
21045
|
+
const leftSources = sourcesOf(left[target]);
|
|
21046
|
+
const rightSources = sourcesOf(right[target]);
|
|
21047
|
+
if (leftSources.length !== rightSources.length) return false;
|
|
21048
|
+
if (leftSources.some((source, position) => source !== rightSources[position])) return false;
|
|
21049
|
+
}
|
|
21050
|
+
return true;
|
|
21051
|
+
}
|
|
21052
|
+
function embeddingFits(entry, embedding) {
|
|
21053
|
+
const embeddingConfig = entry.config.embedding;
|
|
21054
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter === null) return embedding === null;
|
|
21055
|
+
return embedding !== null && embedding.adapter === entry.embeddingAdapter && sameMapping(embedding.mapping, embeddingConfig.fields);
|
|
21056
|
+
}
|
|
21057
|
+
function embeddingOfDocuments(embedding, documents2) {
|
|
21058
|
+
if (embedding === null) return null;
|
|
21059
|
+
const fields = /* @__PURE__ */ new Map();
|
|
21060
|
+
for (const document of documents2) {
|
|
21061
|
+
const embedded2 = embedding.fields.get(document);
|
|
21062
|
+
if (embedded2 !== void 0) fields.set(document, embedded2);
|
|
21063
|
+
}
|
|
21064
|
+
return { adapter: embedding.adapter, mapping: embedding.mapping, fields };
|
|
21065
|
+
}
|
|
21066
|
+
function writable(preparations) {
|
|
21067
|
+
return preparations.filter((preparation) => preparation.failure === void 0);
|
|
21068
|
+
}
|
|
21069
|
+
function failAll(preparations, failure) {
|
|
21070
|
+
for (const preparation of writable(preparations)) preparation.failure = failure;
|
|
21071
|
+
}
|
|
21072
|
+
function requireFieldsOf(required, preparations) {
|
|
21073
|
+
if (required === void 0 || required.length === 0) return;
|
|
21074
|
+
for (const preparation of writable(preparations)) {
|
|
21075
|
+
try {
|
|
21076
|
+
validateRequiredFields(preparation.document, required);
|
|
21077
|
+
} catch (err) {
|
|
21078
|
+
preparation.failure = err;
|
|
21079
|
+
}
|
|
21080
|
+
}
|
|
21081
|
+
}
|
|
21082
|
+
async function requireMappedVectors(entry, preparations) {
|
|
21083
|
+
const embeddingConfig = entry.config.embedding;
|
|
21084
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter !== null) return;
|
|
21085
|
+
const { assertDocumentCarriesMappedVectors } = await loadEmbedCode();
|
|
21086
|
+
for (const preparation of writable(preparations)) {
|
|
21087
|
+
try {
|
|
21088
|
+
assertDocumentCarriesMappedVectors(
|
|
21089
|
+
preparation.document,
|
|
21090
|
+
embeddingConfig,
|
|
21091
|
+
entry.embeddingAdapterName
|
|
21092
|
+
);
|
|
21093
|
+
} catch (err) {
|
|
21094
|
+
preparation.failure = err;
|
|
21095
|
+
}
|
|
21096
|
+
}
|
|
21097
|
+
}
|
|
21098
|
+
async function embedWith(settings, signal, preparations, earlier, together) {
|
|
21099
|
+
requireFieldsOf(settings.required, preparations);
|
|
21100
|
+
const { clearWriteEmbedding, embedDocumentsForWrite, lacksMappedVectors } = await loadEmbedCode();
|
|
21101
|
+
if (earlier !== null) clearWriteEmbedding(earlier);
|
|
21102
|
+
const { embedding: embeddingConfig, adapter } = settings;
|
|
21103
|
+
const embeddable = writable(preparations);
|
|
21104
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
21105
|
+
if (embeddingConfig === void 0 || adapter === null || embeddable.length === 0) {
|
|
21106
|
+
return { embedding: null, called: false, documentFailures };
|
|
21107
|
+
}
|
|
21108
|
+
const fields = /* @__PURE__ */ new Map();
|
|
21109
|
+
const embedding = { adapter, mapping: embeddingConfig.fields, fields };
|
|
21110
|
+
const documents2 = embeddable.map((preparation) => preparation.document);
|
|
21111
|
+
if (!lacksMappedVectors(documents2, embeddingConfig)) return { embedding, called: false, documentFailures };
|
|
21112
|
+
for (let chunkStart = 0; chunkStart < embeddable.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21113
|
+
const chunk = embeddable.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
21114
|
+
const embedded2 = await embedDocumentsForWrite(
|
|
21115
|
+
chunk.map((preparation) => preparation.document),
|
|
21116
|
+
embeddingConfig,
|
|
21117
|
+
adapter,
|
|
21118
|
+
signal,
|
|
21119
|
+
together
|
|
21120
|
+
);
|
|
21121
|
+
for (const [document, embeddedFields] of embedded2.embedding.fields) fields.set(document, embeddedFields);
|
|
21122
|
+
for (const [position, error] of embedded2.failed) chunk[position].failure = error;
|
|
21123
|
+
for (const [position, error] of embedded2.documentFailures) documentFailures.set(chunk[position], error);
|
|
21124
|
+
}
|
|
21125
|
+
return { embedding, called: true, documentFailures };
|
|
21126
|
+
}
|
|
21127
|
+
function settleDocumentFailures(entry, pass) {
|
|
21128
|
+
if (!embeddingFits(entry, pass.embedding)) return [...pass.documentFailures.keys()];
|
|
21129
|
+
for (const [preparation, failure] of pass.documentFailures) preparation.failure = failure;
|
|
21130
|
+
return [];
|
|
21131
|
+
}
|
|
21132
|
+
async function runBeforeWrites(preparations) {
|
|
21133
|
+
for (const preparation of preparations) {
|
|
21134
|
+
if (preparation.beforeWrite === null || preparation.failure !== void 0) continue;
|
|
21135
|
+
try {
|
|
21136
|
+
await preparation.beforeWrite();
|
|
21137
|
+
} catch (err) {
|
|
21138
|
+
preparation.failure = err;
|
|
21139
|
+
}
|
|
21140
|
+
}
|
|
21141
|
+
}
|
|
21142
|
+
async function embedThenRunHooks(settings, signal, preparations, earlier) {
|
|
21143
|
+
const pass = await embedWith(settings, signal, preparations, earlier, true);
|
|
21144
|
+
await runBeforeWrites(preparations.filter((preparation) => !pass.documentFailures.has(preparation)));
|
|
21145
|
+
return pass;
|
|
21146
|
+
}
|
|
21147
|
+
async function runHooksThenEmbedEach(ctx, indexName, preparations, earlier) {
|
|
21148
|
+
let settings = null;
|
|
21149
|
+
let embedding = null;
|
|
21150
|
+
let called = false;
|
|
21151
|
+
let uncleared = earlier;
|
|
21152
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
21153
|
+
for (const preparation of preparations) {
|
|
21154
|
+
await runBeforeWrites([preparation]);
|
|
21155
|
+
settings ??= settingsOf(ctx.requireIndex(indexName));
|
|
21156
|
+
const pass = await embedWith(settings, ctx.abortController.signal, [preparation], uncleared, false);
|
|
21157
|
+
uncleared = null;
|
|
21158
|
+
called ||= pass.called;
|
|
21159
|
+
for (const [failed, failure] of pass.documentFailures) documentFailures.set(failed, failure);
|
|
21160
|
+
if (pass.embedding === null) continue;
|
|
21161
|
+
if (embedding === null) embedding = pass.embedding;
|
|
21162
|
+
else for (const [document, fields] of pass.embedding.fields) embedding.fields.set(document, fields);
|
|
21163
|
+
}
|
|
21164
|
+
return { embedding, called, documentFailures };
|
|
21165
|
+
}
|
|
21166
|
+
function adapterChangesRefusal(indexName, embeddings) {
|
|
21167
|
+
return new NarsilError(
|
|
21168
|
+
ErrorCodes.WORKER_BUSY,
|
|
21169
|
+
`Changes to index "${indexName}" replaced its embedding adapter after each of ${embeddings} embeddings of this write`,
|
|
21170
|
+
{ indexName, embeddings }
|
|
21171
|
+
);
|
|
21172
|
+
}
|
|
21173
|
+
async function prepareDocumentWrites(ctx, indexName, preparations, hold, together, earlier = null) {
|
|
21174
|
+
let entry = ctx.requireIndex(indexName);
|
|
21175
|
+
let embedding = earlier;
|
|
21176
|
+
if (writable(preparations).length === 0) return { entry, embedding };
|
|
21177
|
+
const signal = ctx.abortController.signal;
|
|
21178
|
+
let embeddings = 0;
|
|
21179
|
+
let unhooked = [];
|
|
21180
|
+
const hooked = preparations.some((preparation) => preparation.beforeWrite !== null);
|
|
21181
|
+
if (hooked || embedsThroughAdapter(entry) && !embeddingFits(entry, embedding)) {
|
|
21182
|
+
if (together && !embedsThroughAdapter(entry)) {
|
|
21183
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
21184
|
+
await requireMappedVectors(entry, preparations);
|
|
21185
|
+
}
|
|
21186
|
+
const settings = settingsOf(entry);
|
|
21187
|
+
const uncleared = embedding;
|
|
21188
|
+
const pass = await runOutside(
|
|
21189
|
+
hold,
|
|
21190
|
+
() => together ? embedThenRunHooks(settings, signal, preparations, uncleared) : runHooksThenEmbedEach(ctx, indexName, preparations, uncleared)
|
|
21191
|
+
);
|
|
21192
|
+
embedding = pass.embedding;
|
|
21193
|
+
if (pass.called) embeddings += 1;
|
|
21194
|
+
entry = ctx.requireIndex(indexName);
|
|
21195
|
+
unhooked = settleDocumentFailures(entry, pass);
|
|
21196
|
+
}
|
|
21197
|
+
while (writable(preparations).length > 0 && !embeddingFits(entry, embedding)) {
|
|
21198
|
+
if (embeddings >= MAX_CALLS_ACROSS_INDEX_CHANGES) {
|
|
21199
|
+
failAll(preparations, adapterChangesRefusal(indexName, embeddings));
|
|
21200
|
+
break;
|
|
21201
|
+
}
|
|
21202
|
+
const uncleared = embedding;
|
|
21203
|
+
const settings = settingsOf(entry);
|
|
21204
|
+
const hooking = unhooked;
|
|
21205
|
+
const pass = await runOutside(hold, async () => {
|
|
21206
|
+
const embedded2 = await embedWith(settings, signal, preparations, uncleared, together);
|
|
21207
|
+
await runBeforeWrites(hooking.filter((preparation) => !embedded2.documentFailures.has(preparation)));
|
|
21208
|
+
return embedded2;
|
|
21209
|
+
});
|
|
21210
|
+
embedding = pass.embedding;
|
|
21211
|
+
if (pass.called) embeddings += 1;
|
|
21212
|
+
entry = ctx.requireIndex(indexName);
|
|
21213
|
+
const stale = settleDocumentFailures(entry, pass);
|
|
21214
|
+
unhooked = hooking.filter((preparation) => stale.includes(preparation));
|
|
21215
|
+
}
|
|
21216
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
21217
|
+
await requireMappedVectors(entry, preparations);
|
|
21218
|
+
return { entry, embedding };
|
|
21219
|
+
}
|
|
21220
|
+
async function prepareDocumentWrite(ctx, indexName, document, hold, beforeWrite) {
|
|
21221
|
+
const preparation = { document, beforeWrite, failure: void 0 };
|
|
21222
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
21223
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
21224
|
+
return entry;
|
|
21225
|
+
}
|
|
21226
|
+
|
|
21227
|
+
// src/engine/mutations/insert-batch-admission.ts
|
|
21228
|
+
function pendingInsertOf(ctx, indexName, docId, document, hooked) {
|
|
21229
|
+
const pending = {
|
|
21230
|
+
docId,
|
|
21231
|
+
preparation: { document, beforeWrite: null, failure: void 0 },
|
|
21232
|
+
refusedByHook: false
|
|
21233
|
+
};
|
|
21234
|
+
if (hooked && ctx.pluginRegistry.hasHooks("beforeInsert")) {
|
|
21235
|
+
pending.preparation.beforeWrite = async () => {
|
|
21236
|
+
try {
|
|
21237
|
+
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document });
|
|
21238
|
+
} catch (err) {
|
|
21239
|
+
pending.refusedByHook = true;
|
|
21240
|
+
throw err;
|
|
21241
|
+
}
|
|
21242
|
+
};
|
|
21243
|
+
}
|
|
21244
|
+
return pending;
|
|
21245
|
+
}
|
|
21246
|
+
async function prepareInsertDocuments(ctx, indexName, documents2, start, end, hold, hooked, failed, earlier = null) {
|
|
21247
|
+
const pending = [];
|
|
21248
|
+
for (let i = start; i < end; i++) {
|
|
21249
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
21250
|
+
if (ctx.abortController.signal.aborted) break;
|
|
21251
|
+
const document = documents2[i];
|
|
21252
|
+
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
21253
|
+
try {
|
|
21254
|
+
const sizing = requireDocumentObject(document);
|
|
21255
|
+
if (sizing !== null) await sizing;
|
|
21256
|
+
validateDocId(docId);
|
|
21257
|
+
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
21258
|
+
} catch (err) {
|
|
21259
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
21260
|
+
}
|
|
21261
|
+
}
|
|
21262
|
+
const { entry, embedding } = await prepareDocumentWrites(
|
|
21263
|
+
ctx,
|
|
21264
|
+
indexName,
|
|
21265
|
+
pending.map((item) => item.preparation),
|
|
21266
|
+
hold,
|
|
21267
|
+
true,
|
|
21268
|
+
earlier
|
|
21269
|
+
);
|
|
21270
|
+
const candidates2 = [];
|
|
21271
|
+
for (const item of pending) {
|
|
21272
|
+
const { document, failure } = item.preparation;
|
|
21273
|
+
if (failure === void 0) {
|
|
21274
|
+
candidates2.push({ docId: item.docId, document });
|
|
21275
|
+
continue;
|
|
21276
|
+
}
|
|
21277
|
+
const docId = item.refusedByHook ? item.docId : providedDocId(document) ?? "";
|
|
21278
|
+
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
21279
|
+
}
|
|
21280
|
+
return { entry, candidates: candidates2, embedding };
|
|
21281
|
+
}
|
|
21282
|
+
async function admitBatchDocuments(ctx, indexName, documents2, failed, hold) {
|
|
21283
|
+
const { entry, candidates: candidates2, embedding } = await prepareInsertDocuments(
|
|
21284
|
+
ctx,
|
|
21285
|
+
indexName,
|
|
21286
|
+
documents2,
|
|
21287
|
+
0,
|
|
21288
|
+
documents2.length,
|
|
21289
|
+
hold,
|
|
21290
|
+
true,
|
|
21291
|
+
failed
|
|
21292
|
+
);
|
|
21293
|
+
const manager = ctx.requireManager(indexName);
|
|
21294
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
21295
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
21296
|
+
const strict = entry.config.strict === true;
|
|
21297
|
+
const admitted = [];
|
|
21298
|
+
const admittedIds = /* @__PURE__ */ new Set();
|
|
21299
|
+
const pending = createPendingAdmissions();
|
|
21300
|
+
for (let position = 0; position < candidates2.length; position++) {
|
|
21301
|
+
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
21302
|
+
if (ctx.abortController.signal.aborted) break;
|
|
21303
|
+
const { docId, document } = candidates2[position];
|
|
21304
|
+
try {
|
|
21305
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
21306
|
+
document,
|
|
21307
|
+
vectorFieldPaths
|
|
21308
|
+
);
|
|
21309
|
+
if (extractedVectors.size > 0) {
|
|
21310
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
21311
|
+
}
|
|
21312
|
+
admitInsert(ctx, indexName, manager, docId, pending);
|
|
21313
|
+
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
21314
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
21315
|
+
}
|
|
21316
|
+
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
21317
|
+
if (strict) {
|
|
21318
|
+
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
21319
|
+
}
|
|
21320
|
+
admittedIds.add(docId);
|
|
21321
|
+
recordPendingAdmission(ctx, pending, manager, docId);
|
|
21322
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
21323
|
+
} catch (err) {
|
|
21324
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
21325
|
+
}
|
|
21326
|
+
}
|
|
21327
|
+
return { admitted, embedding };
|
|
21328
|
+
}
|
|
21329
|
+
|
|
21092
21330
|
// src/engine/mutations/durable-rollback.ts
|
|
21093
21331
|
function holdVectorsForRollback(indexes, docId) {
|
|
21094
21332
|
const releases = [];
|
|
@@ -21324,84 +21562,29 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
21324
21562
|
return applications;
|
|
21325
21563
|
}
|
|
21326
21564
|
|
|
21327
|
-
// src/engine/mutations/insert-batch-
|
|
21328
|
-
async function
|
|
21329
|
-
|
|
21330
|
-
const manager = ctx.
|
|
21331
|
-
|
|
21332
|
-
|
|
21333
|
-
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
21334
|
-
const strict = entry.config.strict === true;
|
|
21335
|
-
const required = entry.config.required;
|
|
21336
|
-
const admitted = [];
|
|
21337
|
-
const admittedIds = /* @__PURE__ */ new Set();
|
|
21338
|
-
const pending = createPendingAdmissions();
|
|
21339
|
-
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21340
|
-
if (ctx.abortController.signal.aborted) break;
|
|
21341
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
21342
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
21343
|
-
if (required && required.length > 0) {
|
|
21344
|
-
collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
21345
|
-
}
|
|
21346
|
-
await embedChunkDocuments(
|
|
21347
|
-
entry,
|
|
21348
|
-
documents2,
|
|
21349
|
-
chunkStart,
|
|
21350
|
-
chunkEnd,
|
|
21351
|
-
ctx.abortController.signal,
|
|
21352
|
-
chunkFailedIndexes,
|
|
21353
|
-
failed
|
|
21354
|
-
);
|
|
21355
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
21356
|
-
if (ctx.abortController.signal.aborted) break;
|
|
21357
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
21358
|
-
const docId = providedDocId(documents2[i]) ?? ctx.idGenerator();
|
|
21565
|
+
// src/engine/mutations/insert-batch-documents.ts
|
|
21566
|
+
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
21567
|
+
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
21568
|
+
const manager = ctx.executor.getManager(indexName);
|
|
21569
|
+
await runOutside(hold, async () => {
|
|
21570
|
+
for (const doc of inserted) {
|
|
21359
21571
|
try {
|
|
21360
|
-
|
|
21361
|
-
if (sizing !== null) await sizing;
|
|
21362
|
-
validateDocId(docId);
|
|
21363
|
-
if (hasBeforeHook) {
|
|
21364
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents2[i] });
|
|
21365
|
-
}
|
|
21366
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
21367
|
-
documents2[i],
|
|
21368
|
-
vectorFieldPaths
|
|
21369
|
-
);
|
|
21370
|
-
if (extractedVectors.size > 0) {
|
|
21371
|
-
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
21372
|
-
}
|
|
21373
|
-
admitInsert(ctx, indexName, manager, docId, pending);
|
|
21374
|
-
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
21375
|
-
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
21376
|
-
}
|
|
21377
|
-
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
21378
|
-
if (strict) {
|
|
21379
|
-
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
21380
|
-
}
|
|
21381
|
-
admittedIds.add(docId);
|
|
21382
|
-
recordPendingAdmission(ctx, pending, manager, docId);
|
|
21383
|
-
admitted.push({ docId, document: documents2[i], partitionDoc, extractedVectors });
|
|
21572
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
21384
21573
|
} catch (err) {
|
|
21385
|
-
|
|
21574
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
21386
21575
|
}
|
|
21387
21576
|
}
|
|
21388
|
-
|
|
21389
|
-
|
|
21390
|
-
}
|
|
21391
|
-
}
|
|
21392
|
-
return admitted;
|
|
21577
|
+
});
|
|
21578
|
+
return ctx.executor.getManager(indexName) !== manager;
|
|
21393
21579
|
}
|
|
21394
|
-
|
|
21395
|
-
// src/engine/mutations/insert-batch-documents.ts
|
|
21396
|
-
async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed) {
|
|
21397
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
21580
|
+
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
21398
21581
|
const succeeded = [];
|
|
21399
|
-
const buffered = /* @__PURE__ */ new Set();
|
|
21400
21582
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
21401
21583
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21402
21584
|
if (ctx.abortController.signal.aborted) break;
|
|
21403
21585
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
21404
21586
|
const applications = await applyInsertChunk(ctx, indexName, chunk, options);
|
|
21587
|
+
const inserted = [];
|
|
21405
21588
|
for (let i = 0; i < chunk.length; i++) {
|
|
21406
21589
|
const doc = chunk[i];
|
|
21407
21590
|
const application = applications[i];
|
|
@@ -21410,30 +21593,103 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
|
|
|
21410
21593
|
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
21411
21594
|
continue;
|
|
21412
21595
|
}
|
|
21413
|
-
|
|
21414
|
-
|
|
21415
|
-
succeeded.push(doc.docId);
|
|
21416
|
-
continue;
|
|
21417
|
-
}
|
|
21596
|
+
succeeded.push(doc.docId);
|
|
21597
|
+
if (application.status === "buffered") continue;
|
|
21418
21598
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
21419
21599
|
touchedVectorFields.add(fieldPath);
|
|
21420
21600
|
}
|
|
21421
|
-
|
|
21422
|
-
try {
|
|
21423
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
21424
|
-
} catch (err) {
|
|
21425
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
21426
|
-
}
|
|
21427
|
-
}
|
|
21428
|
-
succeeded.push(doc.docId);
|
|
21601
|
+
inserted.push(doc);
|
|
21429
21602
|
}
|
|
21430
21603
|
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
21604
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
21605
|
+
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
21606
|
+
}
|
|
21431
21607
|
if (applied < admitted.length) {
|
|
21432
21608
|
ctx.checkHeapPressure(indexName);
|
|
21433
21609
|
await new Promise((r) => setTimeout(r, 0));
|
|
21434
21610
|
}
|
|
21435
21611
|
}
|
|
21436
|
-
return { succeeded,
|
|
21612
|
+
return { succeeded, touchedVectorFields, unrecorded: [] };
|
|
21613
|
+
}
|
|
21614
|
+
|
|
21615
|
+
// src/engine/mutations/insert-batch-chunks.ts
|
|
21616
|
+
async function admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
21617
|
+
const { entry, candidates: candidates2 } = await prepareInsertDocuments(
|
|
21618
|
+
ctx,
|
|
21619
|
+
indexName,
|
|
21620
|
+
documents2,
|
|
21621
|
+
chunkStart,
|
|
21622
|
+
chunkEnd,
|
|
21623
|
+
hold,
|
|
21624
|
+
hooked,
|
|
21625
|
+
failed,
|
|
21626
|
+
earlier
|
|
21627
|
+
);
|
|
21628
|
+
const manager = ctx.requireManager(indexName);
|
|
21629
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
21630
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
21631
|
+
const admitted = [];
|
|
21632
|
+
for (const { docId, document } of candidates2) {
|
|
21633
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
21634
|
+
if (ctx.abortController.signal.aborted) break;
|
|
21635
|
+
try {
|
|
21636
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
21637
|
+
document,
|
|
21638
|
+
vectorFieldPaths
|
|
21639
|
+
);
|
|
21640
|
+
if (extractedVectors.size > 0) {
|
|
21641
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
21642
|
+
}
|
|
21643
|
+
admitInsert(ctx, indexName, manager, docId);
|
|
21644
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
21645
|
+
} catch (err) {
|
|
21646
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
21647
|
+
}
|
|
21648
|
+
}
|
|
21649
|
+
return admitted;
|
|
21650
|
+
}
|
|
21651
|
+
async function insertInChunks(ctx, indexName, documents2, options, hold, hooked, failed, embedded2 = null) {
|
|
21652
|
+
const succeeded = [];
|
|
21653
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
21654
|
+
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21655
|
+
if (ctx.abortController.signal.aborted) break;
|
|
21656
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
21657
|
+
const earlier = embeddingOfDocuments(embedded2, documents2.slice(chunkStart, chunkEnd));
|
|
21658
|
+
const prepared = await admitChunk(ctx, indexName, documents2, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
21659
|
+
if (prepared.length > 0) {
|
|
21660
|
+
const applications = await applyInsertChunk(ctx, indexName, prepared, options);
|
|
21661
|
+
const inserted = [];
|
|
21662
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
21663
|
+
const doc = prepared[i];
|
|
21664
|
+
const application = applications[i];
|
|
21665
|
+
if (application.status === "skipped") continue;
|
|
21666
|
+
if (application.status === "failed") {
|
|
21667
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
21668
|
+
continue;
|
|
21669
|
+
}
|
|
21670
|
+
succeeded.push(doc.docId);
|
|
21671
|
+
if (application.status === "buffered") continue;
|
|
21672
|
+
for (const fieldPath of doc.extractedVectors.keys()) touchedVectorFields.add(fieldPath);
|
|
21673
|
+
inserted.push(doc);
|
|
21674
|
+
}
|
|
21675
|
+
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
21676
|
+
}
|
|
21677
|
+
if (chunkEnd < documents2.length) {
|
|
21678
|
+
ctx.checkHeapPressure(indexName);
|
|
21679
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
21680
|
+
}
|
|
21681
|
+
}
|
|
21682
|
+
return { succeeded, touchedVectorFields };
|
|
21683
|
+
}
|
|
21684
|
+
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
21685
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
21686
|
+
for (const fieldPath of inserts.touchedVectorFields) {
|
|
21687
|
+
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
21688
|
+
}
|
|
21689
|
+
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
21690
|
+
ctx.checkHeapPressure(indexName);
|
|
21691
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
21692
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
21437
21693
|
}
|
|
21438
21694
|
|
|
21439
21695
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -21538,10 +21794,6 @@ function mainStoreDocument(doc, options) {
|
|
|
21538
21794
|
if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
|
|
21539
21795
|
return structuredClone(doc.partitionDoc);
|
|
21540
21796
|
}
|
|
21541
|
-
async function applyIndividually(ctx, indexName, admitted, options, failed) {
|
|
21542
|
-
const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
|
|
21543
|
-
return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
|
|
21544
|
-
}
|
|
21545
21797
|
function chunksOfWholeSegments(indexName, partitionCount, built2, frozen, memberIndexes, admitted) {
|
|
21546
21798
|
const chunks = [];
|
|
21547
21799
|
let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
@@ -21591,12 +21843,12 @@ function chunksOfWholeSegments(indexName, partitionCount, built2, frozen, member
|
|
|
21591
21843
|
if (chunk.docs.length > 0) chunks.push(chunk);
|
|
21592
21844
|
return chunks;
|
|
21593
21845
|
}
|
|
21594
|
-
async function recordMergedDocuments(ctx, indexName, partitionCount, built2, frozen, memberIndexes, admitted, options, failed) {
|
|
21595
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
21846
|
+
async function recordMergedDocuments(ctx, indexName, partitionCount, built2, frozen, memberIndexes, admitted, options, hold, failed) {
|
|
21596
21847
|
const succeededIds = /* @__PURE__ */ new Set();
|
|
21597
21848
|
const errorOf = /* @__PURE__ */ new Map();
|
|
21598
21849
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
21599
21850
|
const chunks = chunksOfWholeSegments(indexName, partitionCount, built2, frozen, memberIndexes, admitted);
|
|
21851
|
+
const unrecorded = [];
|
|
21600
21852
|
for (let index = 0; index < chunks.length; index++) {
|
|
21601
21853
|
const { docs, attachmentOf } = chunks[index];
|
|
21602
21854
|
const applies = docs.map((doc) => {
|
|
@@ -21606,6 +21858,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21606
21858
|
});
|
|
21607
21859
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
21608
21860
|
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
21861
|
+
const inserted = [];
|
|
21609
21862
|
for (let i = 0; i < docs.length; i++) {
|
|
21610
21863
|
const doc = docs[i];
|
|
21611
21864
|
const failure = failures[i];
|
|
@@ -21637,15 +21890,13 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21637
21890
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
21638
21891
|
touchedVectorFields.add(fieldPath);
|
|
21639
21892
|
}
|
|
21640
|
-
|
|
21641
|
-
try {
|
|
21642
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
21643
|
-
} catch (err) {
|
|
21644
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
21645
|
-
}
|
|
21646
|
-
}
|
|
21893
|
+
inserted.push(doc);
|
|
21647
21894
|
succeededIds.add(doc.docId);
|
|
21648
21895
|
}
|
|
21896
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
21897
|
+
for (const later of chunks.slice(index + 1)) unrecorded.push(...later.docs);
|
|
21898
|
+
break;
|
|
21899
|
+
}
|
|
21649
21900
|
if (index + 1 < chunks.length) {
|
|
21650
21901
|
ctx.checkHeapPressure(indexName);
|
|
21651
21902
|
await new Promise((r) => setTimeout(r, 0));
|
|
@@ -21660,7 +21911,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built2, fro
|
|
|
21660
21911
|
if (!errorOf.has(doc.docId)) continue;
|
|
21661
21912
|
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
21662
21913
|
}
|
|
21663
|
-
return { succeeded, touchedVectorFields };
|
|
21914
|
+
return { succeeded, touchedVectorFields, unrecorded };
|
|
21664
21915
|
}
|
|
21665
21916
|
function frozenSegmentOf(segment, members, options) {
|
|
21666
21917
|
if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
|
|
@@ -21671,14 +21922,15 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
21671
21922
|
segment.segmentId
|
|
21672
21923
|
);
|
|
21673
21924
|
}
|
|
21674
|
-
async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
21925
|
+
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
21675
21926
|
if (admitted.length === 0) {
|
|
21676
|
-
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set() };
|
|
21927
|
+
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] };
|
|
21677
21928
|
}
|
|
21929
|
+
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
21678
21930
|
const manager = ctx.requireManager(indexName);
|
|
21679
21931
|
const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
|
|
21680
21932
|
if (admitted.length < MIN_DOCUMENTS_FOR_SEGMENTS || workers <= 0) {
|
|
21681
|
-
return applyIndividually(
|
|
21933
|
+
return applyIndividually();
|
|
21682
21934
|
}
|
|
21683
21935
|
const docIds = admitted.map((doc) => doc.docId);
|
|
21684
21936
|
const partitionDocuments = admitted.map((doc) => doc.partitionDoc);
|
|
@@ -21691,7 +21943,7 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
21691
21943
|
built2 = null;
|
|
21692
21944
|
}
|
|
21693
21945
|
if (built2 === null) {
|
|
21694
|
-
return applyIndividually(
|
|
21946
|
+
return applyIndividually();
|
|
21695
21947
|
}
|
|
21696
21948
|
const positions2 = new Map(admitted.map((document, index) => [document.docId, index]));
|
|
21697
21949
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -21700,16 +21952,15 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
21700
21952
|
const members = [];
|
|
21701
21953
|
for (const docId of segment.docIds) {
|
|
21702
21954
|
const position = positions2.get(docId);
|
|
21703
|
-
if (position === void 0 || used.has(position))
|
|
21704
|
-
return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
21955
|
+
if (position === void 0 || used.has(position)) return applyIndividually();
|
|
21705
21956
|
used.add(position);
|
|
21706
21957
|
members.push(position);
|
|
21707
21958
|
}
|
|
21708
21959
|
memberIndexes.push(members);
|
|
21709
21960
|
}
|
|
21710
|
-
if (used.size !== admitted.length) return applyIndividually(
|
|
21961
|
+
if (used.size !== admitted.length) return applyIndividually();
|
|
21711
21962
|
if (ctx.isRebalancing(indexName)) {
|
|
21712
|
-
return applyIndividually(
|
|
21963
|
+
return applyIndividually();
|
|
21713
21964
|
}
|
|
21714
21965
|
const frozen = [];
|
|
21715
21966
|
for (let i = 0; i < built2.length; i++) {
|
|
@@ -21718,149 +21969,65 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
21718
21969
|
memberIndexes[i].map((m) => admitted[m]),
|
|
21719
21970
|
options
|
|
21720
21971
|
);
|
|
21721
|
-
if (segment === null) return applyIndividually(
|
|
21972
|
+
if (segment === null) return applyIndividually();
|
|
21722
21973
|
frozen.push(segment);
|
|
21723
21974
|
}
|
|
21724
21975
|
for (const doc of admitted) {
|
|
21725
|
-
if (manager.has(doc.docId)) return applyIndividually(
|
|
21976
|
+
if (manager.has(doc.docId)) return applyIndividually();
|
|
21726
21977
|
}
|
|
21727
|
-
return recordMergedDocuments(
|
|
21978
|
+
return recordMergedDocuments(
|
|
21979
|
+
ctx,
|
|
21980
|
+
indexName,
|
|
21981
|
+
partitionCount,
|
|
21982
|
+
built2,
|
|
21983
|
+
frozen,
|
|
21984
|
+
memberIndexes,
|
|
21985
|
+
admitted,
|
|
21986
|
+
options,
|
|
21987
|
+
hold,
|
|
21988
|
+
failed
|
|
21989
|
+
);
|
|
21728
21990
|
}
|
|
21729
|
-
async function insertBatchViaSegments(ctx, indexName, documents2, options) {
|
|
21991
|
+
async function insertBatchViaSegments(ctx, indexName, documents2, options, hold) {
|
|
21730
21992
|
const failed = [];
|
|
21731
|
-
const admitted = await admitBatchDocuments(ctx, indexName, documents2, failed);
|
|
21732
|
-
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, failed);
|
|
21733
|
-
|
|
21734
|
-
|
|
21735
|
-
|
|
21736
|
-
|
|
21737
|
-
|
|
21738
|
-
|
|
21739
|
-
await ctx
|
|
21740
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
21993
|
+
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents2, failed, hold);
|
|
21994
|
+
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
21995
|
+
if (outcome.unrecorded.length > 0) {
|
|
21996
|
+
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
21997
|
+
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
21998
|
+
outcome.succeeded.push(...rest.succeeded);
|
|
21999
|
+
for (const fieldPath of rest.touchedVectorFields) outcome.touchedVectorFields.add(fieldPath);
|
|
22000
|
+
}
|
|
22001
|
+
await finishInsertBatch(ctx, indexName, outcome, options);
|
|
21741
22002
|
return { succeeded: outcome.succeeded, failed };
|
|
21742
22003
|
}
|
|
21743
22004
|
|
|
21744
22005
|
// src/engine/mutations/insert-batch.ts
|
|
21745
|
-
async function insertDocumentBatch(ctx, indexName, documents2, options) {
|
|
22006
|
+
async function insertDocumentBatch(ctx, indexName, documents2, options, hold) {
|
|
21746
22007
|
ctx.guardShutdown();
|
|
21747
|
-
|
|
22008
|
+
ctx.requireIndex(indexName);
|
|
21748
22009
|
validateBatchList(documents2, "documents");
|
|
21749
22010
|
await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents2.length);
|
|
21750
22011
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
21751
22012
|
if (documents2.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
21752
|
-
return insertBatchViaSegments(ctx, indexName, documents2, options);
|
|
22013
|
+
return insertBatchViaSegments(ctx, indexName, documents2, options, hold);
|
|
21753
22014
|
}
|
|
21754
|
-
const succeeded = [];
|
|
21755
22015
|
const failed = [];
|
|
21756
|
-
const
|
|
21757
|
-
|
|
21758
|
-
|
|
21759
|
-
const batchManager = ctx.requireManager(indexName);
|
|
21760
|
-
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
21761
|
-
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
21762
|
-
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
21763
|
-
for (let chunkStart = 0; chunkStart < documents2.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21764
|
-
if (ctx.abortController.signal.aborted) break;
|
|
21765
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents2.length);
|
|
21766
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
21767
|
-
if (required && required.length > 0) {
|
|
21768
|
-
collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
21769
|
-
}
|
|
21770
|
-
await embedChunkDocuments(
|
|
21771
|
-
entry,
|
|
21772
|
-
documents2,
|
|
21773
|
-
chunkStart,
|
|
21774
|
-
chunkEnd,
|
|
21775
|
-
ctx.abortController.signal,
|
|
21776
|
-
chunkFailedIndexes,
|
|
21777
|
-
failed
|
|
21778
|
-
);
|
|
21779
|
-
async function applyPrepared(prepared2) {
|
|
21780
|
-
const applications = await applyInsertChunk(ctx, indexName, prepared2, options);
|
|
21781
|
-
for (let i = 0; i < prepared2.length; i++) {
|
|
21782
|
-
const doc = prepared2[i];
|
|
21783
|
-
const application = applications[i];
|
|
21784
|
-
if (application.status === "skipped") continue;
|
|
21785
|
-
if (application.status === "failed") {
|
|
21786
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
21787
|
-
continue;
|
|
21788
|
-
}
|
|
21789
|
-
if (application.status === "buffered") {
|
|
21790
|
-
succeeded.push(doc.docId);
|
|
21791
|
-
continue;
|
|
21792
|
-
}
|
|
21793
|
-
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
21794
|
-
touchedVectorFields.add(fieldPath);
|
|
21795
|
-
}
|
|
21796
|
-
if (hasAfterHook) {
|
|
21797
|
-
try {
|
|
21798
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
21799
|
-
} catch (err) {
|
|
21800
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
21801
|
-
}
|
|
21802
|
-
}
|
|
21803
|
-
succeeded.push(doc.docId);
|
|
21804
|
-
}
|
|
21805
|
-
}
|
|
21806
|
-
const prepared = [];
|
|
21807
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
21808
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
21809
|
-
if (ctx.abortController.signal.aborted) break;
|
|
21810
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
21811
|
-
const batchDocId = providedDocId(documents2[i]) ?? ctx.idGenerator();
|
|
21812
|
-
try {
|
|
21813
|
-
const sizing = requireDocumentObject(documents2[i]);
|
|
21814
|
-
if (sizing !== null) await sizing;
|
|
21815
|
-
validateDocId(batchDocId);
|
|
21816
|
-
if (hasBeforeHook) {
|
|
21817
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents2[i] });
|
|
21818
|
-
}
|
|
21819
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
21820
|
-
documents2[i],
|
|
21821
|
-
batchVectorFieldPaths
|
|
21822
|
-
);
|
|
21823
|
-
if (extractedVectors.size > 0) {
|
|
21824
|
-
validateVectorDimensions(extractedVectors, batchVecIndexes);
|
|
21825
|
-
}
|
|
21826
|
-
admitInsert(ctx, indexName, batchManager, batchDocId);
|
|
21827
|
-
prepared.push({
|
|
21828
|
-
docId: batchDocId,
|
|
21829
|
-
document: documents2[i],
|
|
21830
|
-
partitionDoc,
|
|
21831
|
-
extractedVectors
|
|
21832
|
-
});
|
|
21833
|
-
} catch (err) {
|
|
21834
|
-
failed.push({ docId: batchDocId, error: asBatchInsertError(err) });
|
|
21835
|
-
}
|
|
21836
|
-
}
|
|
21837
|
-
if (prepared.length > 0) await applyPrepared(prepared);
|
|
21838
|
-
if (chunkEnd < documents2.length) {
|
|
21839
|
-
ctx.checkHeapPressure(indexName);
|
|
21840
|
-
await new Promise((r) => setTimeout(r, 0));
|
|
21841
|
-
}
|
|
21842
|
-
}
|
|
21843
|
-
for (const fieldPath of touchedVectorFields) {
|
|
21844
|
-
const vecIndex = batchVecIndexes.get(fieldPath);
|
|
21845
|
-
if (vecIndex) {
|
|
21846
|
-
vecIndex.scheduleBuild();
|
|
21847
|
-
}
|
|
21848
|
-
}
|
|
21849
|
-
ctx.checkWatermark(indexName, succeeded);
|
|
21850
|
-
ctx.checkHeapPressure(indexName);
|
|
21851
|
-
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
21852
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
21853
|
-
return { succeeded, failed };
|
|
22016
|
+
const inserts = await insertInChunks(ctx, indexName, documents2, options, hold, true, failed);
|
|
22017
|
+
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
22018
|
+
return { succeeded: inserts.succeeded, failed };
|
|
21854
22019
|
}
|
|
21855
22020
|
|
|
21856
22021
|
// src/engine/mutations/remove.ts
|
|
21857
22022
|
function asRemoveError(err) {
|
|
21858
22023
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
21859
22024
|
}
|
|
21860
|
-
|
|
22025
|
+
function validateRemoval(ctx, docId) {
|
|
21861
22026
|
ctx.guardShutdown();
|
|
21862
22027
|
validateDocId(docId);
|
|
21863
|
-
|
|
22028
|
+
}
|
|
22029
|
+
function prepareRemoval(ctx, indexName, docId) {
|
|
22030
|
+
ctx.guardShutdown();
|
|
21864
22031
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
21865
22032
|
if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
|
|
21866
22033
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
@@ -21910,20 +22077,31 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
21910
22077
|
};
|
|
21911
22078
|
return () => manager.publication.writeNow(indexChangeOutsideExternalWaits(removeFromIndex));
|
|
21912
22079
|
}
|
|
21913
|
-
async function
|
|
22080
|
+
async function settleRemovals(ctx, indexName, docIds, hold) {
|
|
22081
|
+
if (docIds.length === 0) return;
|
|
21914
22082
|
const settling = ctx.orchestrator.settlePublications(indexName, "none");
|
|
21915
22083
|
if (settling !== void 0) await settling;
|
|
21916
|
-
|
|
21917
|
-
|
|
21918
|
-
|
|
21919
|
-
|
|
21920
|
-
|
|
21921
|
-
|
|
22084
|
+
if (!ctx.pluginRegistry.hasHooks("afterRemove")) return;
|
|
22085
|
+
await runOutside(hold, async () => {
|
|
22086
|
+
for (const docId of docIds) {
|
|
22087
|
+
try {
|
|
22088
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
22089
|
+
} catch (err) {
|
|
22090
|
+
console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
22091
|
+
}
|
|
22092
|
+
}
|
|
22093
|
+
});
|
|
21922
22094
|
}
|
|
21923
|
-
async function removeDocument3(ctx, indexName, docId, options) {
|
|
22095
|
+
async function removeDocument3(ctx, indexName, docId, options, hold) {
|
|
21924
22096
|
ctx.guardShutdown();
|
|
21925
22097
|
ctx.requireIndex(indexName);
|
|
21926
|
-
|
|
22098
|
+
validateRemoval(ctx, docId);
|
|
22099
|
+
if (ctx.pluginRegistry.hasHooks("beforeRemove")) {
|
|
22100
|
+
await runOutside(hold, async () => {
|
|
22101
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
|
|
22102
|
+
});
|
|
22103
|
+
}
|
|
22104
|
+
const prepared = prepareRemoval(ctx, indexName, docId);
|
|
21927
22105
|
const apply = applyOfRemoval(ctx, indexName, prepared);
|
|
21928
22106
|
if (ctx.durability) {
|
|
21929
22107
|
try {
|
|
@@ -21937,25 +22115,52 @@ async function removeDocument3(ctx, indexName, docId, options) {
|
|
|
21937
22115
|
} else {
|
|
21938
22116
|
await apply();
|
|
21939
22117
|
}
|
|
21940
|
-
if (!prepared.buffered) await
|
|
22118
|
+
if (!prepared.buffered) await settleRemovals(ctx, indexName, [docId], hold);
|
|
21941
22119
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
21942
22120
|
}
|
|
21943
|
-
async function
|
|
22121
|
+
async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold) {
|
|
22122
|
+
const refused = /* @__PURE__ */ new Map();
|
|
22123
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22124
|
+
try {
|
|
22125
|
+
validateRemoval(ctx, docIds[i]);
|
|
22126
|
+
} catch (err) {
|
|
22127
|
+
refused.set(i, err);
|
|
22128
|
+
}
|
|
22129
|
+
}
|
|
22130
|
+
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
22131
|
+
await runOutside(hold, async () => {
|
|
22132
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22133
|
+
if (refused.has(i)) continue;
|
|
22134
|
+
try {
|
|
22135
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
22136
|
+
} catch (err) {
|
|
22137
|
+
refused.set(i, err);
|
|
22138
|
+
}
|
|
22139
|
+
}
|
|
22140
|
+
});
|
|
22141
|
+
return refused;
|
|
22142
|
+
}
|
|
22143
|
+
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
21944
22144
|
ctx.guardShutdown();
|
|
21945
22145
|
ctx.requireIndex(indexName);
|
|
21946
22146
|
validateBatchList(docIds, "document IDs");
|
|
21947
22147
|
const succeeded = [];
|
|
21948
22148
|
const failed = [];
|
|
21949
|
-
const
|
|
21950
|
-
await
|
|
22149
|
+
const first = ctx.requireManager(indexName);
|
|
22150
|
+
await first.publication.write(() => first.beginBatchRemove());
|
|
22151
|
+
let manager = first;
|
|
21951
22152
|
try {
|
|
21952
22153
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
21953
22154
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
22155
|
+
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
22156
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
22157
|
+
const removed = [];
|
|
21954
22158
|
const prepared = [];
|
|
21955
22159
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
21956
22160
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
21957
22161
|
try {
|
|
21958
|
-
|
|
22162
|
+
if (refused.has(i)) throw refused.get(i);
|
|
22163
|
+
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
21959
22164
|
} catch (err) {
|
|
21960
22165
|
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
21961
22166
|
}
|
|
@@ -21985,23 +22190,33 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
21985
22190
|
continue;
|
|
21986
22191
|
}
|
|
21987
22192
|
removal.releaseVectorHolds?.();
|
|
21988
|
-
if (!removal.buffered)
|
|
22193
|
+
if (!removal.buffered) removed.push(removal.docId);
|
|
21989
22194
|
succeeded.push(removal.docId);
|
|
21990
22195
|
}
|
|
21991
22196
|
} finally {
|
|
21992
22197
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
21993
22198
|
}
|
|
22199
|
+
await settleRemovals(ctx, indexName, removed, hold);
|
|
22200
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
21994
22201
|
if (chunkEnd < docIds.length) {
|
|
21995
22202
|
await new Promise((r) => setTimeout(r, 0));
|
|
21996
22203
|
}
|
|
21997
22204
|
}
|
|
21998
22205
|
} finally {
|
|
21999
|
-
|
|
22206
|
+
const ending = manager;
|
|
22207
|
+
await ending?.publication.write(() => ending.endBatchRemove());
|
|
22000
22208
|
}
|
|
22001
22209
|
ctx.checkHeapPressure(indexName);
|
|
22002
22210
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22003
22211
|
return { succeeded, failed };
|
|
22004
22212
|
}
|
|
22213
|
+
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
22214
|
+
const current = ctx.executor.getManager(indexName);
|
|
22215
|
+
if (current === batchManager) return batchManager;
|
|
22216
|
+
await batchManager?.publication.write(() => batchManager.endBatchRemove());
|
|
22217
|
+
await current?.publication.write(() => current.beginBatchRemove());
|
|
22218
|
+
return current;
|
|
22219
|
+
}
|
|
22005
22220
|
|
|
22006
22221
|
// src/engine/mutations/update.ts
|
|
22007
22222
|
function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
@@ -22035,59 +22250,47 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
22035
22250
|
function asUpdateError(err) {
|
|
22036
22251
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
22037
22252
|
}
|
|
22038
|
-
async function
|
|
22253
|
+
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
22254
|
+
for (const update of updates) {
|
|
22255
|
+
try {
|
|
22256
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
22257
|
+
indexName,
|
|
22258
|
+
docId: update.docId,
|
|
22259
|
+
oldDocument: update.oldDocument ?? {},
|
|
22260
|
+
newDocument: update.document
|
|
22261
|
+
});
|
|
22262
|
+
} catch (err) {
|
|
22263
|
+
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
22264
|
+
}
|
|
22265
|
+
}
|
|
22266
|
+
}
|
|
22267
|
+
async function settleUpdates(ctx, indexName, updates, hold) {
|
|
22268
|
+
if (updates.length === 0) return;
|
|
22039
22269
|
await ctx.orchestrator.settlePublications(indexName);
|
|
22040
|
-
|
|
22041
|
-
await
|
|
22042
|
-
indexName,
|
|
22043
|
-
docId: prepared.docId,
|
|
22044
|
-
oldDocument: prepared.oldDocument ?? {},
|
|
22045
|
-
newDocument: prepared.document
|
|
22046
|
-
});
|
|
22047
|
-
} catch (err) {
|
|
22048
|
-
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
22270
|
+
if (ctx.pluginRegistry.hasHooks("afterUpdate")) {
|
|
22271
|
+
await runOutside(hold, () => runAfterUpdateHooks(ctx, indexName, updates));
|
|
22049
22272
|
}
|
|
22050
|
-
const vecIndexes = ctx.
|
|
22051
|
-
|
|
22052
|
-
|
|
22053
|
-
|
|
22273
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
22274
|
+
if (vecIndexes === void 0) return;
|
|
22275
|
+
for (const update of updates) {
|
|
22276
|
+
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
22277
|
+
if (vec !== null) vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
22278
|
+
}
|
|
22054
22279
|
}
|
|
22055
22280
|
}
|
|
22056
|
-
|
|
22281
|
+
function updatePreparationOf(ctx, indexName, docId, document) {
|
|
22057
22282
|
ctx.guardShutdown();
|
|
22058
22283
|
validateDocId(docId);
|
|
22059
22284
|
requireMatchingDocId(docId, document);
|
|
22060
|
-
|
|
22061
|
-
|
|
22062
|
-
|
|
22063
|
-
|
|
22064
|
-
|
|
22065
|
-
|
|
22066
|
-
newDocument: document
|
|
22067
|
-
});
|
|
22068
|
-
}
|
|
22069
|
-
if (entry.config.required && entry.config.required.length > 0) {
|
|
22070
|
-
validateRequiredFields(document, entry.config.required);
|
|
22071
|
-
}
|
|
22072
|
-
if (entry.config.embedding) {
|
|
22073
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
22074
|
-
if (entry.embeddingAdapter) {
|
|
22075
|
-
await embedDocumentFields(
|
|
22076
|
-
document,
|
|
22077
|
-
entry.config.embedding,
|
|
22078
|
-
entry.embeddingAdapter,
|
|
22079
|
-
ctx.abortController.signal
|
|
22080
|
-
);
|
|
22081
|
-
} else {
|
|
22082
|
-
assertDocumentCarriesMappedVectors(
|
|
22083
|
-
document,
|
|
22084
|
-
entry.config.embedding,
|
|
22085
|
-
entry.embeddingAdapterName
|
|
22086
|
-
);
|
|
22087
|
-
}
|
|
22088
|
-
}
|
|
22285
|
+
if (!ctx.pluginRegistry.hasHooks("beforeUpdate")) return { document, beforeWrite: null, failure: void 0 };
|
|
22286
|
+
const oldDocument = ctx.requireManager(indexName).get(docId) ?? {};
|
|
22287
|
+
const beforeWrite = () => ctx.pluginRegistry.runHook("beforeUpdate", { indexName, docId, oldDocument, newDocument: document });
|
|
22288
|
+
return { document, beforeWrite, failure: void 0 };
|
|
22289
|
+
}
|
|
22290
|
+
async function finishUpdate(ctx, indexName, entry, docId, document) {
|
|
22089
22291
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
22090
22292
|
if (sizing !== null) await sizing;
|
|
22293
|
+
const manager = ctx.requireManager(indexName);
|
|
22091
22294
|
const vecIndexes = manager.getVectorIndexes();
|
|
22092
22295
|
const extractedVectors = /* @__PURE__ */ new Map();
|
|
22093
22296
|
if (vecIndexes.size > 0) {
|
|
@@ -22181,10 +22384,13 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
22181
22384
|
};
|
|
22182
22385
|
return () => manager.publication.write(indexChangeOutsideExternalWaits(updateInIndex));
|
|
22183
22386
|
}
|
|
22184
|
-
async function updateDocument(ctx, indexName, docId, document, options) {
|
|
22387
|
+
async function updateDocument(ctx, indexName, docId, document, options, hold) {
|
|
22185
22388
|
ctx.guardShutdown();
|
|
22186
|
-
|
|
22187
|
-
const
|
|
22389
|
+
ctx.requireIndex(indexName);
|
|
22390
|
+
const preparation = updatePreparationOf(ctx, indexName, docId, document);
|
|
22391
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
22392
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
22393
|
+
const prepared = await finishUpdate(ctx, indexName, entry, docId, document);
|
|
22188
22394
|
const apply = applyOfUpdate(ctx, indexName, prepared);
|
|
22189
22395
|
if (ctx.durability) {
|
|
22190
22396
|
try {
|
|
@@ -22207,10 +22413,12 @@ async function updateDocument(ctx, indexName, docId, document, options) {
|
|
|
22207
22413
|
} else {
|
|
22208
22414
|
await apply();
|
|
22209
22415
|
}
|
|
22210
|
-
if (!prepared.buffered) await
|
|
22416
|
+
if (!prepared.buffered) await settleUpdates(ctx, indexName, [prepared], hold);
|
|
22211
22417
|
ctx.checkHeapPressure(indexName);
|
|
22212
22418
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22213
22419
|
}
|
|
22420
|
+
|
|
22421
|
+
// src/engine/mutations/update-batch.ts
|
|
22214
22422
|
function reportedDocIdOf(update) {
|
|
22215
22423
|
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
22216
22424
|
return typeof docId === "string" ? docId : "";
|
|
@@ -22227,29 +22435,47 @@ function requireUpdateEntry(update) {
|
|
|
22227
22435
|
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
22228
22436
|
);
|
|
22229
22437
|
}
|
|
22230
|
-
async function
|
|
22438
|
+
async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed) {
|
|
22439
|
+
const preparations = [];
|
|
22440
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22441
|
+
const update = updates[i];
|
|
22442
|
+
const docId = reportedDocIdOf(update);
|
|
22443
|
+
try {
|
|
22444
|
+
const { document } = requireUpdateEntry(update);
|
|
22445
|
+
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
22446
|
+
} catch (err) {
|
|
22447
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
22448
|
+
}
|
|
22449
|
+
}
|
|
22450
|
+
const { entry } = await prepareDocumentWrites(
|
|
22451
|
+
ctx,
|
|
22452
|
+
indexName,
|
|
22453
|
+
preparations.map((pending) => pending.preparation),
|
|
22454
|
+
hold,
|
|
22455
|
+
false
|
|
22456
|
+
);
|
|
22457
|
+
const prepared = [];
|
|
22458
|
+
for (const { docId, preparation } of preparations) {
|
|
22459
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
22460
|
+
try {
|
|
22461
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
22462
|
+
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
22463
|
+
} catch (err) {
|
|
22464
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
22465
|
+
}
|
|
22466
|
+
}
|
|
22467
|
+
return prepared;
|
|
22468
|
+
}
|
|
22469
|
+
async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
22231
22470
|
ctx.guardShutdown();
|
|
22232
|
-
|
|
22471
|
+
ctx.requireIndex(indexName);
|
|
22233
22472
|
validateBatchList(updates, "updates");
|
|
22234
22473
|
const succeeded = [];
|
|
22235
22474
|
const failed = [];
|
|
22236
|
-
const updateBatchManager = ctx.requireManager(indexName);
|
|
22237
|
-
const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
|
|
22238
22475
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
22239
22476
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
22240
22477
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
22241
|
-
const prepared =
|
|
22242
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
22243
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
22244
|
-
const update = updates[i];
|
|
22245
|
-
const docId = reportedDocIdOf(update);
|
|
22246
|
-
try {
|
|
22247
|
-
const { document } = requireUpdateEntry(update);
|
|
22248
|
-
prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document));
|
|
22249
|
-
} catch (err) {
|
|
22250
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
22251
|
-
}
|
|
22252
|
-
}
|
|
22478
|
+
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
22253
22479
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
22254
22480
|
let failures;
|
|
22255
22481
|
try {
|
|
@@ -22276,6 +22502,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
22276
22502
|
} finally {
|
|
22277
22503
|
for (const update of prepared) update.releaseVectorHolds?.();
|
|
22278
22504
|
}
|
|
22505
|
+
const settled = [];
|
|
22279
22506
|
for (let i = 0; i < prepared.length; i++) {
|
|
22280
22507
|
const update = prepared[i];
|
|
22281
22508
|
const failure = failures[i];
|
|
@@ -22283,19 +22510,21 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
22283
22510
|
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
22284
22511
|
continue;
|
|
22285
22512
|
}
|
|
22286
|
-
if (!update.buffered)
|
|
22513
|
+
if (!update.buffered) settled.push(update);
|
|
22287
22514
|
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
22288
22515
|
if (vec !== null) touchedVectorFields.add(fieldPath);
|
|
22289
22516
|
}
|
|
22290
22517
|
succeeded.push(update.docId);
|
|
22291
22518
|
}
|
|
22519
|
+
await settleUpdates(ctx, indexName, settled, hold);
|
|
22292
22520
|
if (chunkEnd < updates.length) {
|
|
22293
22521
|
ctx.checkHeapPressure(indexName);
|
|
22294
22522
|
await new Promise((r) => setTimeout(r, 0));
|
|
22295
22523
|
}
|
|
22296
22524
|
}
|
|
22525
|
+
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
22297
22526
|
for (const fieldPath of touchedVectorFields) {
|
|
22298
|
-
|
|
22527
|
+
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
22299
22528
|
}
|
|
22300
22529
|
ctx.checkHeapPressure(indexName);
|
|
22301
22530
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
@@ -22355,22 +22584,22 @@ function updateItemOf(update) {
|
|
|
22355
22584
|
update
|
|
22356
22585
|
};
|
|
22357
22586
|
}
|
|
22358
|
-
async function insertDocument2(ctx, indexName, document, docId, options) {
|
|
22587
|
+
async function insertDocument2(ctx, indexName, document, docId, options, hold) {
|
|
22359
22588
|
const block = blockFor(ctx, indexName);
|
|
22360
|
-
if (block === void 0) return insertDocument(ctx, indexName, document, docId, options);
|
|
22589
|
+
if (block === void 0) return insertDocument(ctx, indexName, document, docId, options, hold);
|
|
22361
22590
|
return withReservation(
|
|
22362
22591
|
block.grouping,
|
|
22363
22592
|
indexName,
|
|
22364
22593
|
document,
|
|
22365
|
-
() => insertDocument(ctx, indexName, document, docId, options)
|
|
22594
|
+
() => insertDocument(ctx, indexName, document, docId, options, hold)
|
|
22366
22595
|
);
|
|
22367
22596
|
}
|
|
22368
|
-
async function insertDocumentBatch2(ctx, indexName, documents2, options) {
|
|
22597
|
+
async function insertDocumentBatch2(ctx, indexName, documents2, options, hold) {
|
|
22369
22598
|
const block = blockFor(ctx, indexName);
|
|
22370
|
-
if (block === void 0) return insertDocumentBatch(ctx, indexName, documents2, options);
|
|
22599
|
+
if (block === void 0) return insertDocumentBatch(ctx, indexName, documents2, options, hold);
|
|
22371
22600
|
const grouping = block.grouping;
|
|
22372
22601
|
if (grouping === void 0 || !Array.isArray(documents2) || documents2.length === 0) {
|
|
22373
|
-
return insertDocumentBatch(ctx, indexName, documents2, options);
|
|
22602
|
+
return insertDocumentBatch(ctx, indexName, documents2, options, hold);
|
|
22374
22603
|
}
|
|
22375
22604
|
const items = documents2.map((document) => ({ docId: providedDocId(document) ?? "", document }));
|
|
22376
22605
|
return writeInGroups(
|
|
@@ -22381,26 +22610,27 @@ async function insertDocumentBatch2(ctx, indexName, documents2, options) {
|
|
|
22381
22610
|
ctx,
|
|
22382
22611
|
indexName,
|
|
22383
22612
|
group.map((item) => item.document),
|
|
22384
|
-
options
|
|
22613
|
+
options,
|
|
22614
|
+
hold
|
|
22385
22615
|
)
|
|
22386
22616
|
);
|
|
22387
22617
|
}
|
|
22388
|
-
async function updateDocument2(ctx, indexName, docId, document, options) {
|
|
22618
|
+
async function updateDocument2(ctx, indexName, docId, document, options, hold) {
|
|
22389
22619
|
const block = blockFor(ctx, indexName);
|
|
22390
|
-
if (block === void 0) return updateDocument(ctx, indexName, docId, document, options);
|
|
22620
|
+
if (block === void 0) return updateDocument(ctx, indexName, docId, document, options, hold);
|
|
22391
22621
|
return withReservation(
|
|
22392
22622
|
block.grouping,
|
|
22393
22623
|
indexName,
|
|
22394
22624
|
document,
|
|
22395
|
-
() => updateDocument(ctx, indexName, docId, document, options)
|
|
22625
|
+
() => updateDocument(ctx, indexName, docId, document, options, hold)
|
|
22396
22626
|
);
|
|
22397
22627
|
}
|
|
22398
|
-
async function updateDocumentBatch2(ctx, indexName, updates, options) {
|
|
22628
|
+
async function updateDocumentBatch2(ctx, indexName, updates, options, hold) {
|
|
22399
22629
|
const block = blockFor(ctx, indexName);
|
|
22400
|
-
if (block === void 0) return updateDocumentBatch(ctx, indexName, updates, options);
|
|
22630
|
+
if (block === void 0) return updateDocumentBatch(ctx, indexName, updates, options, hold);
|
|
22401
22631
|
const grouping = block.grouping;
|
|
22402
22632
|
if (grouping === void 0 || !Array.isArray(updates) || updates.length === 0) {
|
|
22403
|
-
return updateDocumentBatch(ctx, indexName, updates, options);
|
|
22633
|
+
return updateDocumentBatch(ctx, indexName, updates, options, hold);
|
|
22404
22634
|
}
|
|
22405
22635
|
return writeInGroups(
|
|
22406
22636
|
grouping,
|
|
@@ -22410,16 +22640,17 @@ async function updateDocumentBatch2(ctx, indexName, updates, options) {
|
|
|
22410
22640
|
ctx,
|
|
22411
22641
|
indexName,
|
|
22412
22642
|
group.map((item) => item.update),
|
|
22413
|
-
options
|
|
22643
|
+
options,
|
|
22644
|
+
hold
|
|
22414
22645
|
)
|
|
22415
22646
|
);
|
|
22416
22647
|
}
|
|
22417
|
-
async function removeDocument4(ctx, indexName, docId, options) {
|
|
22418
|
-
await removeDocument3(ctx, indexName, docId, options);
|
|
22648
|
+
async function removeDocument4(ctx, indexName, docId, options, hold) {
|
|
22649
|
+
await removeDocument3(ctx, indexName, docId, options, hold);
|
|
22419
22650
|
ctx.memoryWriteBlock?.memoryReleased();
|
|
22420
22651
|
}
|
|
22421
|
-
async function removeDocumentBatch2(ctx, indexName, docIds, options) {
|
|
22422
|
-
const result = await removeDocumentBatch(ctx, indexName, docIds, options);
|
|
22652
|
+
async function removeDocumentBatch2(ctx, indexName, docIds, options, hold) {
|
|
22653
|
+
const result = await removeDocumentBatch(ctx, indexName, docIds, options, hold);
|
|
22423
22654
|
if (result.succeeded.length > 0) ctx.memoryWriteBlock?.memoryReleased();
|
|
22424
22655
|
return result;
|
|
22425
22656
|
}
|
|
@@ -22438,7 +22669,7 @@ var suggestionsCode = codeLoader(async () => {
|
|
|
22438
22669
|
const [msgpack, suggestionStore2, code7] = await Promise.all([
|
|
22439
22670
|
loadMessagePack(),
|
|
22440
22671
|
loadSuggestionStore(),
|
|
22441
|
-
import('./suggestions-leaf-
|
|
22672
|
+
import('./suggestions-leaf-PSUGL6MI.mjs')
|
|
22442
22673
|
]);
|
|
22443
22674
|
code7.bindSuggestionsHost(optionalCodeHost(), msgpack, suggestionStore2);
|
|
22444
22675
|
return code7;
|
|
@@ -24052,7 +24283,7 @@ var workerCopies = codeLoader(async () => {
|
|
|
24052
24283
|
const [segmentCodec2, segmentMerge2, code7] = await Promise.all([
|
|
24053
24284
|
loadSegmentCodec(),
|
|
24054
24285
|
loadSegmentMerge(),
|
|
24055
|
-
import('./worker-copies-
|
|
24286
|
+
import('./worker-copies-MW5O5U3D.mjs')
|
|
24056
24287
|
]);
|
|
24057
24288
|
code7.bindWorkerCopiesHost(optionalCodeHost(), segmentCodec2, segmentMerge2);
|
|
24058
24289
|
return code7;
|
|
@@ -24197,14 +24428,11 @@ function writesHostPart() {
|
|
|
24197
24428
|
admitBufferedUpsert,
|
|
24198
24429
|
admitInsert,
|
|
24199
24430
|
arrayPositionsAreStale,
|
|
24200
|
-
asBatchInsertError,
|
|
24201
24431
|
awaitWriteVisibility,
|
|
24432
|
+
capsEachPartition,
|
|
24202
24433
|
changeIndexOutsideExternalWaits,
|
|
24203
|
-
collectRequiredFieldFailures,
|
|
24204
24434
|
createFrozenSegment,
|
|
24205
|
-
createPendingAdmissions,
|
|
24206
24435
|
deleteNestedValue,
|
|
24207
|
-
embedChunkDocuments,
|
|
24208
24436
|
emitSyncChange,
|
|
24209
24437
|
extractVectorFromDoc,
|
|
24210
24438
|
fnv1a,
|
|
@@ -24226,7 +24454,6 @@ function writesHostPart() {
|
|
|
24226
24454
|
prepareStoredSynonyms,
|
|
24227
24455
|
providedDocId,
|
|
24228
24456
|
readHeapStatistics,
|
|
24229
|
-
recordPendingAdmission,
|
|
24230
24457
|
removeDocumentVectors,
|
|
24231
24458
|
requireDocumentObject,
|
|
24232
24459
|
requireDocumentWithinSizeLimit,
|
|
@@ -26582,7 +26809,7 @@ function checkHeapAfterRecovery(core2) {
|
|
|
26582
26809
|
var loaded3 = null;
|
|
26583
26810
|
function loadRerankRuntime() {
|
|
26584
26811
|
if (loaded3 !== null) return Promise.resolve(loaded3);
|
|
26585
|
-
return import('./runtime-
|
|
26812
|
+
return import('./runtime-UOHD2XUJ.mjs').then((runtime) => {
|
|
26586
26813
|
runtime.bindRerankHost(optionalCodeHost());
|
|
26587
26814
|
loaded3 = runtime;
|
|
26588
26815
|
return runtime;
|
|
@@ -27709,6 +27936,22 @@ function clearEngineIndex(core2, indexName) {
|
|
|
27709
27936
|
});
|
|
27710
27937
|
}
|
|
27711
27938
|
|
|
27939
|
+
// src/engine/index-hold.ts
|
|
27940
|
+
async function holdIndex(indexState, indexName) {
|
|
27941
|
+
let release = await indexState.acquire(indexName);
|
|
27942
|
+
return {
|
|
27943
|
+
release: () => release(),
|
|
27944
|
+
async outside(work) {
|
|
27945
|
+
release();
|
|
27946
|
+
try {
|
|
27947
|
+
return await work();
|
|
27948
|
+
} finally {
|
|
27949
|
+
release = await indexState.acquire(indexName);
|
|
27950
|
+
}
|
|
27951
|
+
}
|
|
27952
|
+
};
|
|
27953
|
+
}
|
|
27954
|
+
|
|
27712
27955
|
// src/narsil/bm25.ts
|
|
27713
27956
|
function bm25Operations(core2) {
|
|
27714
27957
|
return {
|
|
@@ -28583,7 +28826,7 @@ async function executePreflight(request, callerContext) {
|
|
|
28583
28826
|
|
|
28584
28827
|
// src/engine/resolve-vector-text-code.ts
|
|
28585
28828
|
var resolveVectorText = codeLoader(
|
|
28586
|
-
() => import('./resolve-vector-text-
|
|
28829
|
+
() => import('./resolve-vector-text-FU7D4QP5.mjs').then((code7) => {
|
|
28587
28830
|
code7.bindResolveVectorTextHost(optionalCodeHost());
|
|
28588
28831
|
return code7;
|
|
28589
28832
|
}),
|
|
@@ -28596,10 +28839,10 @@ async function enterRead(core2, indexName) {
|
|
|
28596
28839
|
core2.guardShutdown();
|
|
28597
28840
|
if (core2.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
28598
28841
|
}
|
|
28599
|
-
async function
|
|
28600
|
-
if (params.vector?.text === void 0) return
|
|
28601
|
-
const {
|
|
28602
|
-
return
|
|
28842
|
+
async function embeddedFor(core2, indexName, params, signal, earlier) {
|
|
28843
|
+
if (params.vector?.text === void 0) return null;
|
|
28844
|
+
const { embedQueryWithAdapterOf } = await loadResolveVectorText();
|
|
28845
|
+
return embedQueryWithAdapterOf(indexName, params, core2.requireIndex(indexName), signal, earlier);
|
|
28603
28846
|
}
|
|
28604
28847
|
function phraseReadinessOf(core2, indexName, config) {
|
|
28605
28848
|
return () => {
|
|
@@ -28624,24 +28867,17 @@ function broadcastStatsFor(core2, options) {
|
|
|
28624
28867
|
}
|
|
28625
28868
|
return (name, tokens) => invalidation.broadcastStats(name, tokens);
|
|
28626
28869
|
}
|
|
28627
|
-
|
|
28628
|
-
|
|
28629
|
-
|
|
28630
|
-
await
|
|
28631
|
-
|
|
28870
|
+
var EMBEDDING_ADAPTER_CHANGED = 0;
|
|
28871
|
+
var INDEX_REPLACED_DURING_RERANK = 1;
|
|
28872
|
+
async function queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options) {
|
|
28873
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
28874
|
+
let runner;
|
|
28632
28875
|
try {
|
|
28633
28876
|
const entry = core2.requireIndex(indexName);
|
|
28877
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return EMBEDDING_ADAPTER_CHANGED;
|
|
28634
28878
|
const manager = core2.requireManager(indexName);
|
|
28635
|
-
const scoped = scopedParams(params, options);
|
|
28636
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
28637
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
28638
|
-
}
|
|
28639
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
28640
|
-
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
28641
|
-
const resolvedParams = preparedParams.vector?.text === void 0 ? preparedParams : await resolveVectorText2(preparedParams, entry, signal);
|
|
28642
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
28643
|
-
if (loading4 !== null) await loading4;
|
|
28644
28879
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
28880
|
+
runner = rerank?.rerankRunnerFor(core2, options?.signal, hold, indexName, manager);
|
|
28645
28881
|
let analysisStale = false;
|
|
28646
28882
|
const result = await answerFromOneState(
|
|
28647
28883
|
manager,
|
|
@@ -28662,16 +28898,49 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
28662
28898
|
indexBm25: options?.indexBm25,
|
|
28663
28899
|
cursorBinding: lazyQueryBinding(params),
|
|
28664
28900
|
patternWorkCap: core2.patternWorkCap,
|
|
28665
|
-
rerank:
|
|
28901
|
+
rerank: runner,
|
|
28666
28902
|
requirePhraseReady: phraseReadinessOf(core2, indexName, entry.config)
|
|
28667
28903
|
});
|
|
28668
28904
|
},
|
|
28669
28905
|
options?.signal
|
|
28670
28906
|
);
|
|
28907
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
28671
28908
|
if (analysisStale) {
|
|
28672
28909
|
result.analysisStale = true;
|
|
28673
28910
|
}
|
|
28674
28911
|
options?.onRanked?.(manager, result);
|
|
28912
|
+
return result;
|
|
28913
|
+
} catch (error) {
|
|
28914
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
28915
|
+
throw error;
|
|
28916
|
+
} finally {
|
|
28917
|
+
hold.release();
|
|
28918
|
+
}
|
|
28919
|
+
}
|
|
28920
|
+
async function runEngineQuery(core2, indexName, params, options) {
|
|
28921
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
28922
|
+
options?.signal?.throwIfAborted();
|
|
28923
|
+
await enterRead(core2, indexName);
|
|
28924
|
+
core2.requireIndex(indexName);
|
|
28925
|
+
const scoped = scopedParams(params, options);
|
|
28926
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
28927
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
28928
|
+
}
|
|
28929
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
28930
|
+
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
28931
|
+
let embedded2 = null;
|
|
28932
|
+
let replacedReranks = 0;
|
|
28933
|
+
while (true) {
|
|
28934
|
+
embedded2 = await embeddedFor(core2, indexName, preparedParams, signal, embedded2);
|
|
28935
|
+
const resolvedParams = embedded2?.params ?? preparedParams;
|
|
28936
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
28937
|
+
if (loading4 !== null) await loading4;
|
|
28938
|
+
const result = await queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options);
|
|
28939
|
+
if (result === EMBEDDING_ADAPTER_CHANGED) continue;
|
|
28940
|
+
if (result === INDEX_REPLACED_DURING_RERANK) {
|
|
28941
|
+
rerank?.refuseAfterIndexChanges(indexName, ++replacedReranks);
|
|
28942
|
+
continue;
|
|
28943
|
+
}
|
|
28675
28944
|
if (core2.pluginRegistry.hasHooks("afterSearch")) {
|
|
28676
28945
|
const observed = { ...result, hits: [...result.hits] };
|
|
28677
28946
|
try {
|
|
@@ -28681,26 +28950,15 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
28681
28950
|
}
|
|
28682
28951
|
}
|
|
28683
28952
|
return result;
|
|
28684
|
-
} finally {
|
|
28685
|
-
release();
|
|
28686
28953
|
}
|
|
28687
28954
|
}
|
|
28688
|
-
async function
|
|
28689
|
-
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
28690
|
-
options?.signal?.throwIfAborted();
|
|
28691
|
-
await enterRead(core2, indexName);
|
|
28955
|
+
async function preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options) {
|
|
28692
28956
|
const release = await core2.indexState.acquire(indexName);
|
|
28693
28957
|
try {
|
|
28694
28958
|
const entry = core2.requireIndex(indexName);
|
|
28959
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return null;
|
|
28695
28960
|
const manager = core2.requireManager(indexName);
|
|
28696
|
-
const scoped = scopedParams(params, options);
|
|
28697
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
28698
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
28699
|
-
}
|
|
28700
|
-
const resolvedParams = scoped.vector?.text === void 0 ? scoped : await resolveVectorText2(scoped, entry, signal);
|
|
28701
28961
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
28702
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
28703
|
-
if (loading4 !== null) await loading4;
|
|
28704
28962
|
let analysisStale = false;
|
|
28705
28963
|
const result = await answerFromOneState(
|
|
28706
28964
|
manager,
|
|
@@ -28733,6 +28991,25 @@ async function runEnginePreflight(core2, indexName, params, options) {
|
|
|
28733
28991
|
release();
|
|
28734
28992
|
}
|
|
28735
28993
|
}
|
|
28994
|
+
async function runEnginePreflight(core2, indexName, params, options) {
|
|
28995
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
28996
|
+
options?.signal?.throwIfAborted();
|
|
28997
|
+
await enterRead(core2, indexName);
|
|
28998
|
+
core2.requireIndex(indexName);
|
|
28999
|
+
const scoped = scopedParams(params, options);
|
|
29000
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
29001
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
29002
|
+
}
|
|
29003
|
+
let embedded2 = null;
|
|
29004
|
+
while (true) {
|
|
29005
|
+
embedded2 = await embeddedFor(core2, indexName, scoped, signal, embedded2);
|
|
29006
|
+
const resolvedParams = embedded2?.params ?? scoped;
|
|
29007
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
29008
|
+
if (loading4 !== null) await loading4;
|
|
29009
|
+
const result = await preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options);
|
|
29010
|
+
if (result !== null) return result;
|
|
29011
|
+
}
|
|
29012
|
+
}
|
|
28736
29013
|
async function runEngineSuggest(core2, indexName, params, partitionIds) {
|
|
28737
29014
|
const { executeSuggest } = await importOptionalReads();
|
|
28738
29015
|
await enterRead(core2, indexName);
|
|
@@ -28818,11 +29095,11 @@ function createNarsilFromCore(core2, config) {
|
|
|
28818
29095
|
async function withOpenIndex2(indexName, action) {
|
|
28819
29096
|
guardShutdown();
|
|
28820
29097
|
requireIndex(indexName);
|
|
28821
|
-
const
|
|
29098
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
28822
29099
|
try {
|
|
28823
|
-
return await action();
|
|
29100
|
+
return await action(hold);
|
|
28824
29101
|
} finally {
|
|
28825
|
-
release();
|
|
29102
|
+
hold.release();
|
|
28826
29103
|
}
|
|
28827
29104
|
}
|
|
28828
29105
|
const rebalancesStarting = /* @__PURE__ */ new Map();
|
|
@@ -28900,22 +29177,22 @@ function createNarsilFromCore(core2, config) {
|
|
|
28900
29177
|
return stats;
|
|
28901
29178
|
},
|
|
28902
29179
|
insert(indexName, document, docId, options) {
|
|
28903
|
-
return withCountedWrite(indexName, () => insertDocument2(mutationCtx, indexName, document, docId, options));
|
|
29180
|
+
return withCountedWrite(indexName, (hold) => insertDocument2(mutationCtx, indexName, document, docId, options, hold));
|
|
28904
29181
|
},
|
|
28905
29182
|
insertBatch(indexName, documents2, options) {
|
|
28906
|
-
return withCountedWrite(indexName, () => insertDocumentBatch2(mutationCtx, indexName, documents2, options));
|
|
29183
|
+
return withCountedWrite(indexName, (hold) => insertDocumentBatch2(mutationCtx, indexName, documents2, options, hold));
|
|
28907
29184
|
},
|
|
28908
29185
|
remove(indexName, docId, options) {
|
|
28909
|
-
return withCountedWrite(indexName, () => removeDocument4(mutationCtx, indexName, docId, options));
|
|
29186
|
+
return withCountedWrite(indexName, (hold) => removeDocument4(mutationCtx, indexName, docId, options, hold));
|
|
28910
29187
|
},
|
|
28911
29188
|
removeBatch(indexName, docIds, options) {
|
|
28912
|
-
return withCountedWrite(indexName, () => removeDocumentBatch2(mutationCtx, indexName, docIds, options));
|
|
29189
|
+
return withCountedWrite(indexName, (hold) => removeDocumentBatch2(mutationCtx, indexName, docIds, options, hold));
|
|
28913
29190
|
},
|
|
28914
29191
|
update(indexName, docId, document, options) {
|
|
28915
|
-
return withCountedWrite(indexName, () => updateDocument2(mutationCtx, indexName, docId, document, options));
|
|
29192
|
+
return withCountedWrite(indexName, (hold) => updateDocument2(mutationCtx, indexName, docId, document, options, hold));
|
|
28916
29193
|
},
|
|
28917
29194
|
updateBatch(indexName, updates, options) {
|
|
28918
|
-
return withCountedWrite(indexName, () => updateDocumentBatch2(mutationCtx, indexName, updates, options));
|
|
29195
|
+
return withCountedWrite(indexName, (hold) => updateDocumentBatch2(mutationCtx, indexName, updates, options, hold));
|
|
28919
29196
|
},
|
|
28920
29197
|
waitForWrites(indexName) {
|
|
28921
29198
|
guardShutdown();
|
|
@@ -29108,4 +29385,4 @@ async function createNarsil(config) {
|
|
|
29108
29385
|
// src/index.ts
|
|
29109
29386
|
var VERSION = "0.1.0";
|
|
29110
29387
|
|
|
29111
|
-
export { ARRAY_POSITION_GAP, COMPACTION_SEGMENT_TRIGGER, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_COPY_IDLE_TIMEOUT_MS, DEFAULT_COPY_THRESHOLD, GROUP_FORMAT, GROUP_PATTERN_SOURCE, GROWING_PARTIAL_ENTRY_BYTES, GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY, GROWING_PARTIAL_PENDING_ENTRY_BYTES, GROWING_PARTIAL_RUN_BYTES, GROWING_PARTIAL_RUN_MEMBER_BYTES, GrowingSegment, IDLE_MERGE_DELAY_MS, LIVE_TAIL_FLUSH_DOCUMENTS, LIVE_TAIL_FREEZE_FLOOR, LOAD_BUFFER_YIELD_INTERVAL, LONG_QUERY_THREADS, LivePostings2 as LivePostings, MAIN_COPY_LONE_QUERY_DOCUMENTS, MAX_PENDING_REPLICATION_DOCUMENTS, NO_REMOVED_SPELLINGS, PARTIAL_WORD_RUN_CODE_POINTS, POOL_RESTART_DELAY_MAX_MS, POOL_RESTART_DELAY_MS, PoolPlan, REPLICATION_WINDOW, RETIRED_THREADS_EXIT_WAIT_MS, RUN_MEMBER_START_CAPACITY, RUN_TABLE_START_CAPACITY, SEGMENTS_PER_MERGE, SLICE_CAPACITIES, SUGGESTIONS_INDEX_INFIX, SURFACE_TABLE_START_CAPACITY, SegmentRefusal, StringPlan, SurfaceTable, TERM_REGION_GROWTH_FACTOR, TERM_SLOT_LOAD_DENOMINATOR, TERM_SLOT_LOAD_NUMERATOR, TermLookup, VERSION, afterCurrentTurn, arrayPositionsAreStale, assemblePagedFrozenSegment, availablePatternSearch, bindOptionalWritesHost, bindSegmentCodecHost, bindSegmentMergeHost, buildPartialWordTableFrom, clearNormalizationCache, codePointCountOf, compareScores, configurationPayloadOf, configureNormalizationCache, countDocumentSpellings, countSpelling, createBooleanIndex, createCompositePartition, createEncodedDocumentSource, createEngineIndex, createEnumIndex, createFrozenRemovals, createFrozenSegment, createLoadedGeoIndex, createNarsil, createNumericIndex, createPartitionIndex, createPartitionManager, createPartitionRouter, createPublicationGate, createSharedFrozenSegment, createSpellingTable, createSurfaceRegistry, createWorkerFactory, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, dropEngineIndex, emitEngineEvent, emitSyncChange, emitWriteEventAfter, emptyPendingRegions, encodeDocumentTableData, encodeExternalIdTableData, encodeFieldIndexes, encodeFrozenTokenTableData, encodeLookupKey, encodePartialSpellings, encodePartialSpellingsOf, encodeSegmentState, encodeStringBlob, encodeSurfaceTable, endOfValue, engineCoreOf, expansionSource, generateId, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, getVectorFieldPaths, growPartitionsTo, grownCapacity, hasLanguage, hasStopWords, hasTokenizer, holdSyncEpochChange, holdVectorsForRollback, insertDocument, insertDocumentBatch, insertDocumentVectors, isCompositePartition, isSimdAvailable, isSyncEpoch, kWayMerge, liveFrequency, loadDigest, loadGzip, loadOptionalWrites, loadSchemaCodeFor, loadStorageCode, loadSuggestionsCode, lookupOfStoredSynonyms, mergeDocFrequencies, mergeFacets, mergeFieldRegions, mergeKeywordSources, mergePartitionStats, mergePostings, mergePostings2, mergeStoredFields, mergeText, mergeTextAt, newSyncEpoch, oneAtATime, oversampledFacetConfig, oversizedConfigurationPayloadOf, partialSpellingsOfDocuments, partitionDocumentFrequency, partitionStateOf, patternHost, patternIndexOf, prepareConfiguredSynonyms, prepareDocumentText, prepareDocumentVectors, prepareRemovalText, prepareStoredSynonyms, readHeapStatistics, readResidentMemory, rebuildSizeIndexes, reconstructSchemaFromMetadata, recordValueSize, regionsOverlap, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, releaseRange, removeDocument3 as removeDocument, removeDocumentBatch, removeDocumentBatch2, removeDocumentVectors, removedSurfaceSpellings, requireEncodedConfigurationWithinSizeLimit, requireLoadedVectorCode, requirePatternSearch, requireRoom, requireSegment, requireSurfaceFormsForPartialFields, reserveStrings, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, resolveStopWords, returnsEveryMatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, rulesOfStoredSynonyms, sameFields, scanPostingBlocks, spellingCountsOf, storedArrayPositionGap, storedBM25Params, storedDocumentsOf, survivorDocumentsOf, termText, tokenize, topLevelMapKeys, transformToken, typedRegion, updateDocument, updateDocumentBatch, updateDocumentVectors, validateBM25Params, validateEmbeddingConfig, validatePartitionConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, visitDocumentSpellings, visitDocumentSurfaceSpellings, withOpenIndex, workersEnabledByDefault, wrapFrozenTokenTable, writePartitionDocument, yieldToEventLoop };
|
|
29388
|
+
export { ARRAY_POSITION_GAP, COMPACTION_SEGMENT_TRIGGER, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_COPY_IDLE_TIMEOUT_MS, DEFAULT_COPY_THRESHOLD, GROUP_FORMAT, GROUP_PATTERN_SOURCE, GROWING_PARTIAL_ENTRY_BYTES, GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY, GROWING_PARTIAL_PENDING_ENTRY_BYTES, GROWING_PARTIAL_RUN_BYTES, GROWING_PARTIAL_RUN_MEMBER_BYTES, GrowingSegment, IDLE_MERGE_DELAY_MS, LIVE_TAIL_FLUSH_DOCUMENTS, LIVE_TAIL_FREEZE_FLOOR, LOAD_BUFFER_YIELD_INTERVAL, LONG_QUERY_THREADS, LivePostings2 as LivePostings, MAIN_COPY_LONE_QUERY_DOCUMENTS, MAX_PENDING_REPLICATION_DOCUMENTS, NO_REMOVED_SPELLINGS, PARTIAL_WORD_RUN_CODE_POINTS, POOL_RESTART_DELAY_MAX_MS, POOL_RESTART_DELAY_MS, PoolPlan, REPLICATION_WINDOW, RETIRED_THREADS_EXIT_WAIT_MS, RUN_MEMBER_START_CAPACITY, RUN_TABLE_START_CAPACITY, SEGMENTS_PER_MERGE, SLICE_CAPACITIES, SUGGESTIONS_INDEX_INFIX, SURFACE_TABLE_START_CAPACITY, SegmentRefusal, StringPlan, SurfaceTable, TERM_REGION_GROWTH_FACTOR, TERM_SLOT_LOAD_DENOMINATOR, TERM_SLOT_LOAD_NUMERATOR, TermLookup, VERSION, afterCurrentTurn, arrayPositionsAreStale, assemblePagedFrozenSegment, availablePatternSearch, bindOptionalWritesHost, bindSegmentCodecHost, bindSegmentMergeHost, buildPartialWordTableFrom, clearNormalizationCache, codePointCountOf, compareScores, configurationPayloadOf, configureNormalizationCache, countDocumentSpellings, countSpelling, createBooleanIndex, createCompositePartition, createEncodedDocumentSource, createEngineIndex, createEnumIndex, createFrozenRemovals, createFrozenSegment, createLoadedGeoIndex, createNarsil, createNumericIndex, createPartitionIndex, createPartitionManager, createPartitionRouter, createPublicationGate, createSharedFrozenSegment, createSpellingTable, createSurfaceRegistry, createWorkerFactory, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, dropEngineIndex, emitEngineEvent, emitSyncChange, emitWriteEventAfter, emptyPendingRegions, encodeDocumentTableData, encodeExternalIdTableData, encodeFieldIndexes, encodeFrozenTokenTableData, encodeLookupKey, encodePartialSpellings, encodePartialSpellingsOf, encodeSegmentState, encodeStringBlob, encodeSurfaceTable, endOfValue, engineCoreOf, expansionSource, generateId, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, getVectorFieldPaths, growPartitionsTo, grownCapacity, hasLanguage, hasStopWords, hasTokenizer, holdSyncEpochChange, holdVectorsForRollback, insertDocument, insertDocumentBatch, insertDocumentVectors, isCompositePartition, isSimdAvailable, isSyncEpoch, kWayMerge, liveFrequency, loadDigest, loadGzip, loadOptionalWrites, loadSchemaCodeFor, loadStorageCode, loadSuggestionsCode, lookupOfStoredSynonyms, mergeDocFrequencies, mergeFacets, mergeFieldRegions, mergeKeywordSources, mergePartitionStats, mergePostings, mergePostings2, mergeStoredFields, mergeText, mergeTextAt, newSyncEpoch, oneAtATime, oversampledFacetConfig, oversizedConfigurationPayloadOf, partialSpellingsOfDocuments, partitionDocumentFrequency, partitionStateOf, patternHost, patternIndexOf, prepareConfiguredSynonyms, prepareDocumentText, prepareDocumentVectors, prepareDocumentWrite, prepareRemovalText, prepareStoredSynonyms, readHeapStatistics, readResidentMemory, rebuildSizeIndexes, reconstructSchemaFromMetadata, recordValueSize, regionsOverlap, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, releaseRange, removeDocument3 as removeDocument, removeDocumentBatch, removeDocumentBatch2, removeDocumentVectors, removedSurfaceSpellings, requireEncodedConfigurationWithinSizeLimit, requireLoadedVectorCode, requirePatternSearch, requireRoom, requireSegment, requireSurfaceFormsForPartialFields, reserveStrings, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, resolveStopWords, returnsEveryMatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, rulesOfStoredSynonyms, sameFields, scanPostingBlocks, spellingCountsOf, storedArrayPositionGap, storedBM25Params, storedDocumentsOf, survivorDocumentsOf, termText, tokenize, topLevelMapKeys, transformToken, typedRegion, updateDocument, updateDocumentBatch, updateDocumentVectors, validateBM25Params, validateEmbeddingConfig, validatePartitionConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, visitDocumentSpellings, visitDocumentSurfaceSpellings, withOpenIndex, workersEnabledByDefault, wrapFrozenTokenTable, writePartitionDocument, yieldToEventLoop };
|