@delali/narsil 0.3.1-next.183 → 0.3.1-next.184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-ZZUXMJIK.mjs → chunk-CPSHS46N.mjs} +129 -165
- package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
- package/dist/browser/{optional-writes-IQFTXANH.mjs → optional-writes-ZTBHYQMN.mjs} +619 -314
- package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
- package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
- package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
- package/dist/browser/{storage-leaf-2CO6SQ7R.mjs → storage-leaf-4XMCITJ6.mjs} +27 -9
- package/dist/{chunk-PK3BGGHO.mjs → chunk-2KCCQK4R.mjs} +1 -1
- package/dist/{chunk-JVR5KMO4.mjs → chunk-2YWWHMVC.mjs} +2 -2
- package/dist/{chunk-GGG5R4CU.mjs → chunk-3D4IMFSE.mjs} +2 -2
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/{react-native/chunks/chunk-G4YYO5TC.mjs → chunk-4JBTALUC.mjs} +1 -1
- package/dist/{chunk-PXEGKQHS.mjs → chunk-5BUEBIXR.mjs} +2 -2
- package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
- package/dist/{chunk-NPXORGMK.mjs → chunk-7WXGUTWK.mjs} +3 -3
- package/dist/{chunk-2DK74OQL.mjs → chunk-A3OAFUZI.mjs} +1 -1
- package/dist/{chunk-632ABEIR.mjs → chunk-BVZWWMOU.mjs} +1 -1
- package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
- package/dist/{chunk-F2UOTRY7.mjs → chunk-CODGHJVW.mjs} +4 -4
- package/dist/{chunk-6ICOWB6I.mjs → chunk-CYCVPTTR.mjs} +1 -1
- package/dist/{chunk-WUWSUJ2X.mjs → chunk-DU23J5JG.mjs} +5 -5
- package/dist/{chunk-C77OBNPP.mjs → chunk-EDEE7ETQ.mjs} +1 -1
- package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
- package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
- package/dist/{chunk-EGWBZSHK.mjs → chunk-FCQMZDRI.mjs} +2 -2
- package/dist/{chunk-MCAJOLVK.mjs → chunk-G3XRHAJA.mjs} +2 -2
- package/dist/{chunk-FIXUVFUC.mjs → chunk-GGIW6INA.mjs} +7 -6
- package/dist/{chunk-OQANB7SJ.mjs → chunk-H4QYQ5DE.mjs} +1 -1
- package/dist/{chunk-V7F5DOSC.mjs → chunk-HQPK73EW.mjs} +3 -3
- package/dist/{chunk-CKS4QQEC.mjs → chunk-II6XF6M4.mjs} +5 -5
- package/dist/{chunk-3OEXOSWI.mjs → chunk-LR6JJVML.mjs} +4 -4
- package/dist/{chunk-TTTBE4TB.mjs → chunk-M5I64AQW.mjs} +3 -3
- package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
- package/dist/{chunk-M3H7CTIC.mjs → chunk-O337MDRJ.mjs} +5 -5
- package/dist/{chunk-C3EINJNJ.mjs → chunk-SEM3PBRH.mjs} +1 -1
- package/dist/{chunk-CWXR7OP4.mjs → chunk-SL53DHVE.mjs} +804 -527
- package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
- package/dist/{chunk-SA7U67XF.mjs → chunk-URVH6BLO.mjs} +2 -2
- package/dist/{chunk-T3YI4KN6.mjs → chunk-VYSWVQCE.mjs} +1 -1
- package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
- package/dist/{chunk-ZVZ2V2PZ.mjs → chunk-WSIU6YXQ.mjs} +39 -20
- package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
- package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
- package/dist/{direct-executor-JXYIL4OQ.mjs → direct-executor-37IH3ISS.mjs} +5 -4
- package/dist/distribution.mjs +32 -31
- package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +5 -4
- package/dist/index.react-native.mjs +4 -3
- package/dist/{long-query-thread-HPKGRYWM.mjs → long-query-thread-PU2XT527.mjs} +15 -14
- package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
- package/dist/{optional-writes-QE7QUI45.mjs → optional-writes-YIVJMQ4D.mjs} +7 -6
- package/dist/{paged-partition-archive-IK2UOY2G.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +5 -4
- package/dist/persistence/durability/checkpoint-worker.mjs +13 -12
- package/dist/{query-actions-COEFVUH3.mjs → query-actions-7EOKT3WD.mjs} +5 -4
- package/dist/query-suggestions/merge-worker.mjs +8 -7
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.mjs +6 -5
- package/dist/react-native/chunks/{chunk-DZ4FQW5K.mjs → chunk-7TPO4FLE.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-6A5XE2MA.mjs → chunk-CBU4M3EU.mjs} +803 -526
- package/dist/react-native/chunks/{chunk-DN2GVVR4.mjs → chunk-IEY4DARA.mjs} +1 -1
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
- package/dist/react-native/chunks/{chunk-UPN4NUXU.mjs → chunk-T5HF3FW3.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XNIH3WIB.mjs → chunk-ZQPSXN6C.mjs} +1 -1
- package/dist/{chunk-YWSIRKKQ.mjs → react-native/chunks/chunk-ZUFOYZ5P.mjs} +1 -1
- package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
- package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
- package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
- package/dist/react-native/chunks/{optional-writes-YZ7WTR7V.mjs → optional-writes-QETIM6BI.mjs} +5 -4
- package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
- package/dist/react-native/chunks/{restore-queue-IGKD6Q2Y.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
- package/dist/react-native/chunks/{runtime-BUCSAK4U.mjs → runtime-UOHD2XUJ.mjs} +23 -3
- package/dist/react-native/chunks/{segment-codec-leaf-JTTGDT3G.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
- package/dist/react-native/chunks/{segment-merge-leaf-D5ZFJRUB.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
- package/dist/react-native/chunks/{storage-leaf-ZKIM6IF6.mjs → storage-leaf-LNOLKWZB.mjs} +36 -17
- package/dist/react-native/chunks/{store-leaf-PHBOTT5J.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
- package/dist/react-native/chunks/{suggestions-leaf-R7MIH6Q3.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
- package/dist/react-native/chunks/{worker-copies-U35KPVJ7.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
- package/dist/react-native/partial-words.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +4 -3
- package/dist/react-native/sync.mjs +4 -3
- package/dist/react-native/synonyms.mjs +4 -3
- package/dist/{rerank-LYL3D7LU.mjs → rerank-KTV5L7MZ.mjs} +5 -4
- package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
- package/dist/{restore-queue-LF4DTVJC.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
- package/dist/{runtime-4GKUIAKV.mjs → runtime-USQYKN3I.mjs} +23 -3
- package/dist/{segment-codec-leaf-ACALKS2C.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
- package/dist/{segment-merge-leaf-SKMPASFK.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
- package/dist/server.mjs +22 -21
- package/dist/{storage-leaf-OJ55R4C5.mjs → storage-leaf-ZI3H2WJY.mjs} +26 -25
- package/dist/{store-leaf-CWHHEDRK.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
- package/dist/{suggestions-leaf-ZTU3ZTWJ.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
- package/dist/sync/filesystem.mjs +10 -9
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.mjs +5 -4
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +5 -4
- package/dist/{thread-BEGFRULV.mjs → thread-BJXE77AI.mjs} +14 -13
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/{worker-copies-373P6IYZ.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
- package/dist/workers/entry.mjs +11 -10
- package/package.json +7 -7
- package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
- package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, storedText, DEFAULT_PATTERN_WORK_CAP, MAX_FILTER_DEPTH, discardStoredValues,
|
|
1
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, storedText, DEFAULT_PATTERN_WORK_CAP, MAX_FILTER_DEPTH, discardStoredValues, storedValuesOf, bitsetHas, createBitSet, bitsetIsEmpty, bitsetSet, storedValueBytes, prepareStoredValues, cachedStoredFields, bitsetClear, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, buildStoredValues, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, checkedSize, fuzzyPrefixOf, boundedLevenshtein, installStoredFields, FILTER_OPERAND_COPY_DEPTH, fuzzyTermMatches, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIterator, GrowableUint32, bitsetFromSet } from './chunk-4ZICWS4V.mjs';
|
|
2
2
|
import { newSyncEpoch, isSyncEpoch } from './chunk-YJHGOJAC.mjs';
|
|
3
|
-
import { parsedTypeOf, NarsilError, ErrorCodes, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, createNarsilError, withUnpairedSurrogatesReplaced, tokenizeOptions, getNestedValue, isPlainObject, MAX_NESTING_DEPTH, loadMessagePack, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, extractVectorFieldsFromSchema,
|
|
3
|
+
import { parsedTypeOf, NarsilError, ErrorCodes, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, createNarsilError, withUnpairedSurrogatesReplaced, tokenizeOptions, getNestedValue, isPlainObject, MAX_NESTING_DEPTH, loadMessagePack, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, extractVectorFieldsFromSchema, loadedMessagePack, flattenSchema, vectorDimensionOf, isSingleStringFieldType, schemaFieldsOf, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isStringValuedBase, compareCodePoints, isWordIndexedFieldType, messagePackUpperBoundBytes, toComparableSortValue, exceedsCodePoints, SORT_VALUE_MAX_CODE_POINTS, PROTOTYPE_POLLUTION_KEYS, validateDocument, validateDocumentStrict, textFieldsChanged, sortModeOf, defaultSortMode, getFlatSchema, getOrCreateFieldNameIndex, isWordIndexedBase, holdsUnpairedSurrogate, validateRequiredFields, requireDocumentWithinSizeLimit, isNarsilError, PLACEMENT_ROOM_REQUESTS, NEAREST_LINK_FALLBACK_HOSTS, MIN_M, MAX_M, MAX_LAYER_CAP, MAX_FILE_BYTES, INITIAL_UPPER_SLOTS, INITIAL_LIST_CAPACITY, FARTHEST_REJECTED_LINKS_KEPT, EARLIER_RELEASE_MIN_M, DEFAULT_EF_SEARCH, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, CONNECT_RESTARTS_BEFORE_ONE_PASS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, VALID_HNSW_METRICS, MAX_DOCUMENT_NESTING_DEPTH, NATIVE_CRC32_MIN_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, raiseEarlierReleaseGraphSetting, MAX_DOCUMENT_ENCODING_DEPTH, DOCUMENT_REPLY_ENVELOPE_DEPTH, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES, truncateSortString, SORT_MODES, isSortMode, codePointCount } from './chunk-2Q45HI5H.mjs';
|
|
4
4
|
|
|
5
5
|
// src/analysis/registry.ts
|
|
6
6
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -4923,6 +4923,7 @@ var MAX_EXPORT_LINE_BYTES = 117443592;
|
|
|
4923
4923
|
var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
|
|
4924
4924
|
var IMPORT_BATCH_DOCUMENTS = 1e3;
|
|
4925
4925
|
var MAX_IMPORT_REFUSALS_REPORTED = 1e3;
|
|
4926
|
+
var MAX_CALLS_ACROSS_INDEX_CHANGES = 3;
|
|
4926
4927
|
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
4927
4928
|
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
4928
4929
|
var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
|
|
@@ -5385,6 +5386,7 @@ function searchHostPart() {
|
|
|
5385
5386
|
listBindingOf,
|
|
5386
5387
|
MAX_HYBRID_DEPTH,
|
|
5387
5388
|
MAX_PHRASE_WORDS,
|
|
5389
|
+
MAX_CALLS_ACROSS_INDEX_CHANGES,
|
|
5388
5390
|
MAX_QUERY_PHRASES,
|
|
5389
5391
|
MAX_RERANK_TIMEOUT_MS,
|
|
5390
5392
|
MAX_RERANK_WINDOW,
|
|
@@ -12382,7 +12384,7 @@ var loadSegmentCodec = segmentCodec.load;
|
|
|
12382
12384
|
|
|
12383
12385
|
// src/engine/optional-writes-code.ts
|
|
12384
12386
|
var optionalWrites = codeLoader(async () => {
|
|
12385
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
12387
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-ZTBHYQMN.mjs')]);
|
|
12386
12388
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
12387
12389
|
return code7;
|
|
12388
12390
|
}, "write");
|
|
@@ -13581,7 +13583,7 @@ var loadSuggestionStore = suggestionStore.load;
|
|
|
13581
13583
|
|
|
13582
13584
|
// src/narsil/restore-queue-code.ts
|
|
13583
13585
|
var restoreQueue = codeLoader(async () => {
|
|
13584
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
13586
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-SG5O735W.mjs')]);
|
|
13585
13587
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
13586
13588
|
return code7;
|
|
13587
13589
|
}, "restore");
|
|
@@ -14172,16 +14174,6 @@ function registerEngineEmbeddingAdapter(core2, name, adapter) {
|
|
|
14172
14174
|
}
|
|
14173
14175
|
}
|
|
14174
14176
|
|
|
14175
|
-
// src/engine/embed-code.ts
|
|
14176
|
-
var embedCode = codeLoader(
|
|
14177
|
-
() => import('./embed-24IPR4LP.mjs').then((code7) => {
|
|
14178
|
-
code7.bindEmbedHost(optionalCodeHost());
|
|
14179
|
-
return code7;
|
|
14180
|
-
}),
|
|
14181
|
-
"embedding"
|
|
14182
|
-
);
|
|
14183
|
-
var loadEmbedCode = embedCode.load;
|
|
14184
|
-
|
|
14185
14177
|
// src/engine/vector-coordinator.ts
|
|
14186
14178
|
function extractVectorFromDoc(document, fieldPath) {
|
|
14187
14179
|
const value = getNestedValue(document, fieldPath);
|
|
@@ -14380,21 +14372,9 @@ function requireMatchingDocId(docId, document) {
|
|
|
14380
14372
|
{ docId, documentId: embeddedDocId }
|
|
14381
14373
|
);
|
|
14382
14374
|
}
|
|
14383
|
-
function asBatchInsertError(err) {
|
|
14384
|
-
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
14385
|
-
}
|
|
14386
|
-
function createPendingAdmissions() {
|
|
14387
|
-
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
14388
|
-
}
|
|
14389
14375
|
function capsEachPartition(ctx, manager) {
|
|
14390
14376
|
return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
|
|
14391
14377
|
}
|
|
14392
|
-
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
14393
|
-
pending.total += 1;
|
|
14394
|
-
if (!capsEachPartition(ctx, manager)) return;
|
|
14395
|
-
const partitionId = manager.routePartition(docId);
|
|
14396
|
-
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
14397
|
-
}
|
|
14398
14378
|
function admitInsert(ctx, indexName, manager, docId, pending) {
|
|
14399
14379
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
14400
14380
|
if (rebalancing) {
|
|
@@ -14426,66 +14406,6 @@ function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
|
|
|
14426
14406
|
ctx.rebalanceTargetPartitionCount(indexName)
|
|
14427
14407
|
);
|
|
14428
14408
|
}
|
|
14429
|
-
function collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed) {
|
|
14430
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
14431
|
-
try {
|
|
14432
|
-
validateRequiredFields(documents2[i], required);
|
|
14433
|
-
} catch (err) {
|
|
14434
|
-
chunkFailedIndexes.add(i);
|
|
14435
|
-
failed.push({ docId: providedDocId(documents2[i]) ?? "", error: asBatchInsertError(err) });
|
|
14436
|
-
}
|
|
14437
|
-
}
|
|
14438
|
-
}
|
|
14439
|
-
async function embedChunkDocuments(entry, documents2, chunkStart, chunkEnd, signal, chunkFailedIndexes, failed) {
|
|
14440
|
-
if (entry.config.embedding === void 0) return;
|
|
14441
|
-
const { assertDocumentCarriesMappedVectors, embedBatchDocumentFields } = await loadEmbedCode();
|
|
14442
|
-
if (entry.embeddingAdapter === null) {
|
|
14443
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
14444
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
14445
|
-
try {
|
|
14446
|
-
assertDocumentCarriesMappedVectors(
|
|
14447
|
-
documents2[i],
|
|
14448
|
-
entry.config.embedding,
|
|
14449
|
-
entry.embeddingAdapterName
|
|
14450
|
-
);
|
|
14451
|
-
} catch (err) {
|
|
14452
|
-
chunkFailedIndexes.add(i);
|
|
14453
|
-
failed.push({
|
|
14454
|
-
docId: providedDocId(documents2[i]) ?? "",
|
|
14455
|
-
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
14456
|
-
});
|
|
14457
|
-
}
|
|
14458
|
-
}
|
|
14459
|
-
return;
|
|
14460
|
-
}
|
|
14461
|
-
const embeddableSlice = [];
|
|
14462
|
-
const embeddableOriginalIndexes = [];
|
|
14463
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
14464
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
14465
|
-
embeddableSlice.push(documents2[i]);
|
|
14466
|
-
embeddableOriginalIndexes.push(i);
|
|
14467
|
-
}
|
|
14468
|
-
if (embeddableSlice.length === 0) return;
|
|
14469
|
-
try {
|
|
14470
|
-
const embedResult = await embedBatchDocumentFields(
|
|
14471
|
-
embeddableSlice,
|
|
14472
|
-
entry.config.embedding,
|
|
14473
|
-
entry.embeddingAdapter,
|
|
14474
|
-
signal
|
|
14475
|
-
);
|
|
14476
|
-
for (const [sliceIndex, error] of embedResult.failed) {
|
|
14477
|
-
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
14478
|
-
chunkFailedIndexes.add(originalIdx);
|
|
14479
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error });
|
|
14480
|
-
}
|
|
14481
|
-
} catch (err) {
|
|
14482
|
-
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
14483
|
-
for (const originalIdx of embeddableOriginalIndexes) {
|
|
14484
|
-
chunkFailedIndexes.add(originalIdx);
|
|
14485
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error: embeddingError });
|
|
14486
|
-
}
|
|
14487
|
-
}
|
|
14488
|
-
}
|
|
14489
14409
|
|
|
14490
14410
|
// src/engine/mutations/write-visibility.ts
|
|
14491
14411
|
async function awaitWriteVisibility(ctx, indexName) {
|
|
@@ -14494,33 +14414,19 @@ async function awaitWriteVisibility(ctx, indexName) {
|
|
|
14494
14414
|
}
|
|
14495
14415
|
|
|
14496
14416
|
// src/engine/mutations/insert.ts
|
|
14497
|
-
async function insertDocument(ctx, indexName, document, docId, options) {
|
|
14417
|
+
async function insertDocument(ctx, indexName, document, docId, options, hold) {
|
|
14498
14418
|
ctx.guardShutdown();
|
|
14499
|
-
|
|
14419
|
+
let entry = ctx.requireIndex(indexName);
|
|
14500
14420
|
if (docId !== void 0) requireMatchingDocId(docId, document);
|
|
14501
14421
|
const resolvedDocId = docId ?? providedDocId(document) ?? ctx.idGenerator();
|
|
14502
14422
|
validateDocId(resolvedDocId);
|
|
14503
|
-
|
|
14504
|
-
if (
|
|
14423
|
+
const beforeInsert = ctx.pluginRegistry.hasHooks("beforeInsert") ? () => ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document }) : null;
|
|
14424
|
+
if (beforeInsert !== null || entry.config.embedding !== void 0) {
|
|
14425
|
+
const { prepareDocumentWrite } = await loadOptionalWrites();
|
|
14426
|
+
entry = await prepareDocumentWrite(ctx, indexName, document, hold, beforeInsert);
|
|
14427
|
+
} else if (entry.config.required !== void 0) {
|
|
14505
14428
|
validateRequiredFields(document, entry.config.required);
|
|
14506
14429
|
}
|
|
14507
|
-
if (entry.config.embedding) {
|
|
14508
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
14509
|
-
if (entry.embeddingAdapter) {
|
|
14510
|
-
await embedDocumentFields(
|
|
14511
|
-
document,
|
|
14512
|
-
entry.config.embedding,
|
|
14513
|
-
entry.embeddingAdapter,
|
|
14514
|
-
ctx.abortController.signal
|
|
14515
|
-
);
|
|
14516
|
-
} else {
|
|
14517
|
-
assertDocumentCarriesMappedVectors(
|
|
14518
|
-
document,
|
|
14519
|
-
entry.config.embedding,
|
|
14520
|
-
entry.embeddingAdapterName
|
|
14521
|
-
);
|
|
14522
|
-
}
|
|
14523
|
-
}
|
|
14524
14430
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
14525
14431
|
if (sizing !== null) await sizing;
|
|
14526
14432
|
const insertManager = ctx.requireManager(indexName);
|
|
@@ -14600,16 +14506,18 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
14600
14506
|
return resolvedDocId;
|
|
14601
14507
|
}
|
|
14602
14508
|
await ctx.orchestrator.settlePublications(indexName);
|
|
14603
|
-
|
|
14604
|
-
|
|
14605
|
-
|
|
14606
|
-
|
|
14509
|
+
if (ctx.pluginRegistry.hasHooks("afterInsert")) {
|
|
14510
|
+
const runHook = async () => {
|
|
14511
|
+
try {
|
|
14512
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: resolvedDocId, document });
|
|
14513
|
+
} catch (err) {
|
|
14514
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
14515
|
+
}
|
|
14516
|
+
};
|
|
14517
|
+
await (hold === void 0 ? runHook() : hold.outside(runHook));
|
|
14607
14518
|
}
|
|
14608
|
-
|
|
14609
|
-
const
|
|
14610
|
-
if (vecIndex) {
|
|
14611
|
-
vecIndex.scheduleBuild();
|
|
14612
|
-
}
|
|
14519
|
+
if (ctx.executor.getManager(indexName) === insertManager) {
|
|
14520
|
+
for (const fieldPath of extractedVectors.keys()) insertVecIndexes.get(fieldPath)?.scheduleBuild();
|
|
14613
14521
|
}
|
|
14614
14522
|
ctx.checkWatermark(indexName, resolvedDocId);
|
|
14615
14523
|
ctx.checkHeapPressure(indexName);
|
|
@@ -19281,7 +19189,7 @@ var storageCode = codeLoader(async () => {
|
|
|
19281
19189
|
loadSegmentMerge(),
|
|
19282
19190
|
loadSuggestionStore(),
|
|
19283
19191
|
loadVectorCore(),
|
|
19284
|
-
import('./storage-leaf-
|
|
19192
|
+
import('./storage-leaf-4XMCITJ6.mjs')
|
|
19285
19193
|
]);
|
|
19286
19194
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
19287
19195
|
return code7;
|
|
@@ -19903,6 +19811,16 @@ function readHeapStatistics() {
|
|
|
19903
19811
|
return null;
|
|
19904
19812
|
}
|
|
19905
19813
|
|
|
19814
|
+
// src/engine/embed-code.ts
|
|
19815
|
+
var embedCode = codeLoader(
|
|
19816
|
+
() => import('./embed-GG6PS3EA.mjs').then((code7) => {
|
|
19817
|
+
code7.bindEmbedHost(optionalCodeHost());
|
|
19818
|
+
return code7;
|
|
19819
|
+
}),
|
|
19820
|
+
"embedding"
|
|
19821
|
+
);
|
|
19822
|
+
var loadEmbedCode = embedCode.load;
|
|
19823
|
+
|
|
19906
19824
|
// src/engine/serial-index-update.ts
|
|
19907
19825
|
var updatingByCore = /* @__PURE__ */ new WeakMap();
|
|
19908
19826
|
async function withOpenIndex(core2, indexName, action) {
|
|
@@ -19940,14 +19858,11 @@ function writesHostPart() {
|
|
|
19940
19858
|
admitBufferedUpsert,
|
|
19941
19859
|
admitInsert,
|
|
19942
19860
|
arrayPositionsAreStale,
|
|
19943
|
-
asBatchInsertError,
|
|
19944
19861
|
awaitWriteVisibility,
|
|
19862
|
+
capsEachPartition,
|
|
19945
19863
|
changeIndexOutsideExternalWaits,
|
|
19946
|
-
collectRequiredFieldFailures,
|
|
19947
19864
|
createFrozenSegment,
|
|
19948
|
-
createPendingAdmissions,
|
|
19949
19865
|
deleteNestedValue,
|
|
19950
|
-
embedChunkDocuments,
|
|
19951
19866
|
emitSyncChange,
|
|
19952
19867
|
extractVectorFromDoc,
|
|
19953
19868
|
fnv1a,
|
|
@@ -19969,7 +19884,6 @@ function writesHostPart() {
|
|
|
19969
19884
|
prepareStoredSynonyms,
|
|
19970
19885
|
providedDocId,
|
|
19971
19886
|
readHeapStatistics,
|
|
19972
|
-
recordPendingAdmission,
|
|
19973
19887
|
removeDocumentVectors,
|
|
19974
19888
|
requireDocumentObject,
|
|
19975
19889
|
requireDocumentWithinSizeLimit,
|
|
@@ -21640,7 +21554,7 @@ function checkHeapAfterRecovery(core2) {
|
|
|
21640
21554
|
var loaded3 = null;
|
|
21641
21555
|
function loadRerankRuntime() {
|
|
21642
21556
|
if (loaded3 !== null) return Promise.resolve(loaded3);
|
|
21643
|
-
return import('./runtime-
|
|
21557
|
+
return import('./runtime-OMKE7DF4.mjs').then((runtime) => {
|
|
21644
21558
|
runtime.bindRerankHost(optionalCodeHost());
|
|
21645
21559
|
loaded3 = runtime;
|
|
21646
21560
|
return runtime;
|
|
@@ -22014,6 +21928,22 @@ function clearEngineIndex(core2, indexName) {
|
|
|
22014
21928
|
});
|
|
22015
21929
|
}
|
|
22016
21930
|
|
|
21931
|
+
// src/engine/index-hold.ts
|
|
21932
|
+
async function holdIndex(indexState, indexName) {
|
|
21933
|
+
let release = await indexState.acquire(indexName);
|
|
21934
|
+
return {
|
|
21935
|
+
release: () => release(),
|
|
21936
|
+
async outside(work) {
|
|
21937
|
+
release();
|
|
21938
|
+
try {
|
|
21939
|
+
return await work();
|
|
21940
|
+
} finally {
|
|
21941
|
+
release = await indexState.acquire(indexName);
|
|
21942
|
+
}
|
|
21943
|
+
}
|
|
21944
|
+
};
|
|
21945
|
+
}
|
|
21946
|
+
|
|
22017
21947
|
// src/narsil/bm25.ts
|
|
22018
21948
|
function bm25Operations(core2) {
|
|
22019
21949
|
return {
|
|
@@ -22888,7 +22818,7 @@ async function executePreflight(request, callerContext) {
|
|
|
22888
22818
|
|
|
22889
22819
|
// src/engine/resolve-vector-text-code.ts
|
|
22890
22820
|
var resolveVectorText = codeLoader(
|
|
22891
|
-
() => import('./resolve-vector-text-
|
|
22821
|
+
() => import('./resolve-vector-text-XRR2OVHE.mjs').then((code7) => {
|
|
22892
22822
|
code7.bindResolveVectorTextHost(optionalCodeHost());
|
|
22893
22823
|
return code7;
|
|
22894
22824
|
}),
|
|
@@ -22901,10 +22831,10 @@ async function enterRead(core2, indexName) {
|
|
|
22901
22831
|
core2.guardShutdown();
|
|
22902
22832
|
if (core2.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
22903
22833
|
}
|
|
22904
|
-
async function
|
|
22905
|
-
if (params.vector?.text === void 0) return
|
|
22906
|
-
const {
|
|
22907
|
-
return
|
|
22834
|
+
async function embeddedFor(core2, indexName, params, signal, earlier) {
|
|
22835
|
+
if (params.vector?.text === void 0) return null;
|
|
22836
|
+
const { embedQueryWithAdapterOf } = await loadResolveVectorText();
|
|
22837
|
+
return embedQueryWithAdapterOf(indexName, params, core2.requireIndex(indexName), signal, earlier);
|
|
22908
22838
|
}
|
|
22909
22839
|
function phraseReadinessOf(core2, indexName, config) {
|
|
22910
22840
|
return () => {
|
|
@@ -22929,24 +22859,17 @@ function broadcastStatsFor(core2, options) {
|
|
|
22929
22859
|
}
|
|
22930
22860
|
return (name, tokens) => invalidation.broadcastStats(name, tokens);
|
|
22931
22861
|
}
|
|
22932
|
-
|
|
22933
|
-
|
|
22934
|
-
|
|
22935
|
-
await
|
|
22936
|
-
|
|
22862
|
+
var EMBEDDING_ADAPTER_CHANGED = 0;
|
|
22863
|
+
var INDEX_REPLACED_DURING_RERANK = 1;
|
|
22864
|
+
async function queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options) {
|
|
22865
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
22866
|
+
let runner;
|
|
22937
22867
|
try {
|
|
22938
22868
|
const entry = core2.requireIndex(indexName);
|
|
22869
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return EMBEDDING_ADAPTER_CHANGED;
|
|
22939
22870
|
const manager = core2.requireManager(indexName);
|
|
22940
|
-
const scoped = scopedParams(params, options);
|
|
22941
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
22942
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
22943
|
-
}
|
|
22944
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
22945
|
-
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
22946
|
-
const resolvedParams = preparedParams.vector?.text === void 0 ? preparedParams : await resolveVectorText2(preparedParams, entry, signal);
|
|
22947
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
22948
|
-
if (loading4 !== null) await loading4;
|
|
22949
22871
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
22872
|
+
runner = rerank?.rerankRunnerFor(core2, options?.signal, hold, indexName, manager);
|
|
22950
22873
|
let analysisStale = false;
|
|
22951
22874
|
const result = await answerFromOneState(
|
|
22952
22875
|
manager,
|
|
@@ -22967,16 +22890,49 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
22967
22890
|
indexBm25: options?.indexBm25,
|
|
22968
22891
|
cursorBinding: lazyQueryBinding(params),
|
|
22969
22892
|
patternWorkCap: core2.patternWorkCap,
|
|
22970
|
-
rerank:
|
|
22893
|
+
rerank: runner,
|
|
22971
22894
|
requirePhraseReady: phraseReadinessOf(core2, indexName, entry.config)
|
|
22972
22895
|
});
|
|
22973
22896
|
},
|
|
22974
22897
|
options?.signal
|
|
22975
22898
|
);
|
|
22899
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
22976
22900
|
if (analysisStale) {
|
|
22977
22901
|
result.analysisStale = true;
|
|
22978
22902
|
}
|
|
22979
22903
|
options?.onRanked?.(manager, result);
|
|
22904
|
+
return result;
|
|
22905
|
+
} catch (error) {
|
|
22906
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
22907
|
+
throw error;
|
|
22908
|
+
} finally {
|
|
22909
|
+
hold.release();
|
|
22910
|
+
}
|
|
22911
|
+
}
|
|
22912
|
+
async function runEngineQuery(core2, indexName, params, options) {
|
|
22913
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
22914
|
+
options?.signal?.throwIfAborted();
|
|
22915
|
+
await enterRead(core2, indexName);
|
|
22916
|
+
core2.requireIndex(indexName);
|
|
22917
|
+
const scoped = scopedParams(params, options);
|
|
22918
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
22919
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
22920
|
+
}
|
|
22921
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
22922
|
+
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
22923
|
+
let embedded2 = null;
|
|
22924
|
+
let replacedReranks = 0;
|
|
22925
|
+
while (true) {
|
|
22926
|
+
embedded2 = await embeddedFor(core2, indexName, preparedParams, signal, embedded2);
|
|
22927
|
+
const resolvedParams = embedded2?.params ?? preparedParams;
|
|
22928
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
22929
|
+
if (loading4 !== null) await loading4;
|
|
22930
|
+
const result = await queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options);
|
|
22931
|
+
if (result === EMBEDDING_ADAPTER_CHANGED) continue;
|
|
22932
|
+
if (result === INDEX_REPLACED_DURING_RERANK) {
|
|
22933
|
+
rerank?.refuseAfterIndexChanges(indexName, ++replacedReranks);
|
|
22934
|
+
continue;
|
|
22935
|
+
}
|
|
22980
22936
|
if (core2.pluginRegistry.hasHooks("afterSearch")) {
|
|
22981
22937
|
const observed = { ...result, hits: [...result.hits] };
|
|
22982
22938
|
try {
|
|
@@ -22986,26 +22942,15 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
22986
22942
|
}
|
|
22987
22943
|
}
|
|
22988
22944
|
return result;
|
|
22989
|
-
} finally {
|
|
22990
|
-
release();
|
|
22991
22945
|
}
|
|
22992
22946
|
}
|
|
22993
|
-
async function
|
|
22994
|
-
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
22995
|
-
options?.signal?.throwIfAborted();
|
|
22996
|
-
await enterRead(core2, indexName);
|
|
22947
|
+
async function preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options) {
|
|
22997
22948
|
const release = await core2.indexState.acquire(indexName);
|
|
22998
22949
|
try {
|
|
22999
22950
|
const entry = core2.requireIndex(indexName);
|
|
22951
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return null;
|
|
23000
22952
|
const manager = core2.requireManager(indexName);
|
|
23001
|
-
const scoped = scopedParams(params, options);
|
|
23002
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
23003
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
23004
|
-
}
|
|
23005
|
-
const resolvedParams = scoped.vector?.text === void 0 ? scoped : await resolveVectorText2(scoped, entry, signal);
|
|
23006
22953
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
23007
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
23008
|
-
if (loading4 !== null) await loading4;
|
|
23009
22954
|
let analysisStale = false;
|
|
23010
22955
|
const result = await answerFromOneState(
|
|
23011
22956
|
manager,
|
|
@@ -23038,6 +22983,25 @@ async function runEnginePreflight(core2, indexName, params, options) {
|
|
|
23038
22983
|
release();
|
|
23039
22984
|
}
|
|
23040
22985
|
}
|
|
22986
|
+
async function runEnginePreflight(core2, indexName, params, options) {
|
|
22987
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
22988
|
+
options?.signal?.throwIfAborted();
|
|
22989
|
+
await enterRead(core2, indexName);
|
|
22990
|
+
core2.requireIndex(indexName);
|
|
22991
|
+
const scoped = scopedParams(params, options);
|
|
22992
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
22993
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
22994
|
+
}
|
|
22995
|
+
let embedded2 = null;
|
|
22996
|
+
while (true) {
|
|
22997
|
+
embedded2 = await embeddedFor(core2, indexName, scoped, signal, embedded2);
|
|
22998
|
+
const resolvedParams = embedded2?.params ?? scoped;
|
|
22999
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
23000
|
+
if (loading4 !== null) await loading4;
|
|
23001
|
+
const result = await preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options);
|
|
23002
|
+
if (result !== null) return result;
|
|
23003
|
+
}
|
|
23004
|
+
}
|
|
23041
23005
|
async function runEngineSuggest(core2, indexName, params, partitionIds) {
|
|
23042
23006
|
const { executeSuggest } = await importOptionalReads();
|
|
23043
23007
|
await enterRead(core2, indexName);
|
|
@@ -23123,11 +23087,11 @@ function createNarsilFromCore(core2, config) {
|
|
|
23123
23087
|
async function withOpenIndex2(indexName, action) {
|
|
23124
23088
|
guardShutdown();
|
|
23125
23089
|
requireIndex(indexName);
|
|
23126
|
-
const
|
|
23090
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
23127
23091
|
try {
|
|
23128
|
-
return await action();
|
|
23092
|
+
return await action(hold);
|
|
23129
23093
|
} finally {
|
|
23130
|
-
release();
|
|
23094
|
+
hold.release();
|
|
23131
23095
|
}
|
|
23132
23096
|
}
|
|
23133
23097
|
const rebalancesStarting = /* @__PURE__ */ new Map();
|
|
@@ -23205,22 +23169,22 @@ function createNarsilFromCore(core2, config) {
|
|
|
23205
23169
|
return stats;
|
|
23206
23170
|
},
|
|
23207
23171
|
insert(indexName, document, docId, options) {
|
|
23208
|
-
return withCountedWrite(indexName, () => insertDocument(mutationCtx, indexName, document, docId, options));
|
|
23172
|
+
return withCountedWrite(indexName, (hold) => insertDocument(mutationCtx, indexName, document, docId, options, hold));
|
|
23209
23173
|
},
|
|
23210
23174
|
insertBatch(indexName, documents2, options) {
|
|
23211
|
-
return withCountedWrite(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents2, options));
|
|
23175
|
+
return withCountedWrite(indexName, (hold) => insertDocumentBatch(mutationCtx, indexName, documents2, options, hold));
|
|
23212
23176
|
},
|
|
23213
23177
|
remove(indexName, docId, options) {
|
|
23214
|
-
return withCountedWrite(indexName, () => removeDocument2(mutationCtx, indexName, docId, options));
|
|
23178
|
+
return withCountedWrite(indexName, (hold) => removeDocument2(mutationCtx, indexName, docId, options, hold));
|
|
23215
23179
|
},
|
|
23216
23180
|
removeBatch(indexName, docIds, options) {
|
|
23217
|
-
return withCountedWrite(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
|
|
23181
|
+
return withCountedWrite(indexName, (hold) => removeDocumentBatch(mutationCtx, indexName, docIds, options, hold));
|
|
23218
23182
|
},
|
|
23219
23183
|
update(indexName, docId, document, options) {
|
|
23220
|
-
return withCountedWrite(indexName, () => updateDocument(mutationCtx, indexName, docId, document, options));
|
|
23184
|
+
return withCountedWrite(indexName, (hold) => updateDocument(mutationCtx, indexName, docId, document, options, hold));
|
|
23221
23185
|
},
|
|
23222
23186
|
updateBatch(indexName, updates, options) {
|
|
23223
|
-
return withCountedWrite(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
|
|
23187
|
+
return withCountedWrite(indexName, (hold) => updateDocumentBatch(mutationCtx, indexName, updates, options, hold));
|
|
23224
23188
|
},
|
|
23225
23189
|
waitForWrites(indexName) {
|
|
23226
23190
|
guardShutdown();
|
|
@@ -218,5 +218,47 @@ async function embedBatchDocumentFields(documents, embeddingConfig, adapter, sig
|
|
|
218
218
|
}
|
|
219
219
|
return { embedded, failed };
|
|
220
220
|
}
|
|
221
|
+
function lacksMappedVectors(documents, embeddingConfig) {
|
|
222
|
+
const targets = Object.keys(embeddingConfig.fields);
|
|
223
|
+
for (const document of documents) {
|
|
224
|
+
for (const target of targets) {
|
|
225
|
+
const existing = resolveFieldValue(document, target);
|
|
226
|
+
if (existing === void 0 || existing === null) return true;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return false;
|
|
230
|
+
}
|
|
231
|
+
function clearWriteEmbedding(earlier) {
|
|
232
|
+
for (const [document, fields] of earlier.fields) {
|
|
233
|
+
for (const field of fields) deleteFieldValue(document, field);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
async function embedDocumentsForWrite(documents, embeddingConfig, adapter, signal, together) {
|
|
237
|
+
const fields = /* @__PURE__ */ new Map();
|
|
238
|
+
const failed = /* @__PURE__ */ new Map();
|
|
239
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
240
|
+
if (together && documents.length > 1) {
|
|
241
|
+
try {
|
|
242
|
+
const result = await embedBatchDocumentFields(documents, embeddingConfig, adapter, signal);
|
|
243
|
+
for (const [position, embedded] of result.embedded) fields.set(documents[position], embedded);
|
|
244
|
+
for (const [position, error] of result.failed) documentFailures.set(position, error);
|
|
245
|
+
} catch (err) {
|
|
246
|
+
const error = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
247
|
+
for (let position = 0; position < documents.length; position++) failed.set(position, error);
|
|
248
|
+
}
|
|
249
|
+
} else {
|
|
250
|
+
for (let position = 0; position < documents.length; position++) {
|
|
251
|
+
try {
|
|
252
|
+
fields.set(
|
|
253
|
+
documents[position],
|
|
254
|
+
await embedDocumentFields(documents[position], embeddingConfig, adapter, signal)
|
|
255
|
+
);
|
|
256
|
+
} catch (err) {
|
|
257
|
+
failed.set(position, err);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return { embedding: { adapter, mapping: embeddingConfig.fields, fields }, failed, documentFailures };
|
|
262
|
+
}
|
|
221
263
|
|
|
222
|
-
export { assertDocumentCarriesMappedVectors, bindEmbedHost, embedBatchDocumentFields, embedDocumentFields };
|
|
264
|
+
export { assertDocumentCarriesMappedVectors, bindEmbedHost, clearWriteEmbedding, embedBatchDocumentFields, embedDocumentFields, embedDocumentsForWrite, lacksMappedVectors };
|