@delali/narsil 0.3.1-next.181 → 0.3.1-next.184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-CPSHS46N.mjs} +139 -177
- package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
- package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
- package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-ZTBHYQMN.mjs} +621 -314
- package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
- package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
- package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
- package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-4XMCITJ6.mjs} +886 -259
- package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
- package/dist/{chunk-KHJPMZE2.mjs → chunk-2KCCQK4R.mjs} +1 -1
- package/dist/{chunk-2KWTQ2P7.mjs → chunk-2YWWHMVC.mjs} +2 -2
- package/dist/{chunk-Y27E2MGC.mjs → chunk-3D4IMFSE.mjs} +2 -2
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/{chunk-XFOL77WH.mjs → chunk-4JBTALUC.mjs} +1 -1
- package/dist/{chunk-ISJ35523.mjs → chunk-5BUEBIXR.mjs} +2 -2
- package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
- package/dist/{chunk-UISV46WX.mjs → chunk-7WXGUTWK.mjs} +3 -3
- package/dist/{chunk-QFJMRB7D.mjs → chunk-A3OAFUZI.mjs} +1 -1
- package/dist/{chunk-GO6TRSJ6.mjs → chunk-BVZWWMOU.mjs} +1 -1
- package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
- package/dist/{chunk-XQQKETAF.mjs → chunk-CODGHJVW.mjs} +4 -4
- package/dist/{chunk-34IOOYXA.mjs → chunk-CYCVPTTR.mjs} +2 -2
- package/dist/{chunk-26E5MO3Y.mjs → chunk-DU23J5JG.mjs} +5 -5
- package/dist/{chunk-JCWC75NH.mjs → chunk-EDEE7ETQ.mjs} +1 -1
- package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
- package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
- package/dist/{chunk-BIIROQ5Q.mjs → chunk-FCQMZDRI.mjs} +2 -2
- package/dist/{chunk-YXZDTKQS.mjs → chunk-G3XRHAJA.mjs} +2 -2
- package/dist/{chunk-XTSFRWAS.mjs → chunk-GGIW6INA.mjs} +7 -6
- package/dist/{chunk-R7PLSVK3.mjs → chunk-H4QYQ5DE.mjs} +1 -1
- package/dist/{chunk-A4XYDUVN.mjs → chunk-HQPK73EW.mjs} +3 -3
- package/dist/{chunk-YPV2L7M5.mjs → chunk-II6XF6M4.mjs} +5 -5
- package/dist/{chunk-F3FDHXCJ.mjs → chunk-LR6JJVML.mjs} +251 -50
- package/dist/{chunk-ZB5X5WAK.mjs → chunk-M5I64AQW.mjs} +11 -6
- package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
- package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
- package/dist/{chunk-XITVDRRH.mjs → chunk-O337MDRJ.mjs} +6 -6
- package/dist/{chunk-XNFBYAXI.mjs → chunk-SEM3PBRH.mjs} +1 -1
- package/dist/{chunk-2C3BXZGA.mjs → chunk-SL53DHVE.mjs} +814 -539
- package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
- package/dist/{chunk-LOQHA6QK.mjs → chunk-URVH6BLO.mjs} +2 -2
- package/dist/{chunk-JDIATDPY.mjs → chunk-VYSWVQCE.mjs} +1 -1
- package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
- package/dist/{chunk-7FC2ZILY.mjs → chunk-WSIU6YXQ.mjs} +67 -78
- package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
- package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
- package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
- package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
- package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-37IH3ISS.mjs} +5 -4
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +398 -144
- package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
- package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
- package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +5 -4
- package/dist/index.react-native.mjs +4 -3
- package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-PU2XT527.mjs} +15 -14
- package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
- package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-YIVJMQ4D.mjs} +9 -6
- package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +5 -4
- package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -16
- package/dist/{query-actions-2JOLPELC.mjs → query-actions-7EOKT3WD.mjs} +5 -4
- package/dist/query-suggestions/merge-worker.mjs +8 -7
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +6 -5
- package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-7TPO4FLE.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-CBU4M3EU.mjs} +813 -538
- package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-IEY4DARA.mjs} +1 -1
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
- package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-T5HF3FW3.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-ZQPSXN6C.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-ZUFOYZ5P.mjs} +1 -1
- package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
- package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
- package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
- package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
- package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-QETIM6BI.mjs} +7 -4
- package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
- package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
- package/dist/{runtime-4GKUIAKV.mjs → react-native/chunks/runtime-UOHD2XUJ.mjs} +23 -3
- package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
- package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
- package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-LNOLKWZB.mjs} +895 -267
- package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
- package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
- package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
- package/dist/react-native/partial-words.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +4 -3
- package/dist/react-native/sync.mjs +4 -3
- package/dist/react-native/synonyms.mjs +4 -3
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-E2HFKKGJ.mjs → rerank-KTV5L7MZ.mjs} +5 -4
- package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
- package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
- package/dist/{react-native/chunks/runtime-BUCSAK4U.mjs → runtime-USQYKN3I.mjs} +23 -3
- package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
- package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +23 -22
- package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-ZI3H2WJY.mjs} +595 -164
- package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
- package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +11 -10
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +5 -4
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +5 -4
- package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
- package/dist/{thread-OWXVP2ZA.mjs → thread-BJXE77AI.mjs} +14 -13
- package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
- package/dist/workers/entry.mjs +11 -10
- package/openapi.json +32 -0
- package/package.json +7 -7
- package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
- package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, storedText, DEFAULT_PATTERN_WORK_CAP, MAX_FILTER_DEPTH, discardStoredValues,
|
|
1
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, storedText, DEFAULT_PATTERN_WORK_CAP, MAX_FILTER_DEPTH, discardStoredValues, storedValuesOf, bitsetHas, createBitSet, bitsetIsEmpty, bitsetSet, storedValueBytes, prepareStoredValues, cachedStoredFields, bitsetClear, bitsetAndNotInto, bitsetAndInto, bitsetOrInto, buildStoredValues, MAX_FILTER_TESTS, TYPO_CLAIM_SLOTS_PER_WORD, checkedSize, fuzzyPrefixOf, boundedLevenshtein, installStoredFields, FILTER_OPERAND_COPY_DEPTH, fuzzyTermMatches, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES, bitsetIterator, GrowableUint32, bitsetFromSet } from './chunk-4ZICWS4V.mjs';
|
|
2
2
|
import { newSyncEpoch, isSyncEpoch } from './chunk-YJHGOJAC.mjs';
|
|
3
|
-
import { parsedTypeOf, NarsilError, ErrorCodes, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, createNarsilError, withUnpairedSurrogatesReplaced, tokenizeOptions, getNestedValue, isPlainObject, MAX_NESTING_DEPTH, loadMessagePack, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, extractVectorFieldsFromSchema,
|
|
3
|
+
import { parsedTypeOf, NarsilError, ErrorCodes, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, createNarsilError, withUnpairedSurrogatesReplaced, tokenizeOptions, getNestedValue, isPlainObject, MAX_NESTING_DEPTH, loadMessagePack, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, extractVectorFieldsFromSchema, loadedMessagePack, flattenSchema, vectorDimensionOf, isSingleStringFieldType, schemaFieldsOf, partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isStringValuedBase, compareCodePoints, isWordIndexedFieldType, messagePackUpperBoundBytes, toComparableSortValue, exceedsCodePoints, SORT_VALUE_MAX_CODE_POINTS, PROTOTYPE_POLLUTION_KEYS, validateDocument, validateDocumentStrict, textFieldsChanged, sortModeOf, defaultSortMode, getFlatSchema, getOrCreateFieldNameIndex, isWordIndexedBase, holdsUnpairedSurrogate, validateRequiredFields, requireDocumentWithinSizeLimit, isNarsilError, PLACEMENT_ROOM_REQUESTS, NEAREST_LINK_FALLBACK_HOSTS, MIN_M, MAX_M, MAX_LAYER_CAP, MAX_FILE_BYTES, INITIAL_UPPER_SLOTS, INITIAL_LIST_CAPACITY, FARTHEST_REJECTED_LINKS_KEPT, EARLIER_RELEASE_MIN_M, DEFAULT_EF_SEARCH, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, CONNECT_RESTARTS_BEFORE_ONE_PASS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, requireValidFieldTypes, parseVectorDimension, isStorageMode, isQuantizationMode, VALID_HNSW_METRICS, MAX_DOCUMENT_NESTING_DEPTH, NATIVE_CRC32_MIN_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, raiseEarlierReleaseGraphSetting, MAX_DOCUMENT_ENCODING_DEPTH, DOCUMENT_REPLY_ENVELOPE_DEPTH, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER, withSortableOption, VECTOR_PATTERN, isListBase, isGeopointOrVectorType, MAX_COVERING_CELLS, LOCATION_CODE_BITS_PER_AXIS, LOCATION_CELLS_PER_AXIS, FULL_TURN_DEGREES, EARTH_RADIUS_METERS, DEG_TO_RAD, ANTIMERIDIAN_SPAN_DEGREES, truncateSortString, SORT_MODES, isSortMode, codePointCount } from './chunk-2Q45HI5H.mjs';
|
|
4
4
|
|
|
5
5
|
// src/analysis/registry.ts
|
|
6
6
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -4923,6 +4923,7 @@ var MAX_EXPORT_LINE_BYTES = 117443592;
|
|
|
4923
4923
|
var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
|
|
4924
4924
|
var IMPORT_BATCH_DOCUMENTS = 1e3;
|
|
4925
4925
|
var MAX_IMPORT_REFUSALS_REPORTED = 1e3;
|
|
4926
|
+
var MAX_CALLS_ACROSS_INDEX_CHANGES = 3;
|
|
4926
4927
|
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
4927
4928
|
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
4928
4929
|
var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
|
|
@@ -5385,6 +5386,7 @@ function searchHostPart() {
|
|
|
5385
5386
|
listBindingOf,
|
|
5386
5387
|
MAX_HYBRID_DEPTH,
|
|
5387
5388
|
MAX_PHRASE_WORDS,
|
|
5389
|
+
MAX_CALLS_ACROSS_INDEX_CHANGES,
|
|
5388
5390
|
MAX_QUERY_PHRASES,
|
|
5389
5391
|
MAX_RERANK_TIMEOUT_MS,
|
|
5390
5392
|
MAX_RERANK_WINDOW,
|
|
@@ -11575,8 +11577,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
11575
11577
|
if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
|
|
11576
11578
|
return removals.spellingsOf(ordinal);
|
|
11577
11579
|
},
|
|
11578
|
-
prepareTombstone(docId, removedSpellings, analysis) {
|
|
11579
|
-
const ordinal = docStore.getInternalId(docId);
|
|
11580
|
+
prepareTombstone(docId, removedSpellings, analysis, ordinal = docStore.getInternalId(docId)) {
|
|
11580
11581
|
if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
|
|
11581
11582
|
const publishTombstone = tombstones.prepare(ordinal);
|
|
11582
11583
|
const strings4 = analysis === void 0 ? null : prepareFrozenStrings(
|
|
@@ -12383,7 +12384,7 @@ var loadSegmentCodec = segmentCodec.load;
|
|
|
12383
12384
|
|
|
12384
12385
|
// src/engine/optional-writes-code.ts
|
|
12385
12386
|
var optionalWrites = codeLoader(async () => {
|
|
12386
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
12387
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-ZTBHYQMN.mjs')]);
|
|
12387
12388
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
12388
12389
|
return code7;
|
|
12389
12390
|
}, "write");
|
|
@@ -13582,7 +13583,7 @@ var loadSuggestionStore = suggestionStore.load;
|
|
|
13582
13583
|
|
|
13583
13584
|
// src/narsil/restore-queue-code.ts
|
|
13584
13585
|
var restoreQueue = codeLoader(async () => {
|
|
13585
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
13586
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-SG5O735W.mjs')]);
|
|
13586
13587
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
13587
13588
|
return code7;
|
|
13588
13589
|
}, "restore");
|
|
@@ -14173,16 +14174,6 @@ function registerEngineEmbeddingAdapter(core2, name, adapter) {
|
|
|
14173
14174
|
}
|
|
14174
14175
|
}
|
|
14175
14176
|
|
|
14176
|
-
// src/engine/embed-code.ts
|
|
14177
|
-
var embedCode = codeLoader(
|
|
14178
|
-
() => import('./embed-24IPR4LP.mjs').then((code7) => {
|
|
14179
|
-
code7.bindEmbedHost(optionalCodeHost());
|
|
14180
|
-
return code7;
|
|
14181
|
-
}),
|
|
14182
|
-
"embedding"
|
|
14183
|
-
);
|
|
14184
|
-
var loadEmbedCode = embedCode.load;
|
|
14185
|
-
|
|
14186
14177
|
// src/engine/vector-coordinator.ts
|
|
14187
14178
|
function extractVectorFromDoc(document, fieldPath) {
|
|
14188
14179
|
const value = getNestedValue(document, fieldPath);
|
|
@@ -14381,21 +14372,9 @@ function requireMatchingDocId(docId, document) {
|
|
|
14381
14372
|
{ docId, documentId: embeddedDocId }
|
|
14382
14373
|
);
|
|
14383
14374
|
}
|
|
14384
|
-
function asBatchInsertError(err) {
|
|
14385
|
-
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
14386
|
-
}
|
|
14387
|
-
function createPendingAdmissions() {
|
|
14388
|
-
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
14389
|
-
}
|
|
14390
14375
|
function capsEachPartition(ctx, manager) {
|
|
14391
14376
|
return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
|
|
14392
14377
|
}
|
|
14393
|
-
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
14394
|
-
pending.total += 1;
|
|
14395
|
-
if (!capsEachPartition(ctx, manager)) return;
|
|
14396
|
-
const partitionId = manager.routePartition(docId);
|
|
14397
|
-
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
14398
|
-
}
|
|
14399
14378
|
function admitInsert(ctx, indexName, manager, docId, pending) {
|
|
14400
14379
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
14401
14380
|
if (rebalancing) {
|
|
@@ -14427,66 +14406,6 @@ function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
|
|
|
14427
14406
|
ctx.rebalanceTargetPartitionCount(indexName)
|
|
14428
14407
|
);
|
|
14429
14408
|
}
|
|
14430
|
-
function collectRequiredFieldFailures(documents2, chunkStart, chunkEnd, required, chunkFailedIndexes, failed) {
|
|
14431
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
14432
|
-
try {
|
|
14433
|
-
validateRequiredFields(documents2[i], required);
|
|
14434
|
-
} catch (err) {
|
|
14435
|
-
chunkFailedIndexes.add(i);
|
|
14436
|
-
failed.push({ docId: providedDocId(documents2[i]) ?? "", error: asBatchInsertError(err) });
|
|
14437
|
-
}
|
|
14438
|
-
}
|
|
14439
|
-
}
|
|
14440
|
-
async function embedChunkDocuments(entry, documents2, chunkStart, chunkEnd, signal, chunkFailedIndexes, failed) {
|
|
14441
|
-
if (entry.config.embedding === void 0) return;
|
|
14442
|
-
const { assertDocumentCarriesMappedVectors, embedBatchDocumentFields } = await loadEmbedCode();
|
|
14443
|
-
if (entry.embeddingAdapter === null) {
|
|
14444
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
14445
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
14446
|
-
try {
|
|
14447
|
-
assertDocumentCarriesMappedVectors(
|
|
14448
|
-
documents2[i],
|
|
14449
|
-
entry.config.embedding,
|
|
14450
|
-
entry.embeddingAdapterName
|
|
14451
|
-
);
|
|
14452
|
-
} catch (err) {
|
|
14453
|
-
chunkFailedIndexes.add(i);
|
|
14454
|
-
failed.push({
|
|
14455
|
-
docId: providedDocId(documents2[i]) ?? "",
|
|
14456
|
-
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
14457
|
-
});
|
|
14458
|
-
}
|
|
14459
|
-
}
|
|
14460
|
-
return;
|
|
14461
|
-
}
|
|
14462
|
-
const embeddableSlice = [];
|
|
14463
|
-
const embeddableOriginalIndexes = [];
|
|
14464
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
14465
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
14466
|
-
embeddableSlice.push(documents2[i]);
|
|
14467
|
-
embeddableOriginalIndexes.push(i);
|
|
14468
|
-
}
|
|
14469
|
-
if (embeddableSlice.length === 0) return;
|
|
14470
|
-
try {
|
|
14471
|
-
const embedResult = await embedBatchDocumentFields(
|
|
14472
|
-
embeddableSlice,
|
|
14473
|
-
entry.config.embedding,
|
|
14474
|
-
entry.embeddingAdapter,
|
|
14475
|
-
signal
|
|
14476
|
-
);
|
|
14477
|
-
for (const [sliceIndex, error] of embedResult.failed) {
|
|
14478
|
-
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
14479
|
-
chunkFailedIndexes.add(originalIdx);
|
|
14480
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error });
|
|
14481
|
-
}
|
|
14482
|
-
} catch (err) {
|
|
14483
|
-
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
14484
|
-
for (const originalIdx of embeddableOriginalIndexes) {
|
|
14485
|
-
chunkFailedIndexes.add(originalIdx);
|
|
14486
|
-
failed.push({ docId: providedDocId(documents2[originalIdx]) ?? "", error: embeddingError });
|
|
14487
|
-
}
|
|
14488
|
-
}
|
|
14489
|
-
}
|
|
14490
14409
|
|
|
14491
14410
|
// src/engine/mutations/write-visibility.ts
|
|
14492
14411
|
async function awaitWriteVisibility(ctx, indexName) {
|
|
@@ -14495,33 +14414,19 @@ async function awaitWriteVisibility(ctx, indexName) {
|
|
|
14495
14414
|
}
|
|
14496
14415
|
|
|
14497
14416
|
// src/engine/mutations/insert.ts
|
|
14498
|
-
async function insertDocument(ctx, indexName, document, docId, options) {
|
|
14417
|
+
async function insertDocument(ctx, indexName, document, docId, options, hold) {
|
|
14499
14418
|
ctx.guardShutdown();
|
|
14500
|
-
|
|
14419
|
+
let entry = ctx.requireIndex(indexName);
|
|
14501
14420
|
if (docId !== void 0) requireMatchingDocId(docId, document);
|
|
14502
14421
|
const resolvedDocId = docId ?? providedDocId(document) ?? ctx.idGenerator();
|
|
14503
14422
|
validateDocId(resolvedDocId);
|
|
14504
|
-
|
|
14505
|
-
if (
|
|
14423
|
+
const beforeInsert = ctx.pluginRegistry.hasHooks("beforeInsert") ? () => ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document }) : null;
|
|
14424
|
+
if (beforeInsert !== null || entry.config.embedding !== void 0) {
|
|
14425
|
+
const { prepareDocumentWrite } = await loadOptionalWrites();
|
|
14426
|
+
entry = await prepareDocumentWrite(ctx, indexName, document, hold, beforeInsert);
|
|
14427
|
+
} else if (entry.config.required !== void 0) {
|
|
14506
14428
|
validateRequiredFields(document, entry.config.required);
|
|
14507
14429
|
}
|
|
14508
|
-
if (entry.config.embedding) {
|
|
14509
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
14510
|
-
if (entry.embeddingAdapter) {
|
|
14511
|
-
await embedDocumentFields(
|
|
14512
|
-
document,
|
|
14513
|
-
entry.config.embedding,
|
|
14514
|
-
entry.embeddingAdapter,
|
|
14515
|
-
ctx.abortController.signal
|
|
14516
|
-
);
|
|
14517
|
-
} else {
|
|
14518
|
-
assertDocumentCarriesMappedVectors(
|
|
14519
|
-
document,
|
|
14520
|
-
entry.config.embedding,
|
|
14521
|
-
entry.embeddingAdapterName
|
|
14522
|
-
);
|
|
14523
|
-
}
|
|
14524
|
-
}
|
|
14525
14430
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
14526
14431
|
if (sizing !== null) await sizing;
|
|
14527
14432
|
const insertManager = ctx.requireManager(indexName);
|
|
@@ -14601,16 +14506,18 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
14601
14506
|
return resolvedDocId;
|
|
14602
14507
|
}
|
|
14603
14508
|
await ctx.orchestrator.settlePublications(indexName);
|
|
14604
|
-
|
|
14605
|
-
|
|
14606
|
-
|
|
14607
|
-
|
|
14509
|
+
if (ctx.pluginRegistry.hasHooks("afterInsert")) {
|
|
14510
|
+
const runHook = async () => {
|
|
14511
|
+
try {
|
|
14512
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: resolvedDocId, document });
|
|
14513
|
+
} catch (err) {
|
|
14514
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
14515
|
+
}
|
|
14516
|
+
};
|
|
14517
|
+
await (hold === void 0 ? runHook() : hold.outside(runHook));
|
|
14608
14518
|
}
|
|
14609
|
-
|
|
14610
|
-
const
|
|
14611
|
-
if (vecIndex) {
|
|
14612
|
-
vecIndex.scheduleBuild();
|
|
14613
|
-
}
|
|
14519
|
+
if (ctx.executor.getManager(indexName) === insertManager) {
|
|
14520
|
+
for (const fieldPath of extractedVectors.keys()) insertVecIndexes.get(fieldPath)?.scheduleBuild();
|
|
14614
14521
|
}
|
|
14615
14522
|
ctx.checkWatermark(indexName, resolvedDocId);
|
|
14616
14523
|
ctx.checkHeapPressure(indexName);
|
|
@@ -19282,7 +19189,7 @@ var storageCode = codeLoader(async () => {
|
|
|
19282
19189
|
loadSegmentMerge(),
|
|
19283
19190
|
loadSuggestionStore(),
|
|
19284
19191
|
loadVectorCore(),
|
|
19285
|
-
import('./storage-leaf-
|
|
19192
|
+
import('./storage-leaf-4XMCITJ6.mjs')
|
|
19286
19193
|
]);
|
|
19287
19194
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
19288
19195
|
return code7;
|
|
@@ -19904,6 +19811,16 @@ function readHeapStatistics() {
|
|
|
19904
19811
|
return null;
|
|
19905
19812
|
}
|
|
19906
19813
|
|
|
19814
|
+
// src/engine/embed-code.ts
|
|
19815
|
+
var embedCode = codeLoader(
|
|
19816
|
+
() => import('./embed-GG6PS3EA.mjs').then((code7) => {
|
|
19817
|
+
code7.bindEmbedHost(optionalCodeHost());
|
|
19818
|
+
return code7;
|
|
19819
|
+
}),
|
|
19820
|
+
"embedding"
|
|
19821
|
+
);
|
|
19822
|
+
var loadEmbedCode = embedCode.load;
|
|
19823
|
+
|
|
19907
19824
|
// src/engine/serial-index-update.ts
|
|
19908
19825
|
var updatingByCore = /* @__PURE__ */ new WeakMap();
|
|
19909
19826
|
async function withOpenIndex(core2, indexName, action) {
|
|
@@ -19941,14 +19858,11 @@ function writesHostPart() {
|
|
|
19941
19858
|
admitBufferedUpsert,
|
|
19942
19859
|
admitInsert,
|
|
19943
19860
|
arrayPositionsAreStale,
|
|
19944
|
-
asBatchInsertError,
|
|
19945
19861
|
awaitWriteVisibility,
|
|
19862
|
+
capsEachPartition,
|
|
19946
19863
|
changeIndexOutsideExternalWaits,
|
|
19947
|
-
collectRequiredFieldFailures,
|
|
19948
19864
|
createFrozenSegment,
|
|
19949
|
-
createPendingAdmissions,
|
|
19950
19865
|
deleteNestedValue,
|
|
19951
|
-
embedChunkDocuments,
|
|
19952
19866
|
emitSyncChange,
|
|
19953
19867
|
extractVectorFromDoc,
|
|
19954
19868
|
fnv1a,
|
|
@@ -19970,7 +19884,6 @@ function writesHostPart() {
|
|
|
19970
19884
|
prepareStoredSynonyms,
|
|
19971
19885
|
providedDocId,
|
|
19972
19886
|
readHeapStatistics,
|
|
19973
|
-
recordPendingAdmission,
|
|
19974
19887
|
removeDocumentVectors,
|
|
19975
19888
|
requireDocumentObject,
|
|
19976
19889
|
requireDocumentWithinSizeLimit,
|
|
@@ -20226,7 +20139,8 @@ var DURABILITY_CONFIG_KEYS = {
|
|
|
20226
20139
|
segmentMaxBytes: true,
|
|
20227
20140
|
checkpointIntervalMs: true,
|
|
20228
20141
|
checkpointMutationThreshold: true,
|
|
20229
|
-
compactionThreshold: true
|
|
20142
|
+
compactionThreshold: true,
|
|
20143
|
+
rewriteBytesPerCheckpoint: true
|
|
20230
20144
|
};
|
|
20231
20145
|
var WORKER_CONFIG_KEYS = {
|
|
20232
20146
|
enabled: true,
|
|
@@ -21068,7 +20982,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
21068
20982
|
|
|
21069
20983
|
// src/engine/durability-config-code.ts
|
|
21070
20984
|
var durabilityConfig = codeLoader(
|
|
21071
|
-
() => import('./durability-config-
|
|
20985
|
+
() => import('./durability-config-75NSFCQN.mjs').then((code7) => {
|
|
21072
20986
|
code7.bindDurabilityConfigHost(optionalCodeHost());
|
|
21073
20987
|
return code7;
|
|
21074
20988
|
}),
|
|
@@ -21640,7 +21554,7 @@ function checkHeapAfterRecovery(core2) {
|
|
|
21640
21554
|
var loaded3 = null;
|
|
21641
21555
|
function loadRerankRuntime() {
|
|
21642
21556
|
if (loaded3 !== null) return Promise.resolve(loaded3);
|
|
21643
|
-
return import('./runtime-
|
|
21557
|
+
return import('./runtime-OMKE7DF4.mjs').then((runtime) => {
|
|
21644
21558
|
runtime.bindRerankHost(optionalCodeHost());
|
|
21645
21559
|
loaded3 = runtime;
|
|
21646
21560
|
return runtime;
|
|
@@ -21787,12 +21701,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
21787
21701
|
queueDiskAdoption: orchestrator.queueDiskAdoption,
|
|
21788
21702
|
awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
|
|
21789
21703
|
publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
|
|
21790
|
-
recordCheckpoint(indexName, documentCount, partitionCount) {
|
|
21791
|
-
const entry = indexRegistry.get(indexName);
|
|
21792
|
-
if (entry === void 0) return;
|
|
21793
|
-
entry.documentCount = documentCount;
|
|
21794
|
-
entry.partitionCount = partitionCount;
|
|
21795
|
-
},
|
|
21796
21704
|
engineCore: () => engineCore
|
|
21797
21705
|
});
|
|
21798
21706
|
const invalidationCode = config?.invalidation === void 0 ? null : durabilityWiring ?? await loadStorageCode();
|
|
@@ -22020,6 +21928,22 @@ function clearEngineIndex(core2, indexName) {
|
|
|
22020
21928
|
});
|
|
22021
21929
|
}
|
|
22022
21930
|
|
|
21931
|
+
// src/engine/index-hold.ts
|
|
21932
|
+
async function holdIndex(indexState, indexName) {
|
|
21933
|
+
let release = await indexState.acquire(indexName);
|
|
21934
|
+
return {
|
|
21935
|
+
release: () => release(),
|
|
21936
|
+
async outside(work) {
|
|
21937
|
+
release();
|
|
21938
|
+
try {
|
|
21939
|
+
return await work();
|
|
21940
|
+
} finally {
|
|
21941
|
+
release = await indexState.acquire(indexName);
|
|
21942
|
+
}
|
|
21943
|
+
}
|
|
21944
|
+
};
|
|
21945
|
+
}
|
|
21946
|
+
|
|
22023
21947
|
// src/narsil/bm25.ts
|
|
22024
21948
|
function bm25Operations(core2) {
|
|
22025
21949
|
return {
|
|
@@ -22894,7 +22818,7 @@ async function executePreflight(request, callerContext) {
|
|
|
22894
22818
|
|
|
22895
22819
|
// src/engine/resolve-vector-text-code.ts
|
|
22896
22820
|
var resolveVectorText = codeLoader(
|
|
22897
|
-
() => import('./resolve-vector-text-
|
|
22821
|
+
() => import('./resolve-vector-text-XRR2OVHE.mjs').then((code7) => {
|
|
22898
22822
|
code7.bindResolveVectorTextHost(optionalCodeHost());
|
|
22899
22823
|
return code7;
|
|
22900
22824
|
}),
|
|
@@ -22907,10 +22831,10 @@ async function enterRead(core2, indexName) {
|
|
|
22907
22831
|
core2.guardShutdown();
|
|
22908
22832
|
if (core2.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
22909
22833
|
}
|
|
22910
|
-
async function
|
|
22911
|
-
if (params.vector?.text === void 0) return
|
|
22912
|
-
const {
|
|
22913
|
-
return
|
|
22834
|
+
async function embeddedFor(core2, indexName, params, signal, earlier) {
|
|
22835
|
+
if (params.vector?.text === void 0) return null;
|
|
22836
|
+
const { embedQueryWithAdapterOf } = await loadResolveVectorText();
|
|
22837
|
+
return embedQueryWithAdapterOf(indexName, params, core2.requireIndex(indexName), signal, earlier);
|
|
22914
22838
|
}
|
|
22915
22839
|
function phraseReadinessOf(core2, indexName, config) {
|
|
22916
22840
|
return () => {
|
|
@@ -22935,24 +22859,17 @@ function broadcastStatsFor(core2, options) {
|
|
|
22935
22859
|
}
|
|
22936
22860
|
return (name, tokens) => invalidation.broadcastStats(name, tokens);
|
|
22937
22861
|
}
|
|
22938
|
-
|
|
22939
|
-
|
|
22940
|
-
|
|
22941
|
-
await
|
|
22942
|
-
|
|
22862
|
+
var EMBEDDING_ADAPTER_CHANGED = 0;
|
|
22863
|
+
var INDEX_REPLACED_DURING_RERANK = 1;
|
|
22864
|
+
async function queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options) {
|
|
22865
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
22866
|
+
let runner;
|
|
22943
22867
|
try {
|
|
22944
22868
|
const entry = core2.requireIndex(indexName);
|
|
22869
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return EMBEDDING_ADAPTER_CHANGED;
|
|
22945
22870
|
const manager = core2.requireManager(indexName);
|
|
22946
|
-
const scoped = scopedParams(params, options);
|
|
22947
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
22948
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
22949
|
-
}
|
|
22950
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
22951
|
-
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
22952
|
-
const resolvedParams = preparedParams.vector?.text === void 0 ? preparedParams : await resolveVectorText2(preparedParams, entry, signal);
|
|
22953
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
22954
|
-
if (loading4 !== null) await loading4;
|
|
22955
22871
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
22872
|
+
runner = rerank?.rerankRunnerFor(core2, options?.signal, hold, indexName, manager);
|
|
22956
22873
|
let analysisStale = false;
|
|
22957
22874
|
const result = await answerFromOneState(
|
|
22958
22875
|
manager,
|
|
@@ -22973,16 +22890,49 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
22973
22890
|
indexBm25: options?.indexBm25,
|
|
22974
22891
|
cursorBinding: lazyQueryBinding(params),
|
|
22975
22892
|
patternWorkCap: core2.patternWorkCap,
|
|
22976
|
-
rerank:
|
|
22893
|
+
rerank: runner,
|
|
22977
22894
|
requirePhraseReady: phraseReadinessOf(core2, indexName, entry.config)
|
|
22978
22895
|
});
|
|
22979
22896
|
},
|
|
22980
22897
|
options?.signal
|
|
22981
22898
|
);
|
|
22899
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
22982
22900
|
if (analysisStale) {
|
|
22983
22901
|
result.analysisStale = true;
|
|
22984
22902
|
}
|
|
22985
22903
|
options?.onRanked?.(manager, result);
|
|
22904
|
+
return result;
|
|
22905
|
+
} catch (error) {
|
|
22906
|
+
if (runner?.replaced) return INDEX_REPLACED_DURING_RERANK;
|
|
22907
|
+
throw error;
|
|
22908
|
+
} finally {
|
|
22909
|
+
hold.release();
|
|
22910
|
+
}
|
|
22911
|
+
}
|
|
22912
|
+
async function runEngineQuery(core2, indexName, params, options) {
|
|
22913
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
22914
|
+
options?.signal?.throwIfAborted();
|
|
22915
|
+
await enterRead(core2, indexName);
|
|
22916
|
+
core2.requireIndex(indexName);
|
|
22917
|
+
const scoped = scopedParams(params, options);
|
|
22918
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
22919
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
22920
|
+
}
|
|
22921
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await loadRerankRuntime();
|
|
22922
|
+
const preparedParams = rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped);
|
|
22923
|
+
let embedded2 = null;
|
|
22924
|
+
let replacedReranks = 0;
|
|
22925
|
+
while (true) {
|
|
22926
|
+
embedded2 = await embeddedFor(core2, indexName, preparedParams, signal, embedded2);
|
|
22927
|
+
const resolvedParams = embedded2?.params ?? preparedParams;
|
|
22928
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
22929
|
+
if (loading4 !== null) await loading4;
|
|
22930
|
+
const result = await queryUnderHold(core2, indexName, params, resolvedParams, embedded2, rerank, options);
|
|
22931
|
+
if (result === EMBEDDING_ADAPTER_CHANGED) continue;
|
|
22932
|
+
if (result === INDEX_REPLACED_DURING_RERANK) {
|
|
22933
|
+
rerank?.refuseAfterIndexChanges(indexName, ++replacedReranks);
|
|
22934
|
+
continue;
|
|
22935
|
+
}
|
|
22986
22936
|
if (core2.pluginRegistry.hasHooks("afterSearch")) {
|
|
22987
22937
|
const observed = { ...result, hits: [...result.hits] };
|
|
22988
22938
|
try {
|
|
@@ -22992,26 +22942,15 @@ async function runEngineQuery(core2, indexName, params, options) {
|
|
|
22992
22942
|
}
|
|
22993
22943
|
}
|
|
22994
22944
|
return result;
|
|
22995
|
-
} finally {
|
|
22996
|
-
release();
|
|
22997
22945
|
}
|
|
22998
22946
|
}
|
|
22999
|
-
async function
|
|
23000
|
-
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
23001
|
-
options?.signal?.throwIfAborted();
|
|
23002
|
-
await enterRead(core2, indexName);
|
|
22947
|
+
async function preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options) {
|
|
23003
22948
|
const release = await core2.indexState.acquire(indexName);
|
|
23004
22949
|
try {
|
|
23005
22950
|
const entry = core2.requireIndex(indexName);
|
|
22951
|
+
if (embedded2 !== null && entry.embeddingAdapter !== embedded2.adapter) return null;
|
|
23006
22952
|
const manager = core2.requireManager(indexName);
|
|
23007
|
-
const scoped = scopedParams(params, options);
|
|
23008
|
-
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
23009
|
-
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
23010
|
-
}
|
|
23011
|
-
const resolvedParams = scoped.vector?.text === void 0 ? scoped : await resolveVectorText2(scoped, entry, signal);
|
|
23012
22953
|
const workerSearch = core2.orchestrator.hasWorkerPool() ? core2.orchestrator.searchViaWorker.bind(core2.orchestrator) : void 0;
|
|
23013
|
-
const loading4 = loadQueryCode(resolvedParams);
|
|
23014
|
-
if (loading4 !== null) await loading4;
|
|
23015
22954
|
let analysisStale = false;
|
|
23016
22955
|
const result = await answerFromOneState(
|
|
23017
22956
|
manager,
|
|
@@ -23044,6 +22983,25 @@ async function runEnginePreflight(core2, indexName, params, options) {
|
|
|
23044
22983
|
release();
|
|
23045
22984
|
}
|
|
23046
22985
|
}
|
|
22986
|
+
async function runEnginePreflight(core2, indexName, params, options) {
|
|
22987
|
+
const signal = options?.signal === void 0 ? core2.abortController.signal : AbortSignal.any([core2.abortController.signal, options.signal]);
|
|
22988
|
+
options?.signal?.throwIfAborted();
|
|
22989
|
+
await enterRead(core2, indexName);
|
|
22990
|
+
core2.requireIndex(indexName);
|
|
22991
|
+
const scoped = scopedParams(params, options);
|
|
22992
|
+
if (core2.pluginRegistry.hasHooks("beforeSearch")) {
|
|
22993
|
+
await core2.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
22994
|
+
}
|
|
22995
|
+
let embedded2 = null;
|
|
22996
|
+
while (true) {
|
|
22997
|
+
embedded2 = await embeddedFor(core2, indexName, scoped, signal, embedded2);
|
|
22998
|
+
const resolvedParams = embedded2?.params ?? scoped;
|
|
22999
|
+
const loading4 = loadQueryCode(resolvedParams);
|
|
23000
|
+
if (loading4 !== null) await loading4;
|
|
23001
|
+
const result = await preflightUnderHold(core2, indexName, params, resolvedParams, embedded2, options);
|
|
23002
|
+
if (result !== null) return result;
|
|
23003
|
+
}
|
|
23004
|
+
}
|
|
23047
23005
|
async function runEngineSuggest(core2, indexName, params, partitionIds) {
|
|
23048
23006
|
const { executeSuggest } = await importOptionalReads();
|
|
23049
23007
|
await enterRead(core2, indexName);
|
|
@@ -23129,11 +23087,11 @@ function createNarsilFromCore(core2, config) {
|
|
|
23129
23087
|
async function withOpenIndex2(indexName, action) {
|
|
23130
23088
|
guardShutdown();
|
|
23131
23089
|
requireIndex(indexName);
|
|
23132
|
-
const
|
|
23090
|
+
const hold = await holdIndex(core2.indexState, indexName);
|
|
23133
23091
|
try {
|
|
23134
|
-
return await action();
|
|
23092
|
+
return await action(hold);
|
|
23135
23093
|
} finally {
|
|
23136
|
-
release();
|
|
23094
|
+
hold.release();
|
|
23137
23095
|
}
|
|
23138
23096
|
}
|
|
23139
23097
|
const rebalancesStarting = /* @__PURE__ */ new Map();
|
|
@@ -23196,6 +23154,8 @@ function createNarsilFromCore(core2, config) {
|
|
|
23196
23154
|
documentCount: manager?.countDocuments() ?? entry.documentCount,
|
|
23197
23155
|
partitionCount: manager?.partitionCount ?? entry.partitionCount,
|
|
23198
23156
|
estimatedMemoryBytes: manager?.estimateMemoryBytes() ?? 0,
|
|
23157
|
+
savedBytes: entry.storedSizes?.savedBytes,
|
|
23158
|
+
logBytes: entry.storedSizes?.logBytes,
|
|
23199
23159
|
language: entry.language.name,
|
|
23200
23160
|
schema: entry.config.schema,
|
|
23201
23161
|
heldSearchStates: manager?.heldSearchStates ?? 0
|
|
@@ -23203,26 +23163,28 @@ function createNarsilFromCore(core2, config) {
|
|
|
23203
23163
|
},
|
|
23204
23164
|
getPartitionStats(indexName) {
|
|
23205
23165
|
guardShutdown();
|
|
23206
|
-
requireIndex(indexName);
|
|
23207
|
-
|
|
23166
|
+
const entry = requireIndex(indexName);
|
|
23167
|
+
const stats = executor.getManager(indexName)?.getPartitionStats() ?? [];
|
|
23168
|
+
entry.storedSizes?.addTo(stats);
|
|
23169
|
+
return stats;
|
|
23208
23170
|
},
|
|
23209
23171
|
insert(indexName, document, docId, options) {
|
|
23210
|
-
return withCountedWrite(indexName, () => insertDocument(mutationCtx, indexName, document, docId, options));
|
|
23172
|
+
return withCountedWrite(indexName, (hold) => insertDocument(mutationCtx, indexName, document, docId, options, hold));
|
|
23211
23173
|
},
|
|
23212
23174
|
insertBatch(indexName, documents2, options) {
|
|
23213
|
-
return withCountedWrite(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents2, options));
|
|
23175
|
+
return withCountedWrite(indexName, (hold) => insertDocumentBatch(mutationCtx, indexName, documents2, options, hold));
|
|
23214
23176
|
},
|
|
23215
23177
|
remove(indexName, docId, options) {
|
|
23216
|
-
return withCountedWrite(indexName, () => removeDocument2(mutationCtx, indexName, docId, options));
|
|
23178
|
+
return withCountedWrite(indexName, (hold) => removeDocument2(mutationCtx, indexName, docId, options, hold));
|
|
23217
23179
|
},
|
|
23218
23180
|
removeBatch(indexName, docIds, options) {
|
|
23219
|
-
return withCountedWrite(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
|
|
23181
|
+
return withCountedWrite(indexName, (hold) => removeDocumentBatch(mutationCtx, indexName, docIds, options, hold));
|
|
23220
23182
|
},
|
|
23221
23183
|
update(indexName, docId, document, options) {
|
|
23222
|
-
return withCountedWrite(indexName, () => updateDocument(mutationCtx, indexName, docId, document, options));
|
|
23184
|
+
return withCountedWrite(indexName, (hold) => updateDocument(mutationCtx, indexName, docId, document, options, hold));
|
|
23223
23185
|
},
|
|
23224
23186
|
updateBatch(indexName, updates, options) {
|
|
23225
|
-
return withCountedWrite(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
|
|
23187
|
+
return withCountedWrite(indexName, (hold) => updateDocumentBatch(mutationCtx, indexName, updates, options, hold));
|
|
23226
23188
|
},
|
|
23227
23189
|
waitForWrites(indexName) {
|
|
23228
23190
|
guardShutdown();
|
|
@@ -11,12 +11,14 @@ var WAL_ONLY_FIELDS = ["directory", "mode", "flushIntervalMs", "segmentMaxBytes"
|
|
|
11
11
|
function configError(message) {
|
|
12
12
|
return new NarsilError(ErrorCodes.CONFIG_INVALID, message);
|
|
13
13
|
}
|
|
14
|
-
function requireKnob(name, value, minimum) {
|
|
14
|
+
function requireKnob(name, value, minimum, whole = false) {
|
|
15
15
|
if (value === void 0) {
|
|
16
16
|
return;
|
|
17
17
|
}
|
|
18
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < minimum) {
|
|
19
|
-
throw configError(
|
|
18
|
+
if (typeof value !== "number" || !(whole ? Number.isSafeInteger(value) : Number.isFinite(value)) || value < minimum) {
|
|
19
|
+
throw configError(
|
|
20
|
+
`durability.${name} must be ${whole ? "an integer" : "a finite number"} of at least ${minimum}, got ${String(value)}`
|
|
21
|
+
);
|
|
20
22
|
}
|
|
21
23
|
}
|
|
22
24
|
function requireValidDurabilityConfig(durability) {
|
|
@@ -36,6 +38,7 @@ function requireValidDurabilityConfig(durability) {
|
|
|
36
38
|
);
|
|
37
39
|
}
|
|
38
40
|
requireKnob("compactionThreshold", durability.compactionThreshold, 1);
|
|
41
|
+
requireKnob("rewriteBytesPerCheckpoint", durability.rewriteBytesPerCheckpoint, 1, true);
|
|
39
42
|
if (durability.tier === "snapshot") {
|
|
40
43
|
for (const field of WAL_ONLY_FIELDS) {
|
|
41
44
|
if (durability[field] !== void 0) {
|
|
@@ -218,5 +218,47 @@ async function embedBatchDocumentFields(documents, embeddingConfig, adapter, sig
|
|
|
218
218
|
}
|
|
219
219
|
return { embedded, failed };
|
|
220
220
|
}
|
|
221
|
+
function lacksMappedVectors(documents, embeddingConfig) {
|
|
222
|
+
const targets = Object.keys(embeddingConfig.fields);
|
|
223
|
+
for (const document of documents) {
|
|
224
|
+
for (const target of targets) {
|
|
225
|
+
const existing = resolveFieldValue(document, target);
|
|
226
|
+
if (existing === void 0 || existing === null) return true;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return false;
|
|
230
|
+
}
|
|
231
|
+
function clearWriteEmbedding(earlier) {
|
|
232
|
+
for (const [document, fields] of earlier.fields) {
|
|
233
|
+
for (const field of fields) deleteFieldValue(document, field);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
async function embedDocumentsForWrite(documents, embeddingConfig, adapter, signal, together) {
|
|
237
|
+
const fields = /* @__PURE__ */ new Map();
|
|
238
|
+
const failed = /* @__PURE__ */ new Map();
|
|
239
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
240
|
+
if (together && documents.length > 1) {
|
|
241
|
+
try {
|
|
242
|
+
const result = await embedBatchDocumentFields(documents, embeddingConfig, adapter, signal);
|
|
243
|
+
for (const [position, embedded] of result.embedded) fields.set(documents[position], embedded);
|
|
244
|
+
for (const [position, error] of result.failed) documentFailures.set(position, error);
|
|
245
|
+
} catch (err) {
|
|
246
|
+
const error = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
247
|
+
for (let position = 0; position < documents.length; position++) failed.set(position, error);
|
|
248
|
+
}
|
|
249
|
+
} else {
|
|
250
|
+
for (let position = 0; position < documents.length; position++) {
|
|
251
|
+
try {
|
|
252
|
+
fields.set(
|
|
253
|
+
documents[position],
|
|
254
|
+
await embedDocumentFields(documents[position], embeddingConfig, adapter, signal)
|
|
255
|
+
);
|
|
256
|
+
} catch (err) {
|
|
257
|
+
failed.set(position, err);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return { embedding: { adapter, mapping: embeddingConfig.fields, fields }, failed, documentFailures };
|
|
262
|
+
}
|
|
221
263
|
|
|
222
|
-
export { assertDocumentCarriesMappedVectors, bindEmbedHost, embedBatchDocumentFields, embedDocumentFields };
|
|
264
|
+
export { assertDocumentCarriesMappedVectors, bindEmbedHost, clearWriteEmbedding, embedBatchDocumentFields, embedDocumentFields, embedDocumentsForWrite, lacksMappedVectors };
|