@delali/narsil 0.3.1-next.91 → 0.3.1-next.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-4JTBL63P.mjs} +4 -4
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-HC6DMR4T.mjs} +238 -114
- package/dist/browser/{chunk-66KYAOIK.mjs → chunk-PP6N5CCS.mjs} +2 -2
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
- package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-PPTZ565G.mjs} +362 -103
- package/dist/browser/{write-K5V4XQ72.mjs → write-2ZAA7UDH.mjs} +4 -4
- package/dist/{capabilities-CvExBThq.d.ts → capabilities-xsJfdFoQ.d.ts} +2 -2
- package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
- package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
- package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-A4G7JSYO.mjs → chunk-C45FUYPI.mjs} +98 -49
- package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-67Z2V6TC.mjs → chunk-EWBXO45L.mjs} +3 -3
- package/dist/{chunk-3TTYK5A7.mjs → chunk-EX62BQT6.mjs} +9 -9
- package/dist/{chunk-3RDS4UHC.mjs → chunk-EZD4MWOU.mjs} +4 -4
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
- package/dist/{chunk-7OQHSDK6.mjs → chunk-J53IHLG7.mjs} +7 -7
- package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/{chunk-T4DB5YFB.mjs → chunk-QWWCLRXX.mjs} +6 -6
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
- package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/{chunk-XLJ2S2BH.mjs → chunk-UAUZGXJP.mjs} +40 -21
- package/dist/{chunk-EONIHG3V.mjs → chunk-WBS3EYC4.mjs} +7 -7
- package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
- package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
- package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
- package/dist/{client-Cc42b42j.d.ts → client-CgayPw2R.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +1 -0
- package/dist/{config-DyRowvVP.d.ts → config-3kBsQ0Pi.d.ts} +1 -1
- package/dist/{config-BYne-yCI.d.ts → config-B-RnO6dr.d.ts} +1 -1
- package/dist/direct-executor-IJTRU3YN.mjs +22 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +26 -26
- package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-Y55SCPFC.mjs} +518 -91
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-96_kLtSY.d.ts → engine-DSUfY2F0.d.ts} +11 -11
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +18 -18
- package/dist/{memory-dd3GESdc.d.ts → memory-wuXcYOD1.d.ts} +96 -76
- package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-4SG45BLF.mjs +35 -0
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +10 -10
- package/dist/snapshot-XBKE7PUG.mjs +738 -0
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DA1Ff0G4.d.ts → tasks-Bj9WBlFG.d.ts} +1 -1
- package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-OJR3H4BI.mjs → write-BMNV7P2H.mjs} +20 -20
- package/package.json +7 -7
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-CHXVCBN7.mjs +0 -22
- package/dist/snapshot-JFZVOGZV.mjs +0 -479
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, toReducedSortValue, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, resolveRequestThreadCount, STATISTICS_BROADCAST_INTERVAL_MS,
|
|
1
|
+
import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, resolveRequestThreadCount, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, defaultSortMode, holdsUnpairedSurrogate, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, compareComparableValues, readSortField, exceedsCodePoints, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, compareComparableKeys, sortModeOf, toComparableSortValue, DEFAULT_FUZZY_PREFIX_LENGTH, SORT_VALUE_MAX_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, truncateSortString, DEFAULT_MAX_PER_GROUP, compareSortValues, isSortMode, SORT_MODES, MIN_DOCUMENTS_FOR_SEGMENTS, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-HARBSUWP.mjs';
|
|
2
2
|
import { Decoder, encode, decode } from '@msgpack/msgpack';
|
|
3
3
|
|
|
4
4
|
// src/analysis/registry.ts
|
|
@@ -3049,6 +3049,7 @@ function createDocumentStore() {
|
|
|
3049
3049
|
const reverseMap = [];
|
|
3050
3050
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
3051
3051
|
let nextInternalId = 0;
|
|
3052
|
+
let numberingResets = 0;
|
|
3052
3053
|
let sortedIds = null;
|
|
3053
3054
|
let storedContentBytes = 0;
|
|
3054
3055
|
function replaceStored(docId, document, fieldLengths) {
|
|
@@ -3087,6 +3088,7 @@ function createDocumentStore() {
|
|
|
3087
3088
|
forwardMap.clear();
|
|
3088
3089
|
reverseMap.length = 0;
|
|
3089
3090
|
nextInternalId = 0;
|
|
3091
|
+
numberingResets++;
|
|
3090
3092
|
sortedIds = null;
|
|
3091
3093
|
}
|
|
3092
3094
|
const idResolver = {
|
|
@@ -3195,6 +3197,9 @@ function createDocumentStore() {
|
|
|
3195
3197
|
internalIdCapacity() {
|
|
3196
3198
|
return nextInternalId;
|
|
3197
3199
|
},
|
|
3200
|
+
numberingGeneration() {
|
|
3201
|
+
return numberingResets;
|
|
3202
|
+
},
|
|
3198
3203
|
contentBytes() {
|
|
3199
3204
|
return storedContentBytes;
|
|
3200
3205
|
},
|
|
@@ -7696,7 +7701,7 @@ function estimatePartitionBytes(state) {
|
|
|
7696
7701
|
bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7697
7702
|
for (const index of state.patternIndexes.values()) bytes += index.bytes();
|
|
7698
7703
|
bytes += state.partialWords.bytes();
|
|
7699
|
-
return bytes + sortColumnBytes(state);
|
|
7704
|
+
return bytes + sortColumnBytes(state) + (state.vectorOrdinalMaps?.bytes ?? 0);
|
|
7700
7705
|
}
|
|
7701
7706
|
|
|
7702
7707
|
// src/core/partition/merge.ts
|
|
@@ -9192,7 +9197,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9192
9197
|
lastSchemaRef: null,
|
|
9193
9198
|
trackPositions,
|
|
9194
9199
|
sortColumns: null,
|
|
9195
|
-
scoreBuffer: null
|
|
9200
|
+
scoreBuffer: null,
|
|
9201
|
+
vectorOrdinalMaps: null
|
|
9196
9202
|
};
|
|
9197
9203
|
function clearAll() {
|
|
9198
9204
|
state.invertedIdx.clear();
|
|
@@ -9214,6 +9220,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9214
9220
|
state.lastSchemaRef = null;
|
|
9215
9221
|
state.sortColumns = null;
|
|
9216
9222
|
state.scoreBuffer = null;
|
|
9223
|
+
state.vectorOrdinalMaps = null;
|
|
9217
9224
|
}
|
|
9218
9225
|
const partition = {
|
|
9219
9226
|
get partitionId() {
|
|
@@ -9368,6 +9375,9 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9368
9375
|
estimateMemoryBytes() {
|
|
9369
9376
|
return estimatePartitionBytes(state);
|
|
9370
9377
|
},
|
|
9378
|
+
readStates() {
|
|
9379
|
+
return [state];
|
|
9380
|
+
},
|
|
9371
9381
|
searchFulltext(params) {
|
|
9372
9382
|
return searchFulltext(state, params);
|
|
9373
9383
|
},
|
|
@@ -9859,7 +9869,7 @@ function frozenSegmentBytes(segment) {
|
|
|
9859
9869
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
9860
9870
|
}
|
|
9861
9871
|
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
9862
|
-
return bytes + segment.partialWords.bytes() + segment.removalBytes();
|
|
9872
|
+
return bytes + segment.partialWords.bytes() + segment.removalBytes() + (segment.vectorOrdinalMaps?.bytes ?? 0);
|
|
9863
9873
|
}
|
|
9864
9874
|
|
|
9865
9875
|
// src/core/partition/composite/compaction.ts
|
|
@@ -9932,6 +9942,24 @@ function compositeFieldValues(frozen, live, docId, fieldPaths) {
|
|
|
9932
9942
|
}
|
|
9933
9943
|
return live.fieldValues(docId, fieldPaths);
|
|
9934
9944
|
}
|
|
9945
|
+
function* compositeDocIds(frozen, live) {
|
|
9946
|
+
for (const segment of frozen) {
|
|
9947
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
9948
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
9949
|
+
if (docId !== void 0) yield docId;
|
|
9950
|
+
}
|
|
9951
|
+
}
|
|
9952
|
+
yield* live.docIds();
|
|
9953
|
+
}
|
|
9954
|
+
function compositeSortedDocIds(frozen, live) {
|
|
9955
|
+
const merged = [];
|
|
9956
|
+
for (const segment of frozen) {
|
|
9957
|
+
merged.push(...segment.docStore.sortedDocIds());
|
|
9958
|
+
}
|
|
9959
|
+
merged.push(...live.sortedDocIds());
|
|
9960
|
+
merged.sort(compareCodePoints);
|
|
9961
|
+
return merged;
|
|
9962
|
+
}
|
|
9935
9963
|
|
|
9936
9964
|
// src/core/partition/composite/filters.ts
|
|
9937
9965
|
var WORD_BITS = 32;
|
|
@@ -10607,23 +10635,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10607
10635
|
}
|
|
10608
10636
|
return total;
|
|
10609
10637
|
},
|
|
10610
|
-
|
|
10611
|
-
|
|
10612
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
10613
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
10614
|
-
if (docId !== void 0) yield docId;
|
|
10615
|
-
}
|
|
10616
|
-
}
|
|
10617
|
-
yield* live.docIds();
|
|
10638
|
+
docIds() {
|
|
10639
|
+
return compositeDocIds(frozen, live);
|
|
10618
10640
|
},
|
|
10619
10641
|
sortedDocIds() {
|
|
10620
|
-
|
|
10621
|
-
for (const segment of frozen) {
|
|
10622
|
-
merged.push(...segment.docStore.sortedDocIds());
|
|
10623
|
-
}
|
|
10624
|
-
merged.push(...live.sortedDocIds());
|
|
10625
|
-
merged.sort(compareCodePoints);
|
|
10626
|
-
return merged;
|
|
10642
|
+
return compositeSortedDocIds(frozen, live);
|
|
10627
10643
|
},
|
|
10628
10644
|
releaseSortedDocIds() {
|
|
10629
10645
|
for (const segment of frozen) {
|
|
@@ -10637,6 +10653,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10637
10653
|
total += frozenSegmentBytes(segment);
|
|
10638
10654
|
}
|
|
10639
10655
|
return total;
|
|
10656
|
+
},
|
|
10657
|
+
readStates() {
|
|
10658
|
+
return subs();
|
|
10640
10659
|
}
|
|
10641
10660
|
};
|
|
10642
10661
|
}
|
|
@@ -12199,6 +12218,9 @@ function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
|
12199
12218
|
internalIdCapacity() {
|
|
12200
12219
|
return capacity;
|
|
12201
12220
|
},
|
|
12221
|
+
numberingGeneration() {
|
|
12222
|
+
return 0;
|
|
12223
|
+
},
|
|
12202
12224
|
contentBytes() {
|
|
12203
12225
|
return documents.byteLength;
|
|
12204
12226
|
},
|
|
@@ -13009,6 +13031,11 @@ function createFrozenInvertedReader(table, views, counts) {
|
|
|
13009
13031
|
};
|
|
13010
13032
|
}
|
|
13011
13033
|
|
|
13034
|
+
// src/runtime/process-limit.browser.ts
|
|
13035
|
+
function readResidentMemory() {
|
|
13036
|
+
return null;
|
|
13037
|
+
}
|
|
13038
|
+
|
|
13012
13039
|
// src/core/partition/frozen/live-counts.ts
|
|
13013
13040
|
var UINT32_BYTES = 4;
|
|
13014
13041
|
var FLOAT64_BYTES = 8;
|
|
@@ -13820,6 +13847,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
13820
13847
|
lastSchemaRef: null,
|
|
13821
13848
|
sortColumns: null,
|
|
13822
13849
|
scoreBuffer: null,
|
|
13850
|
+
vectorOrdinalMaps: null,
|
|
13823
13851
|
liveDocumentCount() {
|
|
13824
13852
|
return source.documentCount - tombstones.size;
|
|
13825
13853
|
},
|
|
@@ -14855,7 +14883,7 @@ function requireLoadedVectorCode() {
|
|
|
14855
14883
|
function loadVectorCode() {
|
|
14856
14884
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
14857
14885
|
if (loading === null) {
|
|
14858
|
-
loading = import('./module-
|
|
14886
|
+
loading = import('./module-O5WH32EL.mjs').then(
|
|
14859
14887
|
(code) => {
|
|
14860
14888
|
loaded = code;
|
|
14861
14889
|
return code;
|
|
@@ -14950,7 +14978,7 @@ function heldVectorOf(held, fieldName, docId) {
|
|
|
14950
14978
|
return held.fields.get(fieldName)?.view.vectorOf(docId);
|
|
14951
14979
|
}
|
|
14952
14980
|
|
|
14953
|
-
// src/workers/direct-executor.ts
|
|
14981
|
+
// src/workers/direct-executor/index.ts
|
|
14954
14982
|
var NO_VECTOR_WORKER_COPIES = { enabled: false };
|
|
14955
14983
|
function createDirectExecutor(options) {
|
|
14956
14984
|
const indexes = /* @__PURE__ */ new Map();
|
|
@@ -14966,7 +14994,7 @@ function createDirectExecutor(options) {
|
|
|
14966
14994
|
}
|
|
14967
14995
|
return entry;
|
|
14968
14996
|
}
|
|
14969
|
-
function vectorIndexesFor(indexName, config) {
|
|
14997
|
+
function vectorIndexesFor(indexName, config, copies = vectorWorkerCopies) {
|
|
14970
14998
|
const vectorIndexes = /* @__PURE__ */ new Map();
|
|
14971
14999
|
const vectorFields = extractVectorFieldsFromSchema(config.schema);
|
|
14972
15000
|
if (vectorFields.size === 0) return vectorIndexes;
|
|
@@ -14978,7 +15006,7 @@ function createDirectExecutor(options) {
|
|
|
14978
15006
|
fieldPath,
|
|
14979
15007
|
dim,
|
|
14980
15008
|
config.vectorPromotion,
|
|
14981
|
-
|
|
15009
|
+
copies,
|
|
14982
15010
|
indexName,
|
|
14983
15011
|
config.vectorPromotion?.storage ?? vectorStorage
|
|
14984
15012
|
)
|
|
@@ -14998,11 +15026,16 @@ function createDirectExecutor(options) {
|
|
|
14998
15026
|
indexName
|
|
14999
15027
|
});
|
|
15000
15028
|
}
|
|
15029
|
+
const entry = buildEntry(indexName, config, language, analysisStale, awaitedCopyLoad, vectorWorkerCopies);
|
|
15030
|
+
indexes.set(indexName, entry);
|
|
15031
|
+
return entry.manager;
|
|
15032
|
+
}
|
|
15033
|
+
function buildEntry(indexName, config, language, analysisStale, awaitedCopyLoad, copies) {
|
|
15001
15034
|
const router = createPartitionRouter();
|
|
15002
15035
|
const partitionCount = config.partitions?.maxPartitions ?? 1;
|
|
15003
|
-
const vectorIndexes = vectorIndexesFor(indexName, config);
|
|
15036
|
+
const vectorIndexes = vectorIndexesFor(indexName, config, copies);
|
|
15004
15037
|
const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
|
|
15005
|
-
|
|
15038
|
+
return {
|
|
15006
15039
|
manager,
|
|
15007
15040
|
config,
|
|
15008
15041
|
language,
|
|
@@ -15018,11 +15051,9 @@ function createDirectExecutor(options) {
|
|
|
15018
15051
|
vectorCopies: createHeldVectorCopies(),
|
|
15019
15052
|
analysisStale,
|
|
15020
15053
|
awaitedCopyLoad
|
|
15021
|
-
}
|
|
15022
|
-
return manager;
|
|
15054
|
+
};
|
|
15023
15055
|
}
|
|
15024
|
-
function
|
|
15025
|
-
const entry = requireIndex(indexName);
|
|
15056
|
+
function disposeEntry(entry) {
|
|
15026
15057
|
for (const vectorIndex of entry.vectorIndexes.values()) {
|
|
15027
15058
|
vectorIndex.dispose();
|
|
15028
15059
|
}
|
|
@@ -15030,8 +15061,16 @@ function createDirectExecutor(options) {
|
|
|
15030
15061
|
for (const partition of entry.manager.getAllPartitions()) {
|
|
15031
15062
|
partition.clear();
|
|
15032
15063
|
}
|
|
15064
|
+
}
|
|
15065
|
+
function dropIndex(indexName) {
|
|
15066
|
+
disposeEntry(requireIndex(indexName));
|
|
15033
15067
|
indexes.delete(indexName);
|
|
15034
15068
|
}
|
|
15069
|
+
function installDetachedIndex(detached) {
|
|
15070
|
+
const previous = indexes.get(detached.indexName);
|
|
15071
|
+
indexes.set(detached.indexName, detached.entry);
|
|
15072
|
+
if (previous !== void 0 && previous !== detached.entry) disposeEntry(previous);
|
|
15073
|
+
}
|
|
15035
15074
|
function listIndexes() {
|
|
15036
15075
|
return Array.from(indexes.keys());
|
|
15037
15076
|
}
|
|
@@ -15219,7 +15258,11 @@ function createDirectExecutor(options) {
|
|
|
15219
15258
|
},
|
|
15220
15259
|
createIndex,
|
|
15221
15260
|
dropIndex,
|
|
15222
|
-
listIndexes
|
|
15261
|
+
listIndexes,
|
|
15262
|
+
vectorWorkerCopies,
|
|
15263
|
+
createDetachedEntry: (indexName, config, language, copies) => buildEntry(indexName, config, language, false, null, copies),
|
|
15264
|
+
installDetachedIndex,
|
|
15265
|
+
discardDetachedIndex: (detached) => disposeEntry(detached.entry)
|
|
15223
15266
|
};
|
|
15224
15267
|
}
|
|
15225
15268
|
|
|
@@ -15918,6 +15961,7 @@ function createLifecycleEntry(state) {
|
|
|
15918
15961
|
waiterCount: 0,
|
|
15919
15962
|
closePromise: null,
|
|
15920
15963
|
dropPromise: null,
|
|
15964
|
+
replacePromise: null,
|
|
15921
15965
|
drainResolvers: /* @__PURE__ */ new Set()
|
|
15922
15966
|
};
|
|
15923
15967
|
}
|
|
@@ -15957,7 +16001,7 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
15957
16001
|
return entry;
|
|
15958
16002
|
}
|
|
15959
16003
|
function publicState(entry) {
|
|
15960
|
-
return entry.state === "closing" || entry.state === "dropping" ? "open" : entry.state;
|
|
16004
|
+
return entry.state === "closing" || entry.state === "dropping" || entry.state === "replacing" ? "open" : entry.state;
|
|
15961
16005
|
}
|
|
15962
16006
|
async function waitForDrain(entry) {
|
|
15963
16007
|
if (entry.activeOperations === 0) return;
|
|
@@ -15997,6 +16041,11 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
15997
16041
|
entry = entryOf(indexName);
|
|
15998
16042
|
continue;
|
|
15999
16043
|
}
|
|
16044
|
+
if (entry.state === "replacing" && entry.replacePromise !== null) {
|
|
16045
|
+
await entry.replacePromise.catch(() => void 0);
|
|
16046
|
+
entry = entryOf(indexName);
|
|
16047
|
+
continue;
|
|
16048
|
+
}
|
|
16000
16049
|
if (entry.state === "closing" && entry.closePromise !== null) {
|
|
16001
16050
|
await waitForTransition(
|
|
16002
16051
|
indexName,
|
|
@@ -16012,6 +16061,7 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
16012
16061
|
function transitionPending(indexName, entry) {
|
|
16013
16062
|
if (entries.get(indexName) !== entry) return true;
|
|
16014
16063
|
if (entry.state === "dropping" && entry.dropPromise !== null) return true;
|
|
16064
|
+
if (entry.state === "replacing" && entry.replacePromise !== null) return true;
|
|
16015
16065
|
return entry.state === "closing" && entry.closePromise !== null;
|
|
16016
16066
|
}
|
|
16017
16067
|
async function reopen(indexName, explicit) {
|
|
@@ -16059,9 +16109,10 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
16059
16109
|
entry = entryOf(indexName);
|
|
16060
16110
|
continue;
|
|
16061
16111
|
}
|
|
16062
|
-
|
|
16112
|
+
const settling = entry.reopenPromise ?? entry.replacePromise;
|
|
16113
|
+
if (settling !== null) {
|
|
16063
16114
|
if (automatic) return false;
|
|
16064
|
-
await
|
|
16115
|
+
await settling.catch(() => void 0);
|
|
16065
16116
|
entry = entryOf(indexName);
|
|
16066
16117
|
continue;
|
|
16067
16118
|
}
|
|
@@ -16103,13 +16154,9 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
16103
16154
|
await entry.dropPromise;
|
|
16104
16155
|
return;
|
|
16105
16156
|
}
|
|
16106
|
-
|
|
16107
|
-
|
|
16108
|
-
|
|
16109
|
-
continue;
|
|
16110
|
-
}
|
|
16111
|
-
if (entry.closePromise !== null) {
|
|
16112
|
-
await entry.closePromise.catch(() => void 0);
|
|
16157
|
+
const pending = entry.replacePromise ?? entry.reopenPromise ?? entry.closePromise;
|
|
16158
|
+
if (pending !== null) {
|
|
16159
|
+
await pending.catch(() => void 0);
|
|
16113
16160
|
entry = entryOf(indexName);
|
|
16114
16161
|
continue;
|
|
16115
16162
|
}
|
|
@@ -16130,6 +16177,11 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
16130
16177
|
entry.dropPromise = run;
|
|
16131
16178
|
await run;
|
|
16132
16179
|
}
|
|
16180
|
+
const replaceContext = {
|
|
16181
|
+
entryOf,
|
|
16182
|
+
waitForDrain,
|
|
16183
|
+
forget: (indexName) => entries.delete(indexName)
|
|
16184
|
+
};
|
|
16133
16185
|
function exceedsLimits() {
|
|
16134
16186
|
let open = 0;
|
|
16135
16187
|
let bytes = 0;
|
|
@@ -16223,6 +16275,25 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
16223
16275
|
drop(indexName, action) {
|
|
16224
16276
|
return dropEntry(indexName, action);
|
|
16225
16277
|
},
|
|
16278
|
+
async replace(indexName, action) {
|
|
16279
|
+
const { replaceExclusively } = await import('./replace-RURXTGVG.mjs');
|
|
16280
|
+
await replaceExclusively(replaceContext, indexName, action);
|
|
16281
|
+
},
|
|
16282
|
+
async registerReplacing(indexName, action) {
|
|
16283
|
+
const entry = createLifecycleEntry("open");
|
|
16284
|
+
entries.set(indexName, entry);
|
|
16285
|
+
entry.state = "replacing";
|
|
16286
|
+
const run = import('./replace-RURXTGVG.mjs').then(
|
|
16287
|
+
({ replaceUnregistered }) => replaceUnregistered(replaceContext, indexName, entry, action),
|
|
16288
|
+
(error) => {
|
|
16289
|
+
if (entries.get(indexName) === entry) entries.delete(indexName);
|
|
16290
|
+
throw error;
|
|
16291
|
+
}
|
|
16292
|
+
);
|
|
16293
|
+
entry.replacePromise = run;
|
|
16294
|
+
await run;
|
|
16295
|
+
await enforceLimits(indexName);
|
|
16296
|
+
},
|
|
16226
16297
|
stateOf(indexName) {
|
|
16227
16298
|
return publicState(entryOf(indexName));
|
|
16228
16299
|
},
|
|
@@ -16234,7 +16305,7 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
16234
16305
|
let closed = 0;
|
|
16235
16306
|
let reopens = 0;
|
|
16236
16307
|
for (const entry of entries.values()) {
|
|
16237
|
-
if (entry.state === "open" || entry.state === "closing") open += 1;
|
|
16308
|
+
if (entry.state === "open" || entry.state === "closing" || entry.state === "replacing") open += 1;
|
|
16238
16309
|
else closed += 1;
|
|
16239
16310
|
reopens += entry.reopenCount;
|
|
16240
16311
|
}
|
|
@@ -18647,6 +18718,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
18647
18718
|
reportedIneligible: /* @__PURE__ */ new Set(),
|
|
18648
18719
|
scaledOutIndexes: /* @__PURE__ */ new Set(),
|
|
18649
18720
|
desyncedIndexes: /* @__PURE__ */ new Set(),
|
|
18721
|
+
copyReleaseHolds: /* @__PURE__ */ new Map(),
|
|
18650
18722
|
copyLoadBuffers: /* @__PURE__ */ new Map(),
|
|
18651
18723
|
copyTransitions: /* @__PURE__ */ new Map(),
|
|
18652
18724
|
droppedCopies: /* @__PURE__ */ new Map(),
|
|
@@ -18681,10 +18753,21 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
18681
18753
|
for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
|
|
18682
18754
|
await pending;
|
|
18683
18755
|
}
|
|
18756
|
+
const hold = state.copyReleaseHolds.get(indexName);
|
|
18757
|
+
if (hold !== void 0) {
|
|
18758
|
+
hold.reloads ||= wasScaledOut;
|
|
18759
|
+
return;
|
|
18760
|
+
}
|
|
18684
18761
|
state.desyncedIndexes.delete(indexName);
|
|
18685
18762
|
if (!wasScaledOut) return;
|
|
18686
18763
|
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
18687
18764
|
}
|
|
18765
|
+
const copyReleaseSteps = {
|
|
18766
|
+
awaitReplicationIdle,
|
|
18767
|
+
cancelIdleMerge,
|
|
18768
|
+
heldNowhere: (error) => error instanceof NarsilError && error.code === ErrorCodes.INDEX_NOT_FOUND,
|
|
18769
|
+
resyncIndex
|
|
18770
|
+
};
|
|
18688
18771
|
async function openIndex(indexName) {
|
|
18689
18772
|
state.droppedCopies.delete(indexName);
|
|
18690
18773
|
if (!indexReadyForCopies(state, indexName)) return;
|
|
@@ -18794,7 +18877,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
18794
18877
|
pool.removeIndex(indexName);
|
|
18795
18878
|
}
|
|
18796
18879
|
state.droppedCopies.delete(indexName);
|
|
18797
|
-
state.desyncedIndexes.delete(indexName);
|
|
18880
|
+
if (!state.copyReleaseHolds.has(indexName)) state.desyncedIndexes.delete(indexName);
|
|
18798
18881
|
state.lastAccessAt.delete(indexName);
|
|
18799
18882
|
state.replicationQueues.delete(indexName);
|
|
18800
18883
|
state.segmentLedger.delete(indexName);
|
|
@@ -18840,6 +18923,18 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
18840
18923
|
if (wasScaledOut) withdrawCopies(indexName);
|
|
18841
18924
|
return wasScaledOut || loading2 !== void 0 && loading2.kind !== "drop";
|
|
18842
18925
|
},
|
|
18926
|
+
async releaseIndexCopies(indexName) {
|
|
18927
|
+
const hold = state.copyReleaseHolds.get(indexName);
|
|
18928
|
+
if (hold === void 0) state.copyReleaseHolds.set(indexName, { holders: 1, reloads: false });
|
|
18929
|
+
else hold.holders += 1;
|
|
18930
|
+
state.desyncedIndexes.add(indexName);
|
|
18931
|
+
const { releaseIndexCopies } = await import('./release-76MXRPGV.mjs');
|
|
18932
|
+
return releaseIndexCopies(state, indexName, copyReleaseSteps);
|
|
18933
|
+
},
|
|
18934
|
+
async endCopyRelease(indexName, reloads) {
|
|
18935
|
+
const { endCopyRelease } = await import('./release-76MXRPGV.mjs');
|
|
18936
|
+
await endCopyRelease(state, indexName, reloads, copyReleaseSteps);
|
|
18937
|
+
},
|
|
18843
18938
|
resyncIndex,
|
|
18844
18939
|
noteAccess: (indexName) => noteAccess(state, indexName),
|
|
18845
18940
|
workerCopies,
|
|
@@ -18885,7 +18980,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
18885
18980
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
18886
18981
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
18887
18982
|
}
|
|
18888
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
18983
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-7DKYHKT5.mjs');
|
|
18889
18984
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
18890
18985
|
const executor = createDirectExecutor({
|
|
18891
18986
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19603,6 +19698,9 @@ async function runLifecycleHook(core, hookName, context) {
|
|
|
19603
19698
|
async function announceIndexCreated(core, name, config) {
|
|
19604
19699
|
await runLifecycleHook(core, "onIndexCreate", { indexName: name, config });
|
|
19605
19700
|
}
|
|
19701
|
+
async function announceIndexDropped(core, name, config) {
|
|
19702
|
+
await runLifecycleHook(core, "onIndexDrop", { indexName: name, config });
|
|
19703
|
+
}
|
|
19606
19704
|
async function createEngineIndex(core, config, name, requestedConfig, creation = {}) {
|
|
19607
19705
|
const { indexUuid, adapterBinding = "required", appliesCreationLimits = false } = creation;
|
|
19608
19706
|
const loadingVectorCode = loadVectorCodeFor(requestedConfig.schema);
|
|
@@ -22282,6 +22380,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
22282
22380
|
release();
|
|
22283
22381
|
}
|
|
22284
22382
|
}
|
|
22383
|
+
const restoresInFlight = /* @__PURE__ */ new Map();
|
|
22384
|
+
const restoreHost = {
|
|
22385
|
+
getVectorFieldPaths,
|
|
22386
|
+
announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
|
|
22387
|
+
announceIndexDropped: (name, replacedConfig) => announceIndexDropped(core, name, replacedConfig),
|
|
22388
|
+
defaultEmbeddingAdapter: config?.embedding ?? null,
|
|
22389
|
+
releaseTimeoutMs: DEFAULT_REQUEST_TIMEOUT_MS
|
|
22390
|
+
};
|
|
22285
22391
|
function snapshotDepsOf(indexName) {
|
|
22286
22392
|
return {
|
|
22287
22393
|
indexName,
|
|
@@ -22290,41 +22396,27 @@ function portabilityOperations(core, config, engine) {
|
|
|
22290
22396
|
durability: core.durability
|
|
22291
22397
|
};
|
|
22292
22398
|
}
|
|
22293
|
-
async function restoreIndex(indexName, data) {
|
|
22294
|
-
const { restoreFromSnapshot } = await import('./snapshot-5EJ7I7QY.mjs');
|
|
22295
|
-
await restoreFromSnapshot(indexName, data, {
|
|
22296
|
-
executor: core.executor,
|
|
22297
|
-
indexRegistry: core.indexRegistry,
|
|
22298
|
-
getVectorFieldPaths,
|
|
22299
|
-
dropIndex: (name) => dropEngineIndex(core, name),
|
|
22300
|
-
announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
|
|
22301
|
-
requireManager: core.requireManager,
|
|
22302
|
-
durability: core.durability,
|
|
22303
|
-
filesystemDurability: core.filesystemDurability,
|
|
22304
|
-
embeddingAdapters: core.embeddingAdapters,
|
|
22305
|
-
defaultEmbeddingAdapter: config?.embedding ?? null,
|
|
22306
|
-
markAnalysisStale: core.analysisRebuild.markStale,
|
|
22307
|
-
clearAnalysisStale: core.analysisRebuild.clearStale,
|
|
22308
|
-
indexState: core.indexState
|
|
22309
|
-
});
|
|
22310
|
-
}
|
|
22311
22399
|
return {
|
|
22312
22400
|
snapshot(indexName) {
|
|
22313
22401
|
return withOpenIndex(indexName, async () => {
|
|
22314
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22402
|
+
const { createSnapshot } = await import('./snapshot-PPTZ565G.mjs');
|
|
22315
22403
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22316
22404
|
});
|
|
22317
22405
|
},
|
|
22318
22406
|
async restore(indexName, data) {
|
|
22319
22407
|
core.guardShutdown();
|
|
22320
|
-
|
|
22408
|
+
validateIndexName(indexName);
|
|
22409
|
+
const restored = import('./restore-queue-Z7FZZNKH.mjs').then(
|
|
22410
|
+
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
22411
|
+
);
|
|
22412
|
+
await emitWriteEventAfter(core, indexName, restored);
|
|
22321
22413
|
core.heapPressureNotifier.check(indexName);
|
|
22322
22414
|
},
|
|
22323
22415
|
exportDocuments(indexName) {
|
|
22324
22416
|
return withOpenIndex(indexName, async () => {
|
|
22325
22417
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22326
|
-
import('./snapshot-
|
|
22327
|
-
import('./write-
|
|
22418
|
+
import('./snapshot-PPTZ565G.mjs'),
|
|
22419
|
+
import('./write-2ZAA7UDH.mjs')
|
|
22328
22420
|
]);
|
|
22329
22421
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22330
22422
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -22647,17 +22739,6 @@ function searchOptionsFor(manager, patternWork) {
|
|
|
22647
22739
|
patternWork
|
|
22648
22740
|
};
|
|
22649
22741
|
}
|
|
22650
|
-
function collectFilterDocIds(manager, params, schema, partitionIds, patternWork) {
|
|
22651
|
-
const filterDocIds = /* @__PURE__ */ new Set();
|
|
22652
|
-
if (!params.filters) return filterDocIds;
|
|
22653
|
-
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
22654
|
-
const partitionFiltered = partition.applyFilters(params.filters, schema, patternWork);
|
|
22655
|
-
for (const docId of partitionFiltered) {
|
|
22656
|
-
filterDocIds.add(docId);
|
|
22657
|
-
}
|
|
22658
|
-
}
|
|
22659
|
-
return filterDocIds;
|
|
22660
|
-
}
|
|
22661
22742
|
function vectorResultsToScored(results) {
|
|
22662
22743
|
return results.map((r) => ({
|
|
22663
22744
|
docId: r.docId,
|
|
@@ -23366,6 +23447,47 @@ function minMaxNormalize(docs) {
|
|
|
23366
23447
|
return docs.map((d) => ({ docId: d.docId, score: (d.score - min) / range }));
|
|
23367
23448
|
}
|
|
23368
23449
|
|
|
23450
|
+
// src/engine/query/filter-subsets.ts
|
|
23451
|
+
function collectFilterSubsets(manager, filters, schema, partitionIds, patternWork) {
|
|
23452
|
+
const subsets = [];
|
|
23453
|
+
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
23454
|
+
for (const state of partition.readStates()) {
|
|
23455
|
+
subsets.push({
|
|
23456
|
+
owner: state,
|
|
23457
|
+
bits: applyPartitionFiltersBitset(state, filters, schema, patternWork),
|
|
23458
|
+
numberingGeneration: state.docStore.numberingGeneration()
|
|
23459
|
+
});
|
|
23460
|
+
}
|
|
23461
|
+
}
|
|
23462
|
+
return subsets;
|
|
23463
|
+
}
|
|
23464
|
+
function filterSubsetsMatchNothing(subsets) {
|
|
23465
|
+
for (const subset of subsets) {
|
|
23466
|
+
for (let wordIndex = 0; wordIndex < subset.bits.length; wordIndex++) {
|
|
23467
|
+
if (subset.bits[wordIndex] !== 0) return false;
|
|
23468
|
+
}
|
|
23469
|
+
}
|
|
23470
|
+
return true;
|
|
23471
|
+
}
|
|
23472
|
+
function filterSubsetsRenumbered(subsets) {
|
|
23473
|
+
return subsets.some((subset) => subset.owner.docStore.numberingGeneration() !== subset.numberingGeneration);
|
|
23474
|
+
}
|
|
23475
|
+
async function searchCurrentSubsets(fieldName, first, collectAgain, search) {
|
|
23476
|
+
let subsets = first;
|
|
23477
|
+
for (let attempt = 1; ; attempt++) {
|
|
23478
|
+
const answer = await search(subsets);
|
|
23479
|
+
if (!filterSubsetsRenumbered(subsets)) return answer;
|
|
23480
|
+
if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
|
|
23481
|
+
throw new NarsilError(
|
|
23482
|
+
ErrorCodes.WORKER_BUSY,
|
|
23483
|
+
`The index renumbered its documents during each of ${attempt} filtered searches of vector field "${fieldName}"`,
|
|
23484
|
+
{ fieldName, attempts: attempt }
|
|
23485
|
+
);
|
|
23486
|
+
}
|
|
23487
|
+
subsets = collectAgain();
|
|
23488
|
+
}
|
|
23489
|
+
}
|
|
23490
|
+
|
|
23369
23491
|
// src/engine/query/vector.ts
|
|
23370
23492
|
function oversampleOf(vectorConfig) {
|
|
23371
23493
|
const oversample = vectorConfig.oversample;
|
|
@@ -23390,8 +23512,36 @@ function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
|
23390
23512
|
}
|
|
23391
23513
|
return Math.min(reach + 1, RESULT_WINDOW);
|
|
23392
23514
|
}
|
|
23515
|
+
function filteredSearchOf(params, context) {
|
|
23516
|
+
const filters = params.filters;
|
|
23517
|
+
if (!filters) return void 0;
|
|
23518
|
+
const { manager, config, partitionIds, patternWork } = context;
|
|
23519
|
+
const collectAgain = () => collectFilterSubsets(manager, filters, config.schema, partitionIds, patternWork);
|
|
23520
|
+
return { first: collectAgain(), collectAgain };
|
|
23521
|
+
}
|
|
23522
|
+
function searchField(vecIndex, vectorConfig, queryVector, k, context, filtered) {
|
|
23523
|
+
const options = {
|
|
23524
|
+
metric: vectorConfig.metric ?? "cosine",
|
|
23525
|
+
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
23526
|
+
efSearch: vectorConfig.efSearch,
|
|
23527
|
+
oversample: oversampleOf(vectorConfig)
|
|
23528
|
+
};
|
|
23529
|
+
if (filtered !== void 0) {
|
|
23530
|
+
return searchCurrentSubsets(
|
|
23531
|
+
vecIndex.fieldName,
|
|
23532
|
+
filtered.first,
|
|
23533
|
+
filtered.collectAgain,
|
|
23534
|
+
(filterSubsets) => vecIndex.searchParallel(queryVector, k, { ...options, filterSubsets })
|
|
23535
|
+
);
|
|
23536
|
+
}
|
|
23537
|
+
const filterPartitions = partitionsForVectorSearch(context.manager, vecIndex, context.partitionIds);
|
|
23538
|
+
return vecIndex.searchParallel(
|
|
23539
|
+
queryVector,
|
|
23540
|
+
k,
|
|
23541
|
+
filterPartitions === void 0 ? options : { ...options, filterPartitions }
|
|
23542
|
+
);
|
|
23543
|
+
}
|
|
23393
23544
|
async function executeVectorSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
23394
|
-
const { manager, config, partitionIds } = context;
|
|
23395
23545
|
const vectorConfig = params.vector;
|
|
23396
23546
|
if (!vectorConfig || !vectorConfig.value) {
|
|
23397
23547
|
return { scored: [], totalMatched: 0 };
|
|
@@ -23400,26 +23550,13 @@ async function executeVectorSearch(params, context, limit, offset, cursorDepth =
|
|
|
23400
23550
|
if (!vecIndex) {
|
|
23401
23551
|
return { scored: [], totalMatched: 0 };
|
|
23402
23552
|
}
|
|
23403
|
-
|
|
23404
|
-
|
|
23405
|
-
|
|
23406
|
-
filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds, context.patternWork);
|
|
23407
|
-
if (filterDocIds.size === 0) {
|
|
23408
|
-
return { scored: [], totalMatched: 0 };
|
|
23409
|
-
}
|
|
23410
|
-
} else {
|
|
23411
|
-
filterPartitions = partitionsForVectorSearch(manager, vecIndex, partitionIds);
|
|
23553
|
+
const filtered = filteredSearchOf(params, context);
|
|
23554
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
23555
|
+
return { scored: [], totalMatched: 0 };
|
|
23412
23556
|
}
|
|
23413
23557
|
const queryVec = new Float32Array(vectorConfig.value);
|
|
23414
23558
|
const k = vectorFetchDepth(limit, offset, cursorDepth);
|
|
23415
|
-
const outcome = await vecIndex
|
|
23416
|
-
metric: vectorConfig.metric ?? "cosine",
|
|
23417
|
-
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
23418
|
-
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
23419
|
-
...filterPartitions !== void 0 ? { filterPartitions } : {},
|
|
23420
|
-
efSearch: vectorConfig.efSearch,
|
|
23421
|
-
oversample: oversampleOf(vectorConfig)
|
|
23422
|
-
});
|
|
23559
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVec, k, context, filtered);
|
|
23423
23560
|
const scored = vectorResultsToScored(outcome.results);
|
|
23424
23561
|
return { scored, totalMatched: outcome.matched, matchedExact: outcome.matchedExact };
|
|
23425
23562
|
}
|
|
@@ -23445,34 +23582,21 @@ async function textLeg(textOnlyParams, context) {
|
|
|
23445
23582
|
searchOptionsFor(manager, context.patternWork)
|
|
23446
23583
|
);
|
|
23447
23584
|
}
|
|
23448
|
-
async function vectorLeg(vectorConfig, queryVector, context,
|
|
23449
|
-
const { manager } = context;
|
|
23585
|
+
async function vectorLeg(vectorConfig, queryVector, context, filtered, k) {
|
|
23450
23586
|
const vecIndex = resolveVectorIndex(context, vectorConfig.field);
|
|
23451
23587
|
if (!vecIndex) return { scored: [], holdsEveryMatch: true };
|
|
23452
|
-
const
|
|
23453
|
-
const outcome = await vecIndex.searchParallel(queryVector, k, {
|
|
23454
|
-
metric: vectorConfig.metric ?? "cosine",
|
|
23455
|
-
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
23456
|
-
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
23457
|
-
...filterPartitions !== void 0 ? { filterPartitions } : {},
|
|
23458
|
-
efSearch: vectorConfig.efSearch,
|
|
23459
|
-
oversample: oversampleOf(vectorConfig)
|
|
23460
|
-
});
|
|
23588
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVector, k, context, filtered);
|
|
23461
23589
|
return {
|
|
23462
23590
|
scored: vectorResultsToScored(outcome.results),
|
|
23463
23591
|
holdsEveryMatch: outcome.matchedExact && outcome.results.length === outcome.matched
|
|
23464
23592
|
};
|
|
23465
23593
|
}
|
|
23466
23594
|
async function executeHybridSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
23467
|
-
const { manager, config } = context;
|
|
23468
23595
|
const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
|
|
23469
23596
|
const fusion = resolveHybridFusion(params.hybrid);
|
|
23470
|
-
|
|
23471
|
-
if (
|
|
23472
|
-
|
|
23473
|
-
if (filterDocIds.size === 0) {
|
|
23474
|
-
return { scored: [], totalMatched: 0 };
|
|
23475
|
-
}
|
|
23597
|
+
const filtered = filteredSearchOf(params, context);
|
|
23598
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
23599
|
+
return { scored: [], totalMatched: 0 };
|
|
23476
23600
|
}
|
|
23477
23601
|
if (!vectorConfig || !vectorConfig.value) {
|
|
23478
23602
|
const textOnly = await textLeg(textOnlyParams, context);
|
|
@@ -23482,7 +23606,7 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
|
|
|
23482
23606
|
vectorConfig,
|
|
23483
23607
|
new Float32Array(vectorConfig.value),
|
|
23484
23608
|
context,
|
|
23485
|
-
|
|
23609
|
+
filtered,
|
|
23486
23610
|
vectorFetchDepth(limit, offset, cursorDepth)
|
|
23487
23611
|
);
|
|
23488
23612
|
const [textFanOutResult, vectorOutcome] = await Promise.all([textLeg(textOnlyParams, context), vectorPending]);
|
|
@@ -24314,4 +24438,4 @@ async function createNarsil(config) {
|
|
|
24314
24438
|
// src/index.ts
|
|
24315
24439
|
var VERSION = "0.1.0";
|
|
24316
24440
|
|
|
24317
|
-
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps,
|
|
24441
|
+
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|