@delali/narsil 0.3.1-next.90 → 0.3.1-next.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-4JTBL63P.mjs} +13 -5
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-HC6DMR4T.mjs} +2445 -2107
- package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-PP6N5CCS.mjs} +2 -2
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
- package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-PPTZ565G.mjs} +368 -103
- package/dist/browser/{write-6SBYTS62.mjs → write-2ZAA7UDH.mjs} +4 -4
- package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-xsJfdFoQ.d.ts} +11 -3
- package/dist/chunk-2GZGBOQI.mjs +426 -0
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/chunk-7KC56574.mjs +507 -0
- package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-R6BMMQXH.mjs → chunk-C45FUYPI.mjs} +193 -79
- package/dist/{chunk-EMXAYPTQ.mjs → chunk-CHOGQSHK.mjs} +2326 -3503
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-VY4DXZ5P.mjs → chunk-EWBXO45L.mjs} +3 -3
- package/dist/{chunk-QAMUC2BJ.mjs → chunk-EX62BQT6.mjs} +10 -9
- package/dist/{chunk-MWJIAREY.mjs → chunk-EZD4MWOU.mjs} +4 -4
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-URGLSNKR.mjs → chunk-GJZHU4FC.mjs} +202 -16
- package/dist/{chunk-S4LQDSUL.mjs → chunk-J53IHLG7.mjs} +7 -7
- package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/{chunk-V7YUSKKZ.mjs → chunk-QWWCLRXX.mjs} +10 -6
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-DCK4P5BI.mjs → chunk-R36MIGBU.mjs} +100 -59
- package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
- package/dist/{chunk-XSZVS4S3.mjs → chunk-RUIXODYU.mjs} +9 -4
- package/dist/{chunk-A3UB2MXL.mjs → chunk-SBCEHIPL.mjs} +4 -507
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/chunk-UAUZGXJP.mjs +3591 -0
- package/dist/{chunk-CL4OM7GB.mjs → chunk-WBS3EYC4.mjs} +12 -8
- package/dist/{chunk-J2BPXTD7.mjs → chunk-WLOF6HSP.mjs} +3 -3
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
- package/dist/chunk-ZHS44YKF.mjs +35 -0
- package/dist/{client-BMK53IR2.d.ts → client-CgayPw2R.d.ts} +27 -3
- package/dist/client.d.ts +9 -8
- package/dist/client.mjs +16 -2
- package/dist/{config-UJx1dyX1.d.ts → config-3kBsQ0Pi.d.ts} +2 -2
- package/dist/{config-BUR-7eVQ.d.ts → config-B-RnO6dr.d.ts} +2 -2
- package/dist/direct-executor-IJTRU3YN.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -2
- package/dist/distribution/coordinator/etcd.mjs +5 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -2
- package/dist/distribution/coordinator/in-memory.mjs +4 -0
- package/dist/distribution/transport/grpc.mjs +3 -3
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +43 -9
- package/dist/distribution.mjs +4620 -4293
- package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-Y55SCPFC.mjs} +524 -93
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-9X6W5naj.d.ts → engine-DSUfY2F0.d.ts} +53 -17
- package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +44 -25
- package/dist/index.mjs +22 -18
- package/dist/{memory-CV9NAxLY.d.ts → memory-wuXcYOD1.d.ts} +99 -79
- package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
- package/dist/partial-words.mjs +5 -4
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -23
- package/dist/react.d.ts +8 -7
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-4SG45BLF.mjs +35 -0
- package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
- package/dist/server.d.ts +11 -10
- package/dist/server.mjs +38 -11
- package/dist/snapshot-XBKE7PUG.mjs +738 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-BGaZJyoY.d.ts → tasks-Bj9WBlFG.d.ts} +4 -1
- package/dist/{thread-ZP3UMIZT.mjs → thread-U7XYJWGG.mjs} +17 -12
- package/dist/vector/search-worker.d.ts +2 -1
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +6 -6
- package/dist/{write-PXDKPG2R.mjs → write-BMNV7P2H.mjs} +24 -20
- package/package.json +15 -7
- package/dist/chunk-PR4O7CAP.mjs +0 -18
- package/dist/chunk-X4SEXL4U.mjs +0 -2310
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-EGESD26S.mjs +0 -18
- package/dist/snapshot-22CUEYYW.mjs +0 -469
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { clampRowCount, sortSignatureOf, requireSortableFields, resolveProjection, decodePageCursor, requireMatchingCursor, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-
|
|
1
|
+
import { clampRowCount, sortSignatureOf, requireSortableFields, resolveProjection, decodePageCursor, requireMatchingCursor, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex, applyPartitionFiltersBitset } from './chunk-CHOGQSHK.mjs';
|
|
2
2
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-Q5DC6WAB.mjs';
|
|
3
|
-
import { MIN_OVERSAMPLE } from './chunk-
|
|
4
|
-
import {
|
|
5
|
-
import { partialFieldsOf, flattenSchema } from './chunk-CVQWQVFX.mjs';
|
|
3
|
+
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QY7GC2MW.mjs';
|
|
4
|
+
import { partialFieldsOf, flattenSchema } from './chunk-4ZJWRETC.mjs';
|
|
6
5
|
import { DEFAULT_MAX_PER_GROUP, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FGMGUDHX.mjs';
|
|
7
6
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
8
|
-
import { isWordIndexedFieldType } from './chunk-
|
|
7
|
+
import { isWordIndexedFieldType } from './chunk-BB7MASRQ.mjs';
|
|
8
|
+
import { tokenize } from './chunk-7KC56574.mjs';
|
|
9
9
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { holdsUnpairedSurrogate, compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
13
13
|
|
|
14
14
|
// src/search/cursor-binding.ts
|
|
15
15
|
var ABSENT_TAG = 0;
|
|
@@ -305,27 +305,22 @@ function broadcastStatsForWorker(params, context, scoring) {
|
|
|
305
305
|
if (term === void 0 || term.trim().length === 0) {
|
|
306
306
|
return void 0;
|
|
307
307
|
}
|
|
308
|
-
|
|
308
|
+
const { manager } = context;
|
|
309
|
+
const analysis = { ...manager.analysis, synonyms: manager.synonyms };
|
|
310
|
+
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, analysis, {
|
|
311
|
+
partitions: partitionsFor(manager, context.partitionIds),
|
|
312
|
+
expansion: { options: params, schema: context.config.schema }
|
|
313
|
+
});
|
|
309
314
|
}
|
|
310
315
|
function searchOptionsFor(manager, patternWork) {
|
|
311
316
|
return {
|
|
312
317
|
bm25Params: manager.config.bm25,
|
|
313
318
|
stopWords: manager.analysis.stopWords,
|
|
314
319
|
customTokenizer: manager.analysis.customTokenizer,
|
|
320
|
+
synonyms: manager.synonyms,
|
|
315
321
|
patternWork
|
|
316
322
|
};
|
|
317
323
|
}
|
|
318
|
-
function collectFilterDocIds(manager, params, schema, partitionIds, patternWork) {
|
|
319
|
-
const filterDocIds = /* @__PURE__ */ new Set();
|
|
320
|
-
if (!params.filters) return filterDocIds;
|
|
321
|
-
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
322
|
-
const partitionFiltered = partition.applyFilters(params.filters, schema, patternWork);
|
|
323
|
-
for (const docId of partitionFiltered) {
|
|
324
|
-
filterDocIds.add(docId);
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
return filterDocIds;
|
|
328
|
-
}
|
|
329
324
|
function vectorResultsToScored(results) {
|
|
330
325
|
return results.map((r) => ({
|
|
331
326
|
docId: r.docId,
|
|
@@ -879,12 +874,13 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
879
874
|
const prefixLength = options?.prefixLength ?? 0;
|
|
880
875
|
const completedToken = options?.completedToken;
|
|
881
876
|
const queryTolerances = queryTokens.map((qt) => qt.token === completedToken ? 0 : tolerance);
|
|
877
|
+
const synonyms = options?.synonyms;
|
|
882
878
|
const matchByFieldToken = /* @__PURE__ */ new Map();
|
|
883
879
|
const matchesQuery = (fieldToken) => {
|
|
884
880
|
const known = matchByFieldToken.get(fieldToken);
|
|
885
881
|
if (known !== void 0) return known;
|
|
886
882
|
const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
|
|
887
|
-
const matches = stemmedQueryTokens.some(
|
|
883
|
+
const matches = synonyms?.includes(stemmedField) === true || stemmedQueryTokens.some(
|
|
888
884
|
(stemmedQuery, index) => fuzzyTermMatches(stemmedQuery, stemmedField, queryTolerances[index], prefixLength)
|
|
889
885
|
);
|
|
890
886
|
matchByFieldToken.set(fieldToken, matches);
|
|
@@ -947,7 +943,7 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
|
947
943
|
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
948
944
|
};
|
|
949
945
|
}
|
|
950
|
-
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument) {
|
|
946
|
+
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument, synonymLookup) {
|
|
951
947
|
if (!params.highlight) return;
|
|
952
948
|
const queryTokenResult = tokenize(params.term ?? "", language, {
|
|
953
949
|
stem: true,
|
|
@@ -972,6 +968,7 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
972
968
|
}
|
|
973
969
|
}
|
|
974
970
|
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
971
|
+
const synonyms = params.exact === true ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
|
|
975
972
|
for (const hit of hits) {
|
|
976
973
|
const highlights = {};
|
|
977
974
|
const projected = hit.document;
|
|
@@ -991,7 +988,8 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
991
988
|
completedToken,
|
|
992
989
|
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
993
990
|
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
994
|
-
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0
|
|
991
|
+
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0,
|
|
992
|
+
synonyms
|
|
995
993
|
});
|
|
996
994
|
}
|
|
997
995
|
}
|
|
@@ -1184,6 +1182,47 @@ function minMaxNormalize(docs) {
|
|
|
1184
1182
|
return docs.map((d) => ({ docId: d.docId, score: (d.score - min) / range }));
|
|
1185
1183
|
}
|
|
1186
1184
|
|
|
1185
|
+
// src/engine/query/filter-subsets.ts
|
|
1186
|
+
function collectFilterSubsets(manager, filters, schema, partitionIds, patternWork) {
|
|
1187
|
+
const subsets = [];
|
|
1188
|
+
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
1189
|
+
for (const state of partition.readStates()) {
|
|
1190
|
+
subsets.push({
|
|
1191
|
+
owner: state,
|
|
1192
|
+
bits: applyPartitionFiltersBitset(state, filters, schema, patternWork),
|
|
1193
|
+
numberingGeneration: state.docStore.numberingGeneration()
|
|
1194
|
+
});
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
return subsets;
|
|
1198
|
+
}
|
|
1199
|
+
function filterSubsetsMatchNothing(subsets) {
|
|
1200
|
+
for (const subset of subsets) {
|
|
1201
|
+
for (let wordIndex = 0; wordIndex < subset.bits.length; wordIndex++) {
|
|
1202
|
+
if (subset.bits[wordIndex] !== 0) return false;
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
return true;
|
|
1206
|
+
}
|
|
1207
|
+
function filterSubsetsRenumbered(subsets) {
|
|
1208
|
+
return subsets.some((subset) => subset.owner.docStore.numberingGeneration() !== subset.numberingGeneration);
|
|
1209
|
+
}
|
|
1210
|
+
async function searchCurrentSubsets(fieldName, first, collectAgain, search) {
|
|
1211
|
+
let subsets = first;
|
|
1212
|
+
for (let attempt = 1; ; attempt++) {
|
|
1213
|
+
const answer = await search(subsets);
|
|
1214
|
+
if (!filterSubsetsRenumbered(subsets)) return answer;
|
|
1215
|
+
if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
|
|
1216
|
+
throw new NarsilError(
|
|
1217
|
+
ErrorCodes.WORKER_BUSY,
|
|
1218
|
+
`The index renumbered its documents during each of ${attempt} filtered searches of vector field "${fieldName}"`,
|
|
1219
|
+
{ fieldName, attempts: attempt }
|
|
1220
|
+
);
|
|
1221
|
+
}
|
|
1222
|
+
subsets = collectAgain();
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1187
1226
|
// src/engine/query/vector.ts
|
|
1188
1227
|
function oversampleOf(vectorConfig) {
|
|
1189
1228
|
const oversample = vectorConfig.oversample;
|
|
@@ -1208,8 +1247,36 @@ function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
|
1208
1247
|
}
|
|
1209
1248
|
return Math.min(reach + 1, RESULT_WINDOW);
|
|
1210
1249
|
}
|
|
1250
|
+
function filteredSearchOf(params, context) {
|
|
1251
|
+
const filters = params.filters;
|
|
1252
|
+
if (!filters) return void 0;
|
|
1253
|
+
const { manager, config, partitionIds, patternWork } = context;
|
|
1254
|
+
const collectAgain = () => collectFilterSubsets(manager, filters, config.schema, partitionIds, patternWork);
|
|
1255
|
+
return { first: collectAgain(), collectAgain };
|
|
1256
|
+
}
|
|
1257
|
+
function searchField(vecIndex, vectorConfig, queryVector, k, context, filtered) {
|
|
1258
|
+
const options = {
|
|
1259
|
+
metric: vectorConfig.metric ?? "cosine",
|
|
1260
|
+
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1261
|
+
efSearch: vectorConfig.efSearch,
|
|
1262
|
+
oversample: oversampleOf(vectorConfig)
|
|
1263
|
+
};
|
|
1264
|
+
if (filtered !== void 0) {
|
|
1265
|
+
return searchCurrentSubsets(
|
|
1266
|
+
vecIndex.fieldName,
|
|
1267
|
+
filtered.first,
|
|
1268
|
+
filtered.collectAgain,
|
|
1269
|
+
(filterSubsets) => vecIndex.searchParallel(queryVector, k, { ...options, filterSubsets })
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
const filterPartitions = partitionsForVectorSearch(context.manager, vecIndex, context.partitionIds);
|
|
1273
|
+
return vecIndex.searchParallel(
|
|
1274
|
+
queryVector,
|
|
1275
|
+
k,
|
|
1276
|
+
filterPartitions === void 0 ? options : { ...options, filterPartitions }
|
|
1277
|
+
);
|
|
1278
|
+
}
|
|
1211
1279
|
async function executeVectorSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1212
|
-
const { manager, config, partitionIds } = context;
|
|
1213
1280
|
const vectorConfig = params.vector;
|
|
1214
1281
|
if (!vectorConfig || !vectorConfig.value) {
|
|
1215
1282
|
return { scored: [], totalMatched: 0 };
|
|
@@ -1218,26 +1285,13 @@ async function executeVectorSearch(params, context, limit, offset, cursorDepth =
|
|
|
1218
1285
|
if (!vecIndex) {
|
|
1219
1286
|
return { scored: [], totalMatched: 0 };
|
|
1220
1287
|
}
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds, context.patternWork);
|
|
1225
|
-
if (filterDocIds.size === 0) {
|
|
1226
|
-
return { scored: [], totalMatched: 0 };
|
|
1227
|
-
}
|
|
1228
|
-
} else {
|
|
1229
|
-
filterPartitions = partitionsForVectorSearch(manager, vecIndex, partitionIds);
|
|
1288
|
+
const filtered = filteredSearchOf(params, context);
|
|
1289
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
1290
|
+
return { scored: [], totalMatched: 0 };
|
|
1230
1291
|
}
|
|
1231
1292
|
const queryVec = new Float32Array(vectorConfig.value);
|
|
1232
1293
|
const k = vectorFetchDepth(limit, offset, cursorDepth);
|
|
1233
|
-
const outcome = await vecIndex
|
|
1234
|
-
metric: vectorConfig.metric ?? "cosine",
|
|
1235
|
-
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1236
|
-
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
1237
|
-
...filterPartitions !== void 0 ? { filterPartitions } : {},
|
|
1238
|
-
efSearch: vectorConfig.efSearch,
|
|
1239
|
-
oversample: oversampleOf(vectorConfig)
|
|
1240
|
-
});
|
|
1294
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVec, k, context, filtered);
|
|
1241
1295
|
const scored = vectorResultsToScored(outcome.results);
|
|
1242
1296
|
return { scored, totalMatched: outcome.matched, matchedExact: outcome.matchedExact };
|
|
1243
1297
|
}
|
|
@@ -1263,34 +1317,21 @@ async function textLeg(textOnlyParams, context) {
|
|
|
1263
1317
|
searchOptionsFor(manager, context.patternWork)
|
|
1264
1318
|
);
|
|
1265
1319
|
}
|
|
1266
|
-
async function vectorLeg(vectorConfig, queryVector, context,
|
|
1267
|
-
const { manager } = context;
|
|
1320
|
+
async function vectorLeg(vectorConfig, queryVector, context, filtered, k) {
|
|
1268
1321
|
const vecIndex = resolveVectorIndex(context, vectorConfig.field);
|
|
1269
1322
|
if (!vecIndex) return { scored: [], holdsEveryMatch: true };
|
|
1270
|
-
const
|
|
1271
|
-
const outcome = await vecIndex.searchParallel(queryVector, k, {
|
|
1272
|
-
metric: vectorConfig.metric ?? "cosine",
|
|
1273
|
-
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1274
|
-
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
1275
|
-
...filterPartitions !== void 0 ? { filterPartitions } : {},
|
|
1276
|
-
efSearch: vectorConfig.efSearch,
|
|
1277
|
-
oversample: oversampleOf(vectorConfig)
|
|
1278
|
-
});
|
|
1323
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVector, k, context, filtered);
|
|
1279
1324
|
return {
|
|
1280
1325
|
scored: vectorResultsToScored(outcome.results),
|
|
1281
1326
|
holdsEveryMatch: outcome.matchedExact && outcome.results.length === outcome.matched
|
|
1282
1327
|
};
|
|
1283
1328
|
}
|
|
1284
1329
|
async function executeHybridSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1285
|
-
const { manager, config } = context;
|
|
1286
1330
|
const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
|
|
1287
1331
|
const fusion = resolveHybridFusion(params.hybrid);
|
|
1288
|
-
|
|
1289
|
-
if (
|
|
1290
|
-
|
|
1291
|
-
if (filterDocIds.size === 0) {
|
|
1292
|
-
return { scored: [], totalMatched: 0 };
|
|
1293
|
-
}
|
|
1332
|
+
const filtered = filteredSearchOf(params, context);
|
|
1333
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
1334
|
+
return { scored: [], totalMatched: 0 };
|
|
1294
1335
|
}
|
|
1295
1336
|
if (!vectorConfig || !vectorConfig.value) {
|
|
1296
1337
|
const textOnly = await textLeg(textOnlyParams, context);
|
|
@@ -1300,7 +1341,7 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
|
|
|
1300
1341
|
vectorConfig,
|
|
1301
1342
|
new Float32Array(vectorConfig.value),
|
|
1302
1343
|
context,
|
|
1303
|
-
|
|
1344
|
+
filtered,
|
|
1304
1345
|
vectorFetchDepth(limit, offset, cursorDepth)
|
|
1305
1346
|
);
|
|
1306
1347
|
const [textFanOutResult, vectorOutcome] = await Promise.all([textLeg(textOnlyParams, context), vectorPending]);
|
|
@@ -1504,7 +1545,7 @@ async function executeQuery(request, callerContext) {
|
|
|
1504
1545
|
if (params.highlight) {
|
|
1505
1546
|
const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
|
|
1506
1547
|
const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
|
|
1507
|
-
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument);
|
|
1548
|
+
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument, manager.synonyms);
|
|
1508
1549
|
}
|
|
1509
1550
|
const elapsed = now() - startTime;
|
|
1510
1551
|
return {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-QY7GC2MW.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
3
3
|
|
|
4
4
|
// src/serialization/constants.ts
|
|
5
5
|
var CHECKSUM_YIELD_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
|
-
import { TransportError, TransportErrorCodes } from './chunk-
|
|
2
|
+
import { TransportError, TransportErrorCodes } from './chunk-OHP7QR33.mjs';
|
|
3
3
|
import { encode, decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
5
5
|
// src/distribution/transport/tcp/types.ts
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
2
|
-
import { loadNativeCore } from './chunk-
|
|
2
|
+
import { loadNativeCore } from './chunk-ES33L3HO.mjs';
|
|
3
3
|
import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
4
4
|
import { MAX_SORT_FIELDS } from './chunk-FGMGUDHX.mjs';
|
|
5
|
-
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-
|
|
5
|
+
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
6
6
|
import { mkdtemp, open, rm } from 'fs/promises';
|
|
7
7
|
import { tmpdir } from 'os';
|
|
8
8
|
import path from 'path';
|
|
@@ -14,13 +14,15 @@ var TASK_FILTER_CAPABILITY = "tasks.filter";
|
|
|
14
14
|
var REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
|
|
15
15
|
var INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
|
|
16
16
|
var DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
|
|
17
|
+
var SYNONYMS_CAPABILITY = "indexes.synonyms";
|
|
17
18
|
var SERVER_CAPABILITIES = [
|
|
18
19
|
ASYNC_IMPORT_CAPABILITY,
|
|
19
20
|
TASK_CANCEL_CAPABILITY,
|
|
20
21
|
TASK_FILTER_CAPABILITY,
|
|
21
22
|
REBUILD_ANALYSIS_CAPABILITY,
|
|
22
23
|
INDEX_LIFECYCLE_CAPABILITY,
|
|
23
|
-
DOCUMENT_EXPORT_CAPABILITY
|
|
24
|
+
DOCUMENT_EXPORT_CAPABILITY,
|
|
25
|
+
SYNONYMS_CAPABILITY
|
|
24
26
|
];
|
|
25
27
|
|
|
26
28
|
// src/server/concurrency-gate.ts
|
|
@@ -135,6 +137,7 @@ var STATUS_BY_CODE = {
|
|
|
135
137
|
DOC_ALREADY_EXISTS: 409,
|
|
136
138
|
PARTITION_REBALANCING_BACKPRESSURE: 409,
|
|
137
139
|
PARTITION_CAPACITY_EXCEEDED: 409,
|
|
140
|
+
RESTORE_MEMORY_EXCEEDED: 507,
|
|
138
141
|
WORKER_BUSY: 429,
|
|
139
142
|
PARTITION_CORRUPTED: 503,
|
|
140
143
|
WORKER_CRASHED: 503,
|
|
@@ -1129,6 +1132,8 @@ function serverRoutes(limits) {
|
|
|
1129
1132
|
{ method: "post", path: "/indexes/:name/vectors/_optimize", handler: "optimize", options: namedBody },
|
|
1130
1133
|
{ method: "post", path: "/indexes/:name/_rebalance", handler: "rebalance", options: namedBody },
|
|
1131
1134
|
{ method: "post", path: "/indexes/:name/partition-config", handler: "partitionConfig", options: namedBody },
|
|
1135
|
+
{ method: "get", path: "/indexes/:name/synonyms", handler: "readSynonyms", options: named },
|
|
1136
|
+
{ method: "post", path: "/indexes/:name/synonyms", handler: "replaceSynonyms", options: namedBody },
|
|
1132
1137
|
{
|
|
1133
1138
|
method: "post",
|
|
1134
1139
|
path: "/indexes/:name/restore",
|
|
@@ -1221,4 +1226,4 @@ async function departureConfirmation(word) {
|
|
|
1221
1226
|
if (waiting.async) await waiting.value;
|
|
1222
1227
|
}
|
|
1223
1228
|
|
|
1224
|
-
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearGateSlot, confirmDeparture, corsHeaderReader, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
|
|
1229
|
+
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearGateSlot, confirmDeparture, corsHeaderReader, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
|