@delali/narsil 0.3.1-next.127 → 0.3.1-next.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-LJS6PREA.mjs → chunk-3NVZD52E.mjs} +3 -1
- package/dist/browser/{chunk-4Y2EGA3U.mjs → chunk-5D6NJD6C.mjs} +554 -127
- package/dist/browser/{chunk-6POI2KNI.mjs → chunk-II5POT55.mjs} +3 -3
- package/dist/browser/{chunk-2CBQ2PCC.mjs → chunk-WLD6DDIX.mjs} +1 -1
- package/dist/browser/{chunk-V5DCQIOT.mjs → chunk-WZBTVTSU.mjs} +2 -2
- package/dist/browser/{chunk-B65QVTST.mjs → chunk-XTFJZHQP.mjs} +14 -9
- package/dist/browser/{chunk-P7C2S5JX.mjs → chunk-YAGXZU4W.mjs} +2 -2
- package/dist/browser/{durability-wiring-PUC4QSCR.mjs → durability-wiring-HCCC6BJB.mjs} +6 -6
- package/dist/browser/{module-ACWDC7YI.mjs → module-S2VDU2JE.mjs} +5 -5
- package/dist/browser/{restore-queue-MLUEUEAN.mjs → restore-queue-BZ7WWEC4.mjs} +1 -1
- package/dist/browser/{runtime-BX6XRFYR.mjs → runtime-PL4TTFJE.mjs} +1 -1
- package/dist/browser/{snapshot-U5Y346XG.mjs → snapshot-RTUCHI2L.mjs} +7 -7
- package/dist/browser/{write-GZHR6AVG.mjs → write-IMU7G5NU.mjs} +5 -5
- package/dist/{capabilities-9YwwCcqP.d.ts → capabilities-BTjv-tK8.d.ts} +10 -3
- package/dist/{chunk-N57XXGPG.mjs → chunk-2HQQ7Y5D.mjs} +23 -4
- package/dist/{chunk-ENBN5VQ6.mjs → chunk-2YOAGAGF.mjs} +115 -46
- package/dist/{chunk-QSCVJJW5.mjs → chunk-37RTD6KD.mjs} +8 -8
- package/dist/{chunk-N6J6EJW3.mjs → chunk-3LJVM6XM.mjs} +2 -2
- package/dist/{chunk-VK45J47R.mjs → chunk-3XHFVYIQ.mjs} +1 -1
- package/dist/{chunk-LOBYK2FG.mjs → chunk-43Q7HAI2.mjs} +20 -9
- package/dist/{chunk-6PN3U6XW.mjs → chunk-44TB6AIZ.mjs} +1 -1
- package/dist/{chunk-6ODIY6ZB.mjs → chunk-4XEVROSP.mjs} +1 -1
- package/dist/{chunk-4IHMMDRF.mjs → chunk-5LC5VMDQ.mjs} +4 -4
- package/dist/{chunk-MFPJVITN.mjs → chunk-5VSTAJET.mjs} +2 -2
- package/dist/{chunk-DG62H7OD.mjs → chunk-AFOZR4ZQ.mjs} +2 -2
- package/dist/{chunk-P34VZEXB.mjs → chunk-AN3CUMPZ.mjs} +2 -2
- package/dist/{chunk-4PQNDHSA.mjs → chunk-CSSCLRGY.mjs} +2 -2
- package/dist/{chunk-HWQDZB2U.mjs → chunk-CTF6WFTX.mjs} +3 -3
- package/dist/{chunk-E74YLAIG.mjs → chunk-DX2FI4LU.mjs} +2 -1
- package/dist/{chunk-22CK2LQO.mjs → chunk-EZIGAOUJ.mjs} +5 -5
- package/dist/{chunk-KCQG2PJT.mjs → chunk-F4UKMTKM.mjs} +4 -4
- package/dist/{chunk-542TD4BL.mjs → chunk-FRSWYMXH.mjs} +1 -1
- package/dist/{chunk-ANFNYCQG.mjs → chunk-J4LRE5Q7.mjs} +6 -6
- package/dist/{chunk-OGFSUXYY.mjs → chunk-K73PZTBR.mjs} +4 -4
- package/dist/{chunk-IBGMFR5Y.mjs → chunk-K76F5YYX.mjs} +274 -75
- package/dist/{chunk-FUUP45IS.mjs → chunk-M6MBSVIJ.mjs} +1 -1
- package/dist/{chunk-IR4KQ4WB.mjs → chunk-MGD7XETB.mjs} +32 -30
- package/dist/{chunk-7ZZQNOIY.mjs → chunk-OPO5SE4J.mjs} +1 -1
- package/dist/{chunk-UKXMSO5H.mjs → chunk-PCRWCOZC.mjs} +155 -31
- package/dist/{chunk-HFJXEFFV.mjs → chunk-PTIJAZWL.mjs} +3 -3
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/{chunk-SEG4CL6W.mjs → chunk-QQS4QZQA.mjs} +4 -4
- package/dist/{chunk-OFYKBKKV.mjs → chunk-RC27KF7F.mjs} +1 -1
- package/dist/{chunk-JEXJDCQH.mjs → chunk-RYH3PASD.mjs} +17 -12
- package/dist/{chunk-IE43SRL3.mjs → chunk-TB7YMDD5.mjs} +10 -6
- package/dist/{chunk-ZR63OGT2.mjs → chunk-TY64A3ZW.mjs} +2 -2
- package/dist/{chunk-Z4DZ4KK2.mjs → chunk-UJWGQIXU.mjs} +1 -0
- package/dist/{chunk-OBWUWZRA.mjs → chunk-WI5Q3L43.mjs} +1 -0
- package/dist/{chunk-SJEIIR56.mjs → chunk-WQYB4TNR.mjs} +2 -2
- package/dist/{chunk-JIXO442E.mjs → chunk-WUIDPBSY.mjs} +16 -10
- package/dist/{chunk-62IPM62F.mjs → chunk-XRXBCOA4.mjs} +2 -2
- package/dist/{chunk-U6NGTWI3.mjs → chunk-XTER67YM.mjs} +1 -1
- package/dist/{chunk-LXWDQLTW.mjs → chunk-Y4NOHICF.mjs} +1 -1
- package/dist/{chunk-OCB7TMUQ.mjs → chunk-Y4US5EGH.mjs} +1 -1
- package/dist/{client-DyiYkQ8o.d.ts → client-BOcveV7A.d.ts} +34 -11
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +42 -20
- package/dist/{config-DLVnlByZ.d.ts → config-C3YXk5Rn.d.ts} +2 -2
- package/dist/{config-CtTfxbpV.d.ts → config-CPmIY3aH.d.ts} +2 -2
- package/dist/{direct-executor-6KCVVHYT.mjs → direct-executor-ULA6V5HC.mjs} +12 -12
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +18 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +33 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +71 -43
- package/dist/distribution.mjs +359 -115
- package/dist/{durability-wiring-UGGUNATA.mjs → durability-wiring-6R26AJDI.mjs} +26 -26
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-Bq5wz6c1.d.ts → engine-JjRAXpT3.d.ts} +38 -8
- package/dist/{events-x2aVu0_o.d.ts → events-pM0lna7M.d.ts} +12 -1
- package/dist/{index-Cl5qQauj.d.ts → index-DkQ2DlgR.d.ts} +37 -3
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +19 -19
- package/dist/{long-query-thread-5BIPZVS6.mjs → long-query-thread-6VLA6YJI.mjs} +15 -15
- package/dist/{module-7OHZ2OPI.mjs → module-S4WGMFNW.mjs} +11 -11
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +22 -22
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +1 -0
- package/dist/{rerank-NYX5S7OG.mjs → rerank-V4J4MIWR.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-NVMWJMHI.mjs → restore-queue-PUGWCPCX.mjs} +1 -1
- package/dist/{runtime-XNYW6AWX.mjs → runtime-TBFBQQQO.mjs} +7 -7
- package/dist/{schema-4W_7tk1f.d.ts → schema-DGAP_8HJ.d.ts} +50 -24
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +27 -15
- package/dist/{snapshot-P7IQGNNQ.mjs → snapshot-PUVSMV7E.mjs} +27 -27
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DLQIfXqy.d.ts → tasks-txGvNz6h.d.ts} +4 -6
- package/dist/{thread-QPETQEPK.mjs → thread-TA7QQBWP.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-WZVM7ZL6.mjs → write-TGKYT3C4.mjs} +21 -21
- package/package.json +7 -7
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { parseSynonymRules, encodeSynonymRules, decodeSynonymRules, SYNONYM_EXPANSION_CACHE_ENTRIES, SYNONYMS_ADDED_PER_QUERY } from './chunk-7X5JST5L.mjs';
|
|
2
|
-
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-
|
|
2
|
+
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-RC27KF7F.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
5
5
|
|
|
6
6
|
// src/analysis/registry.ts
|
|
7
7
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -62,6 +62,35 @@ function resolveIndexAnalysis(config) {
|
|
|
62
62
|
};
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
// src/engine/serial-index-update.ts
|
|
66
|
+
var updatingByCore = /* @__PURE__ */ new WeakMap();
|
|
67
|
+
async function withOpenIndex(core, indexName, action) {
|
|
68
|
+
core.guardShutdown();
|
|
69
|
+
core.requireIndex(indexName);
|
|
70
|
+
const release = await core.indexState.acquire(indexName);
|
|
71
|
+
try {
|
|
72
|
+
return await action();
|
|
73
|
+
} finally {
|
|
74
|
+
release();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
async function oneAtATime(core, indexName, action) {
|
|
78
|
+
let chains = updatingByCore.get(core);
|
|
79
|
+
if (chains === void 0) {
|
|
80
|
+
chains = /* @__PURE__ */ new Map();
|
|
81
|
+
updatingByCore.set(core, chains);
|
|
82
|
+
}
|
|
83
|
+
const updating = chains;
|
|
84
|
+
const earlier = updating.get(indexName) ?? Promise.resolve();
|
|
85
|
+
const current = earlier.catch(() => void 0).then(action);
|
|
86
|
+
updating.set(indexName, current);
|
|
87
|
+
try {
|
|
88
|
+
return await current;
|
|
89
|
+
} finally {
|
|
90
|
+
if (updating.get(indexName) === current) updating.delete(indexName);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
65
94
|
// src/core/synonyms/combined-postings.ts
|
|
66
95
|
var NO_DELETED = { size: 0, has: () => false };
|
|
67
96
|
var FIELD_SLOTS = 65536;
|
|
@@ -345,33 +374,6 @@ function storedSynonymRules(encoded) {
|
|
|
345
374
|
}
|
|
346
375
|
|
|
347
376
|
// src/core/synonyms/operations.ts
|
|
348
|
-
var replacingByCore = /* @__PURE__ */ new WeakMap();
|
|
349
|
-
async function withOpenIndex(core, indexName, action) {
|
|
350
|
-
core.guardShutdown();
|
|
351
|
-
core.requireIndex(indexName);
|
|
352
|
-
const release = await core.indexState.acquire(indexName);
|
|
353
|
-
try {
|
|
354
|
-
return await action();
|
|
355
|
-
} finally {
|
|
356
|
-
release();
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
async function oneAtATime(core, indexName, action) {
|
|
360
|
-
let chains = replacingByCore.get(core);
|
|
361
|
-
if (chains === void 0) {
|
|
362
|
-
chains = /* @__PURE__ */ new Map();
|
|
363
|
-
replacingByCore.set(core, chains);
|
|
364
|
-
}
|
|
365
|
-
const replacing = chains;
|
|
366
|
-
const earlier = replacing.get(indexName) ?? Promise.resolve();
|
|
367
|
-
const current = earlier.catch(() => void 0).then(action);
|
|
368
|
-
replacing.set(indexName, current);
|
|
369
|
-
try {
|
|
370
|
-
return await current;
|
|
371
|
-
} finally {
|
|
372
|
-
if (replacing.get(indexName) === current) replacing.delete(indexName);
|
|
373
|
-
}
|
|
374
|
-
}
|
|
375
377
|
async function replaceNow(core, indexName, value, strictWords) {
|
|
376
378
|
const entry = core.requireIndex(indexName);
|
|
377
379
|
const manager = core.requireManager(indexName);
|
|
@@ -423,4 +425,4 @@ var synonymSearchModule = {
|
|
|
423
425
|
applyLoaded: applyLoadedSynonyms
|
|
424
426
|
};
|
|
425
427
|
|
|
426
|
-
export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerStopWords, registerTokenizer, resolveIndexAnalysis, synonymSearchModule };
|
|
428
|
+
export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, oneAtATime, registerStopWords, registerTokenizer, resolveIndexAnalysis, synonymSearchModule, withOpenIndex };
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-
|
|
2
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-
|
|
1
|
+
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, bm25SetsValue, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, mergeBM25Levels, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex, MAX_BM25_K1, MAX_BM25_B } from './chunk-K76F5YYX.mjs';
|
|
2
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-Y4NOHICF.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QSK35UC2.mjs';
|
|
4
|
-
import { flattenSchema, partialFieldsOf } from './chunk-
|
|
5
|
-
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-
|
|
4
|
+
import { flattenSchema, partialFieldsOf } from './chunk-AFOZR4ZQ.mjs';
|
|
5
|
+
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-DX2FI4LU.mjs';
|
|
6
6
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-
|
|
8
|
-
import { tokenize } from './chunk-
|
|
7
|
+
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-4XEVROSP.mjs';
|
|
8
|
+
import { tokenize } from './chunk-RC27KF7F.mjs';
|
|
9
9
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { compareCodePoints, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
|
-
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
11
|
+
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isPlainObject, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
13
13
|
|
|
14
14
|
// src/search/cursor-binding.ts
|
|
15
15
|
var ABSENT_TAG = 0;
|
|
@@ -154,6 +154,27 @@ function boundRerankOf(rerank) {
|
|
|
154
154
|
}
|
|
155
155
|
return present;
|
|
156
156
|
}
|
|
157
|
+
function presentMembersOf(record) {
|
|
158
|
+
const present = {};
|
|
159
|
+
for (const [key, value] of Object.entries(record)) {
|
|
160
|
+
if (value !== void 0) present[key] = value;
|
|
161
|
+
}
|
|
162
|
+
return present;
|
|
163
|
+
}
|
|
164
|
+
function boundBm25Of(bm25) {
|
|
165
|
+
const bound = presentMembersOf(bm25);
|
|
166
|
+
const fields = bm25.fields;
|
|
167
|
+
if (typeof fields !== "object" || fields === null) return bound;
|
|
168
|
+
const boundFields = {};
|
|
169
|
+
for (const [name, pair] of Object.entries(fields)) {
|
|
170
|
+
const members = typeof pair === "object" && pair !== null ? presentMembersOf(pair) : pair;
|
|
171
|
+
if (typeof members === "object" && members !== null && Object.keys(members).length === 0) continue;
|
|
172
|
+
boundFields[name] = members;
|
|
173
|
+
}
|
|
174
|
+
if (Object.keys(boundFields).length === 0) delete bound.fields;
|
|
175
|
+
else bound.fields = boundFields;
|
|
176
|
+
return bound;
|
|
177
|
+
}
|
|
157
178
|
var FILTERS_SLOT = 2;
|
|
158
179
|
var BOUND_SLOT_COUNT = 16;
|
|
159
180
|
function bindingOf(values, clock) {
|
|
@@ -184,6 +205,7 @@ function queryBindingOf(params, clock) {
|
|
|
184
205
|
params.scoring ?? "local"
|
|
185
206
|
];
|
|
186
207
|
if (params.rerank !== void 0) values.push(boundRerankOf(params.rerank));
|
|
208
|
+
if (params.bm25 !== void 0 && bm25SetsValue(params.bm25)) values.push(boundBm25Of(params.bm25));
|
|
187
209
|
return bindingOf(values, clock);
|
|
188
210
|
}
|
|
189
211
|
function lazyQueryBinding(params, clock) {
|
|
@@ -199,6 +221,110 @@ function listBindingOf(filters) {
|
|
|
199
221
|
return bindingOf(values);
|
|
200
222
|
}
|
|
201
223
|
|
|
224
|
+
// src/search/bm25-params.ts
|
|
225
|
+
var BM25_KEYS = /* @__PURE__ */ new Set(["k1", "b", "fields"]);
|
|
226
|
+
var BM25_FIELD_KEYS = /* @__PURE__ */ new Set(["k1", "b"]);
|
|
227
|
+
function displayed(value) {
|
|
228
|
+
if (typeof value === "string") return `"${value.slice(0, BM25_DISPLAYED_NAME_LENGTH)}"`;
|
|
229
|
+
if (typeof value === "number" || typeof value === "boolean" || value === null) return String(value);
|
|
230
|
+
return `a value of type ${typeof value}`;
|
|
231
|
+
}
|
|
232
|
+
function requireValue(label, value, maximum, ranges) {
|
|
233
|
+
if (value === void 0) return;
|
|
234
|
+
const usable = typeof value === "number" && Number.isFinite(value) && (ranges === "stored" || value >= 0 && value <= maximum);
|
|
235
|
+
if (usable) return;
|
|
236
|
+
throw new NarsilError(
|
|
237
|
+
ErrorCodes.CONFIG_INVALID,
|
|
238
|
+
`${label} must be a finite number from 0 to ${maximum}, and this request sets ${displayed(value)}`,
|
|
239
|
+
{ parameter: label }
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
function requireKnownKeys(label, value, known) {
|
|
243
|
+
for (const key of Object.keys(value)) {
|
|
244
|
+
if (known.has(key)) continue;
|
|
245
|
+
throw new NarsilError(
|
|
246
|
+
ErrorCodes.CONFIG_INVALID,
|
|
247
|
+
`${label} takes ${[...known].join(", ")}, and this request sets ${displayed(key)}`,
|
|
248
|
+
{ parameter: label }
|
|
249
|
+
);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
function requireFieldPair(name, pair, ranges) {
|
|
253
|
+
const label = `bm25.fields.${name.slice(0, BM25_DISPLAYED_NAME_LENGTH)}`;
|
|
254
|
+
if (!isPlainObject(pair)) {
|
|
255
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${label} must be an object that sets k1 or b`, {
|
|
256
|
+
parameter: label
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
requireKnownKeys(label, pair, BM25_FIELD_KEYS);
|
|
260
|
+
requireValue(`${label}.k1`, pair.k1, MAX_BM25_K1, ranges);
|
|
261
|
+
requireValue(`${label}.b`, pair.b, MAX_BM25_B, ranges);
|
|
262
|
+
}
|
|
263
|
+
function requireScoredField(name, flatSchema, unknownFieldCode) {
|
|
264
|
+
const fieldType = Object.hasOwn(flatSchema, name) ? flatSchema[name] : void 0;
|
|
265
|
+
if (typeof fieldType === "string" && isWordIndexedFieldType(fieldType)) return;
|
|
266
|
+
throw new NarsilError(
|
|
267
|
+
unknownFieldCode,
|
|
268
|
+
fieldType === void 0 ? `BM25 parameters apply to a text field that the schema declares, and the schema declares no field ${displayed(name)}` : `BM25 parameters apply to a text field alone, and the schema declares field ${displayed(name)} as ${displayed(fieldType)}`,
|
|
269
|
+
{ field: name.slice(0, BM25_DISPLAYED_NAME_LENGTH) }
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
function fieldsOfCheckedSetting(raw, ranges) {
|
|
273
|
+
if (!isPlainObject(raw)) {
|
|
274
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25 must be an object that sets k1, b, or fields");
|
|
275
|
+
}
|
|
276
|
+
requireKnownKeys("bm25", raw, BM25_KEYS);
|
|
277
|
+
requireValue("bm25.k1", raw.k1, MAX_BM25_K1, ranges);
|
|
278
|
+
requireValue("bm25.b", raw.b, MAX_BM25_B, ranges);
|
|
279
|
+
const fields = raw.fields;
|
|
280
|
+
if (fields === void 0) return void 0;
|
|
281
|
+
if (!isPlainObject(fields)) {
|
|
282
|
+
throw new NarsilError(
|
|
283
|
+
ErrorCodes.CONFIG_INVALID,
|
|
284
|
+
"bm25.fields must be an object that maps a field to its parameters"
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
return fields;
|
|
288
|
+
}
|
|
289
|
+
function validateBM25Shape(raw, ranges) {
|
|
290
|
+
const fields = fieldsOfCheckedSetting(raw, ranges);
|
|
291
|
+
if (fields === void 0) return;
|
|
292
|
+
for (const name of Object.keys(fields)) requireFieldPair(name, fields[name], ranges);
|
|
293
|
+
}
|
|
294
|
+
function validateBM25Params(raw, schema, unknownFieldCode, ranges = "settable") {
|
|
295
|
+
const fields = fieldsOfCheckedSetting(raw, ranges);
|
|
296
|
+
if (fields === void 0) return;
|
|
297
|
+
const names = Object.keys(fields);
|
|
298
|
+
if (names.length === 0) return;
|
|
299
|
+
const flatSchema = flattenSchema(schema);
|
|
300
|
+
for (const name of names) {
|
|
301
|
+
requireScoredField(name, flatSchema, unknownFieldCode);
|
|
302
|
+
requireFieldPair(name, fields[name], ranges);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
function presentPair(pair) {
|
|
306
|
+
return {
|
|
307
|
+
...pair.k1 !== void 0 ? { k1: pair.k1 } : {},
|
|
308
|
+
...pair.b !== void 0 ? { b: pair.b } : {}
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
function storedBM25Params(params) {
|
|
312
|
+
const stored = presentPair(params);
|
|
313
|
+
const fields = params.fields;
|
|
314
|
+
if (fields !== void 0) {
|
|
315
|
+
const kept = /* @__PURE__ */ Object.create(null);
|
|
316
|
+
let holdsField = false;
|
|
317
|
+
for (const name of Object.keys(fields)) {
|
|
318
|
+
const pair = presentPair(fields[name]);
|
|
319
|
+
if (pair.k1 === void 0 && pair.b === void 0) continue;
|
|
320
|
+
kept[name] = pair;
|
|
321
|
+
holdsField = true;
|
|
322
|
+
}
|
|
323
|
+
if (holdsField) stored.fields = kept;
|
|
324
|
+
}
|
|
325
|
+
return Object.keys(stored).length === 0 ? void 0 : stored;
|
|
326
|
+
}
|
|
327
|
+
|
|
202
328
|
// src/search/query-fields.ts
|
|
203
329
|
function requireBoostableFields(boost, flatSchema) {
|
|
204
330
|
for (const [field, weight] of Object.entries(boost)) {
|
|
@@ -246,6 +372,7 @@ function requireGroupableFields(fields, flatSchema, strict) {
|
|
|
246
372
|
}
|
|
247
373
|
}
|
|
248
374
|
function requireUsableQueryFields(params, schema, strict = false) {
|
|
375
|
+
if (params.bm25 !== void 0) validateBM25Params(params.bm25, schema, ErrorCodes.SEARCH_INVALID_FIELD);
|
|
249
376
|
const groupFields = params.group?.fields;
|
|
250
377
|
const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
|
|
251
378
|
if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
|
|
@@ -324,9 +451,13 @@ function broadcastStatsForWorker(params, context, scoring) {
|
|
|
324
451
|
expansion: { options: params, schema: context.config.schema }
|
|
325
452
|
});
|
|
326
453
|
}
|
|
327
|
-
function
|
|
454
|
+
function bm25SettingFor(params, context) {
|
|
455
|
+
return mergeBM25Levels(params.bm25, context.indexBm25 ?? context.config.bm25);
|
|
456
|
+
}
|
|
457
|
+
function searchOptionsFor(params, context) {
|
|
458
|
+
const { manager, patternWork } = context;
|
|
328
459
|
return {
|
|
329
|
-
bm25Params:
|
|
460
|
+
bm25Params: bm25SettingFor(params, context),
|
|
330
461
|
stopWords: manager.analysis.stopWords,
|
|
331
462
|
customTokenizer: manager.analysis.customTokenizer,
|
|
332
463
|
synonyms: manager.synonyms,
|
|
@@ -532,16 +663,6 @@ function validateWorkerConfig(workers, lifecycle) {
|
|
|
532
663
|
);
|
|
533
664
|
}
|
|
534
665
|
}
|
|
535
|
-
function validateBM25Params(bm25) {
|
|
536
|
-
if (bm25 === void 0) return;
|
|
537
|
-
const { k1, b } = bm25;
|
|
538
|
-
if (k1 !== void 0 && (!Number.isFinite(k1) || k1 < 0)) {
|
|
539
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.k1 must be a finite number at or above 0", { k1 });
|
|
540
|
-
}
|
|
541
|
-
if (b !== void 0 && (!Number.isFinite(b) || b < 0 || b > 1)) {
|
|
542
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.b must be between 0 and 1", { b });
|
|
543
|
-
}
|
|
544
|
-
}
|
|
545
666
|
function clampLimit(limit) {
|
|
546
667
|
return clampRowCount(limit, DEFAULT_LIMIT);
|
|
547
668
|
}
|
|
@@ -1240,7 +1361,8 @@ async function textLeg(textOnlyParams, context) {
|
|
|
1240
1361
|
textOnlyParams,
|
|
1241
1362
|
broadcastStatsForWorker(textOnlyParams, context, scoring),
|
|
1242
1363
|
context.partitionIds,
|
|
1243
|
-
context.patternWork
|
|
1364
|
+
context.patternWork,
|
|
1365
|
+
bm25SettingFor(textOnlyParams, context)
|
|
1244
1366
|
);
|
|
1245
1367
|
if (viaWorker) return viaWorker;
|
|
1246
1368
|
}
|
|
@@ -1250,7 +1372,7 @@ async function textLeg(textOnlyParams, context) {
|
|
|
1250
1372
|
language,
|
|
1251
1373
|
config.schema,
|
|
1252
1374
|
scoring,
|
|
1253
|
-
searchOptionsFor(
|
|
1375
|
+
searchOptionsFor(textOnlyParams, context)
|
|
1254
1376
|
);
|
|
1255
1377
|
}
|
|
1256
1378
|
async function vectorLeg(vectorConfig, queryVector, context, filtered, k, rescoreWindow) {
|
|
@@ -1348,7 +1470,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
1348
1470
|
const flatSchema = flattenSchema(schema);
|
|
1349
1471
|
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
1350
1472
|
const modes = sortModesOf(normalized);
|
|
1351
|
-
const options = searchOptionsFor(
|
|
1473
|
+
const options = searchOptionsFor(params, context);
|
|
1352
1474
|
const partitions = partitionsFor(manager, context.partitionIds);
|
|
1353
1475
|
const partitionLimit = offset + limit + 1;
|
|
1354
1476
|
const merged = [];
|
|
@@ -1416,13 +1538,13 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
1416
1538
|
}
|
|
1417
1539
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
1418
1540
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
1419
|
-
return import('./runtime-
|
|
1541
|
+
return import('./runtime-TBFBQQQO.mjs').then(
|
|
1420
1542
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
1421
1543
|
);
|
|
1422
1544
|
}
|
|
1423
1545
|
function executeQuery(request, callerContext) {
|
|
1424
1546
|
if (request.rerank === void 0) return runQuery(request, callerContext);
|
|
1425
|
-
return import('./runtime-
|
|
1547
|
+
return import('./runtime-TBFBQQQO.mjs').then(
|
|
1426
1548
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
1427
1549
|
);
|
|
1428
1550
|
}
|
|
@@ -1479,7 +1601,8 @@ async function runQuery(request, callerContext) {
|
|
|
1479
1601
|
params,
|
|
1480
1602
|
broadcastStatsForWorker(params, context, scoring),
|
|
1481
1603
|
context.partitionIds,
|
|
1482
|
-
context.patternWork
|
|
1604
|
+
context.patternWork,
|
|
1605
|
+
bm25SettingFor(params, context)
|
|
1483
1606
|
) : null;
|
|
1484
1607
|
if (workerResult) {
|
|
1485
1608
|
fanOutResult = workerResult;
|
|
@@ -1490,7 +1613,7 @@ async function runQuery(request, callerContext) {
|
|
|
1490
1613
|
language,
|
|
1491
1614
|
config.schema,
|
|
1492
1615
|
scoring,
|
|
1493
|
-
searchOptionsFor(
|
|
1616
|
+
searchOptionsFor(params, context)
|
|
1494
1617
|
);
|
|
1495
1618
|
}
|
|
1496
1619
|
}
|
|
@@ -1640,7 +1763,7 @@ async function executePreflight(request, callerContext) {
|
|
|
1640
1763
|
countExact = result.matchedExact !== false;
|
|
1641
1764
|
} else if (countsWithoutScores(params)) {
|
|
1642
1765
|
totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
|
|
1643
|
-
searchOptions: searchOptionsFor(
|
|
1766
|
+
searchOptions: searchOptionsFor(params, context),
|
|
1644
1767
|
partitionIds: context.partitionIds
|
|
1645
1768
|
});
|
|
1646
1769
|
} else {
|
|
@@ -1650,7 +1773,8 @@ async function executePreflight(request, callerContext) {
|
|
|
1650
1773
|
params,
|
|
1651
1774
|
broadcastStatsForWorker(params, context, scoring),
|
|
1652
1775
|
context.partitionIds,
|
|
1653
|
-
context.patternWork
|
|
1776
|
+
context.patternWork,
|
|
1777
|
+
bm25SettingFor(params, context)
|
|
1654
1778
|
) : null;
|
|
1655
1779
|
if (workerResult) {
|
|
1656
1780
|
totalMatched = workerResult.totalMatched;
|
|
@@ -1661,7 +1785,7 @@ async function executePreflight(request, callerContext) {
|
|
|
1661
1785
|
language,
|
|
1662
1786
|
config.schema,
|
|
1663
1787
|
scoring,
|
|
1664
|
-
searchOptionsFor(
|
|
1788
|
+
searchOptionsFor(params, context)
|
|
1665
1789
|
);
|
|
1666
1790
|
totalMatched = fanOutResult.totalMatched;
|
|
1667
1791
|
}
|
|
@@ -1717,4 +1841,4 @@ function pickDisplaySurface(surfaceOccurrences) {
|
|
|
1717
1841
|
return best;
|
|
1718
1842
|
}
|
|
1719
1843
|
|
|
1720
|
-
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, lazyQueryBinding, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, runtimeHost, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|
|
1844
|
+
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, lazyQueryBinding, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, runtimeHost, storedBM25Params, validateBM25Params, validateBM25Shape, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { admittedHeapBytes, collectGarbageBeforeAnswerRefusal, collectYoungGenerationBeforeRefusal, reservedHeapBytes, admitHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
2
2
|
import { readOldGenerationStatistics, processLimitBytes } from './chunk-ZATRDAYU.mjs';
|
|
3
3
|
import { engineThreadYoungGenerationBytes } from './chunk-EBIHRL7B.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes, ServerErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes, ServerErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
5
5
|
|
|
6
6
|
// src/distribution/cluster-node/constants.ts
|
|
7
7
|
var DEFAULT_PARTITION_COUNT = 5;
|
|
@@ -36,7 +36,7 @@ var CATCH_UP_TICK_MS = 1e3;
|
|
|
36
36
|
var CATCH_UP_FEED_WINDOW_MS = 1e3;
|
|
37
37
|
var MAX_CATCH_UP_IN_FLIGHT_BYTES = 67108864;
|
|
38
38
|
var CLEAR_PAGE_SIZE = 1e3;
|
|
39
|
-
var
|
|
39
|
+
var MAX_SETTINGS_REPLACEMENT_ATTEMPTS = 5;
|
|
40
40
|
var DEFAULT_MAX_PER_SOURCE_SNAPSHOTS = 1;
|
|
41
41
|
var DEFAULT_MAX_STREAMS_PER_INDEX = 4;
|
|
42
42
|
var SNAPSHOT_CHUNK_YIELD_INTERVAL_MS = 8;
|
|
@@ -298,4 +298,4 @@ function refuseEncodeWithoutRoom(documents) {
|
|
|
298
298
|
if (refusal2 !== null) throw refusal2;
|
|
299
299
|
}
|
|
300
300
|
|
|
301
|
-
export { ADMISSION_TIMEOUT_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CASE_FOLD_EXPANSION_FACTOR, CATCH_UP_FEED_WINDOW_MS, CATCH_UP_TICK_MS, CLEAR_PAGE_SIZE, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, COPY_FAILURE_RETRY_DELAY_MS, DECOMMISSION_RECHECK_INTERVAL_MS, DEFAULT_BLOCK_WRITES_ABOVE, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DEFAULT_CAPACITY, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_RESUME_WRITES_BELOW, DISTINCT_TERM_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, ENUM_VALUE_HEAP_BYTES, FETCH_GROUP_DOCUMENT_IDS, FORWARD_RETRY_DELAY_CAP_MS, FORWARD_RETRY_INITIAL_DELAY_MS, FULL_MEMORY_COLLECTION_INTERVAL_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, MAX_DROP_ACKNOWLEDGE_WAIT_MS, MAX_NODE_LEFT_DELAY_MS,
|
|
301
|
+
export { ADMISSION_TIMEOUT_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CASE_FOLD_EXPANSION_FACTOR, CATCH_UP_FEED_WINDOW_MS, CATCH_UP_TICK_MS, CLEAR_PAGE_SIZE, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, COPY_FAILURE_RETRY_DELAY_MS, DECOMMISSION_RECHECK_INTERVAL_MS, DEFAULT_BLOCK_WRITES_ABOVE, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DEFAULT_CAPACITY, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_RESUME_WRITES_BELOW, DISTINCT_TERM_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, ENUM_VALUE_HEAP_BYTES, FETCH_GROUP_DOCUMENT_IDS, FORWARD_RETRY_DELAY_CAP_MS, FORWARD_RETRY_INITIAL_DELAY_MS, FULL_MEMORY_COLLECTION_INTERVAL_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, MAX_DROP_ACKNOWLEDGE_WAIT_MS, MAX_NODE_LEFT_DELAY_MS, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, MAX_SOURCE_ID_LENGTH, MAX_WAIT_FOR_ACTIVE_REPLICAS, MEMORY_SAMPLE_INTERVAL_MS, MIN_CONTROLLER_LEASE_TTL_MS, NORMALIZATION_EXPANSION_FACTOR, PARTIAL_SPELLING_HEAP_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, PARTITION_WRITE_CONCURRENCY, PATTERN_RUN_HEAP_BYTES, PRIMARY_CHANGE_POLL_INTERVAL_MS, PRIMARY_CHANGE_RETRY_WINDOW_MS, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, ROLLBACK_RETRY_DELAY_CAP_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, SEARCH_CONTEXT_REAPER_INTERVAL_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, SYNCED_POSITION_REPORT_TIMEOUT_MS, TERM_UNIT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, WRITE_APPLY_YIELD_INTERVAL_MS, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, answerRefusalOf, createWritePressure, estimateDocumentBytes, estimateDocumentsBytes, estimateForwardedBytes, readMemoryFigures, readRecentMemoryFigures, refuseEncodeWithoutRoom, withAdmittedWrite };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// src/distribution/coordinator/index-metadata-key.ts
|
|
2
|
+
var INDEX_METADATA_KEY_ROOT = "_narsil/index";
|
|
3
|
+
var INDEX_METADATA_KEY_SUFFIX = "/config";
|
|
4
|
+
var INDEX_METADATA_KEY_PREFIX = `${INDEX_METADATA_KEY_ROOT}/`;
|
|
5
|
+
function indexNameOfMetadataSuffix(keySuffix) {
|
|
6
|
+
if (!keySuffix.endsWith(INDEX_METADATA_KEY_SUFFIX)) return null;
|
|
7
|
+
const indexName = keySuffix.slice(0, -INDEX_METADATA_KEY_SUFFIX.length);
|
|
8
|
+
return indexName.length === 0 || indexName.includes("/") ? null : indexName;
|
|
9
|
+
}
|
|
10
|
+
function indexNameOfMetadataKey(key) {
|
|
11
|
+
if (!key.startsWith(INDEX_METADATA_KEY_PREFIX)) return null;
|
|
12
|
+
return indexNameOfMetadataSuffix(key.slice(INDEX_METADATA_KEY_PREFIX.length));
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export { INDEX_METADATA_KEY_PREFIX, indexNameOfMetadataKey, indexNameOfMetadataSuffix };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-
|
|
2
|
-
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-
|
|
1
|
+
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-Y4US5EGH.mjs';
|
|
2
|
+
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-XRXBCOA4.mjs';
|
|
3
3
|
import { osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
4
|
-
import { MAX_DOC_ID_LENGTH } from './chunk-
|
|
4
|
+
import { MAX_DOC_ID_LENGTH } from './chunk-Y4NOHICF.mjs';
|
|
5
5
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
7
7
|
|
|
8
8
|
// src/vector/filter-ordinals.ts
|
|
9
9
|
var UNRESOLVED_ORDINAL = -1;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
2
|
-
import { createNarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { createNarsilError, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/tokenizer/constants.ts
|
|
5
5
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-
|
|
2
|
-
import { generateId, getLanguage } from './chunk-
|
|
1
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-K76F5YYX.mjs';
|
|
2
|
+
import { generateId, getLanguage } from './chunk-M6MBSVIJ.mjs';
|
|
3
3
|
import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
|
|
4
|
-
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-
|
|
5
|
-
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-
|
|
6
|
-
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-
|
|
4
|
+
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-AN3CUMPZ.mjs';
|
|
5
|
+
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-F4UKMTKM.mjs';
|
|
6
|
+
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-AFOZR4ZQ.mjs';
|
|
7
7
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
8
|
-
import { isWordIndexedBase } from './chunk-
|
|
9
|
-
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-
|
|
10
|
-
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-
|
|
8
|
+
import { isWordIndexedBase } from './chunk-4XEVROSP.mjs';
|
|
9
|
+
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-MGD7XETB.mjs';
|
|
10
|
+
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-RC27KF7F.mjs';
|
|
11
11
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
12
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
12
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-XTER67YM.mjs';
|
|
13
13
|
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
14
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
14
|
+
import { NarsilError, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
15
15
|
import { Decoder, decode } from '@msgpack/msgpack';
|
|
16
16
|
|
|
17
17
|
// src/core/synonyms/platform.ts
|
|
@@ -541,7 +541,8 @@ function createPartitionRouter() {
|
|
|
541
541
|
// src/workers/query-actions.ts
|
|
542
542
|
async function runQueryAction(entry, action) {
|
|
543
543
|
const patternWork = joinPatternWorkMeter(action.patternWork, action.params.filters);
|
|
544
|
-
const
|
|
544
|
+
const bm25Params = action.type === "query" ? action.bm25 ?? void 0 : entry.config.bm25;
|
|
545
|
+
const searchOptions = { ...entry.searchOptions, patternWork, bm25Params };
|
|
545
546
|
if (action.type === "query") {
|
|
546
547
|
const result2 = await fanOutQuery(
|
|
547
548
|
entry.manager,
|
|
@@ -3292,7 +3293,6 @@ function createDirectExecutor(options) {
|
|
|
3292
3293
|
config,
|
|
3293
3294
|
language,
|
|
3294
3295
|
searchOptions: {
|
|
3295
|
-
bm25Params: config.bm25,
|
|
3296
3296
|
stopWords: manager.analysis.stopWords,
|
|
3297
3297
|
customTokenizer: manager.analysis.customTokenizer,
|
|
3298
3298
|
get synonyms() {
|
|
@@ -3433,6 +3433,11 @@ function createDirectExecutor(options) {
|
|
|
3433
3433
|
if (entry !== void 0) entry.manager.synonyms = lookupOf(entry.config, entry.language, action.synonyms);
|
|
3434
3434
|
return void 0;
|
|
3435
3435
|
}
|
|
3436
|
+
case "replaceBm25": {
|
|
3437
|
+
const entry = indexes.get(action.indexName);
|
|
3438
|
+
if (entry !== void 0) entry.config.bm25 = action.bm25;
|
|
3439
|
+
return void 0;
|
|
3440
|
+
}
|
|
3436
3441
|
case "get": {
|
|
3437
3442
|
const entry = requireIndex(action.indexName);
|
|
3438
3443
|
return entry.manager.get(action.docId);
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { readRecentMemoryFigures, readMemoryFigures, answerRefusalOf } from './chunk-
|
|
1
|
+
import { readRecentMemoryFigures, readMemoryFigures, answerRefusalOf } from './chunk-PTIJAZWL.mjs';
|
|
2
2
|
import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
3
3
|
import { nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
4
4
|
import { DEFAULT_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
5
|
-
import { loadNativeCore } from './chunk-
|
|
5
|
+
import { loadNativeCore } from './chunk-3XHFVYIQ.mjs';
|
|
6
6
|
import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
7
|
-
import { MAX_SORT_FIELDS } from './chunk-
|
|
7
|
+
import { MAX_SORT_FIELDS } from './chunk-DX2FI4LU.mjs';
|
|
8
8
|
import { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-22YPZVMG.mjs';
|
|
9
|
-
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-
|
|
9
|
+
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
10
10
|
import { randomUUID } from 'crypto';
|
|
11
11
|
import { mkdtemp, open, rm } from 'fs/promises';
|
|
12
12
|
import { tmpdir } from 'os';
|
|
@@ -71,6 +71,7 @@ var REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
|
|
|
71
71
|
var INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
|
|
72
72
|
var DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
|
|
73
73
|
var SYNONYMS_CAPABILITY = "indexes.synonyms";
|
|
74
|
+
var BM25_TUNING_CAPABILITY = "indexes.bm25";
|
|
74
75
|
var SERVER_CAPABILITIES = [
|
|
75
76
|
ASYNC_IMPORT_CAPABILITY,
|
|
76
77
|
TASK_CANCEL_CAPABILITY,
|
|
@@ -78,7 +79,8 @@ var SERVER_CAPABILITIES = [
|
|
|
78
79
|
REBUILD_ANALYSIS_CAPABILITY,
|
|
79
80
|
INDEX_LIFECYCLE_CAPABILITY,
|
|
80
81
|
DOCUMENT_EXPORT_CAPABILITY,
|
|
81
|
-
SYNONYMS_CAPABILITY
|
|
82
|
+
SYNONYMS_CAPABILITY,
|
|
83
|
+
BM25_TUNING_CAPABILITY
|
|
82
84
|
];
|
|
83
85
|
|
|
84
86
|
// src/server/concurrency-gate.ts
|
|
@@ -209,6 +211,7 @@ var STATUS_BY_CODE = {
|
|
|
209
211
|
EMBEDDING_FAILED: 503,
|
|
210
212
|
RERANK_FAILED: 503,
|
|
211
213
|
QUERY_ROUTING_FAILED: 503,
|
|
214
|
+
BM25_NOT_READY: 503,
|
|
212
215
|
QUERY_PARTIAL_FAILURE: 503,
|
|
213
216
|
QUERY_NODE_TIMEOUT: 503,
|
|
214
217
|
QUERY_NO_ACTIVE_REPLICA: 503,
|
|
@@ -1704,6 +1707,7 @@ function serverRoutes(limits) {
|
|
|
1704
1707
|
{ method: "post", path: "/indexes/:name/partition-config", handler: "partitionConfig", options: namedBody },
|
|
1705
1708
|
{ method: "get", path: "/indexes/:name/synonyms", handler: "readSynonyms", options: named },
|
|
1706
1709
|
{ method: "post", path: "/indexes/:name/synonyms", handler: "replaceSynonyms", options: namedBody },
|
|
1710
|
+
{ method: "post", path: "/indexes/:name/bm25", handler: "updateBm25", options: namedBody },
|
|
1707
1711
|
{
|
|
1708
1712
|
method: "post",
|
|
1709
1713
|
path: "/indexes/:name/restore",
|
|
@@ -1830,4 +1834,4 @@ async function departureConfirmation(word) {
|
|
|
1830
1834
|
if (waiting.async) await waiting.value;
|
|
1831
1835
|
}
|
|
1832
1836
|
|
|
1833
|
-
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, MAX_INDEX_TOUCH_INTERVAL_MS, MAX_LONG_QUERIES_IN_FLIGHT, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_CLOSE_AT_EXIT_MS, REQUEST_THREAD_DEPARTURE_DRAIN_MS, REQUEST_THREAD_MOVE_AFTER_MS, RUNNING_TASK_TTL_MS, SEARCH_CLOCK_SAMPLE_UNITS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearBodyBudgetSlot, clearGateSlot, closeEveryThreadAtExit, closeWhenTheProcessExits, confirmDeparture, corsHeaderReader, createBodyBudgetBuffer, createCloseAtExitWords, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedBodyBudget, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, isRemoveOnlyBatch, loadUWebSockets, parseJson, parseJsonOptional, parseRefusalOf, parseTaskListQuery, parseWriteJson, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondAnswer, respondError, respondJson, sendError, sendRelayed, sendStream, sendText, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport, withServerRerankWindow };
|
|
1837
|
+
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, MAX_INDEX_TOUCH_INTERVAL_MS, MAX_LONG_QUERIES_IN_FLIGHT, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_CLOSE_AT_EXIT_MS, REQUEST_THREAD_DEPARTURE_DRAIN_MS, REQUEST_THREAD_MOVE_AFTER_MS, RUNNING_TASK_TTL_MS, SEARCH_CLOCK_SAMPLE_UNITS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearBodyBudgetSlot, clearGateSlot, closeEveryThreadAtExit, closeWhenTheProcessExits, confirmDeparture, corsHeaderReader, createBodyBudgetBuffer, createCloseAtExitWords, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedBodyBudget, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, isRemoveOnlyBatch, loadUWebSockets, parseJson, parseJsonOptional, parseRefusalOf, parseTaskListQuery, parseWriteJson, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondAnswer, respondError, respondJson, sendError, sendRelayed, sendStream, sendText, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport, withServerRerankWindow };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH } from './chunk-DX2FI4LU.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
3
3
|
|
|
4
4
|
// src/search/hybrid-depth.ts
|
|
5
5
|
function fixedHybridDepthOf(hybrid) {
|
|
@@ -32,6 +32,7 @@ var ErrorCodes = {
|
|
|
32
32
|
SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
|
|
33
33
|
SEARCH_RESULT_WINDOW_EXCEEDED: "SEARCH_RESULT_WINDOW_EXCEEDED",
|
|
34
34
|
SEARCH_WORK_CAP_EXCEEDED: "SEARCH_WORK_CAP_EXCEEDED",
|
|
35
|
+
BM25_NOT_READY: "BM25_NOT_READY",
|
|
35
36
|
LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
|
|
36
37
|
ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
|
|
37
38
|
ENVELOPE_INVALID_MAGIC: "ENVELOPE_INVALID_MAGIC",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-
|
|
2
|
-
import { requireValidFieldTypes } from './chunk-
|
|
1
|
+
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-WUIDPBSY.mjs';
|
|
2
|
+
import { requireValidFieldTypes } from './chunk-AFOZR4ZQ.mjs';
|
|
3
3
|
import { VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
4
4
|
import { decode } from '@msgpack/msgpack';
|
|
5
5
|
|