@delali/narsil 0.2.4-next.55 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -28
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-DlYw0Ov0.d.ts → capabilities-DSsOGz6F.d.ts} +2 -2
- package/dist/{chunk-BH5SYB5W.mjs → chunk-24RXHMEL.mjs} +604 -150
- package/dist/{chunk-D2S642IV.mjs → chunk-3LHGNRYE.mjs} +1 -1
- package/dist/{chunk-TVT5V4E7.mjs → chunk-B2TFROFX.mjs} +299 -168
- package/dist/{chunk-L225SHNB.mjs → chunk-L6TNPLAR.mjs} +19 -5
- package/dist/{chunk-U2AL3SON.mjs → chunk-THYRHHO2.mjs} +3 -3
- package/dist/{chunk-NTZYA3SJ.mjs → chunk-U7KYW4BA.mjs} +3 -3
- package/dist/{chunk-FRQVMFY7.mjs → chunk-USKFU7J5.mjs} +1 -0
- package/dist/{chunk-4NVPQ6S5.mjs → chunk-UXNWQZST.mjs} +12 -3
- package/dist/{chunk-RGS7FMCB.mjs → chunk-VMRR3DQU.mjs} +74 -16
- package/dist/{chunk-2GWVQWGA.mjs → chunk-YMZCMH5M.mjs} +45 -1
- package/dist/{chunk-UTOFOJOR.mjs → chunk-YNVIUGF5.mjs} +6 -2
- package/dist/{chunk-5T6AOR37.mjs → chunk-ZNZBXMP5.mjs} +601 -60
- package/dist/{chunk-XNQAIO74.mjs → chunk-ZZT3EAUC.mjs} +177 -309
- package/dist/{client-CQUqt9Ir.d.ts → client-BQK6Tf6A.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +1 -0
- package/dist/{config-CNas6wJH.d.ts → config-BhldZgaK.d.ts} +2 -2
- package/dist/{config-bXxnJrQc.d.ts → config-DmXULPa-.d.ts} +15 -7
- package/dist/direct-executor-ALAVE7FC.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +382 -162
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-Dj4I91Jv.d.ts → engine-CIOyFozK.d.ts} +15 -12
- package/dist/index.browser.mjs +1565 -528
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +11 -11
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-xrXOZMbw.d.ts → memory-yFaVn2xZ.d.ts} +199 -107
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +11 -11
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/{schema-mLpXZBEh.d.ts → schema-d5Afu03A.d.ts} +7 -2
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +10 -10
- package/dist/{tasks-Df73Tjgq.d.ts → tasks-CH2Dod8w.d.ts} +1 -1
- package/dist/{thread-5UP6MGED.mjs → thread-DYHEQRVY.mjs} +14 -10
- package/dist/{types-BTIVO8DI.d.ts → types-DAK7mnUZ.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +4 -4
- package/package.json +9 -9
- package/dist/direct-executor-RBA5O7EN.mjs +0 -12
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-
|
|
2
|
-
import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn } from './chunk-
|
|
3
|
-
import { flattenSchema } from './chunk-
|
|
1
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-3LHGNRYE.mjs';
|
|
2
|
+
import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, requireValidFilter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex, fuzzyTermMatches } from './chunk-ZNZBXMP5.mjs';
|
|
3
|
+
import { DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isTextFieldType, isGeopointOrVectorType } from './chunk-L6TNPLAR.mjs';
|
|
4
4
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
5
|
-
import { compareCodePoints, compareComparableKeys } from './chunk-
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
5
|
+
import { compareCodePoints, compareComparableKeys } from './chunk-YMZCMH5M.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-USKFU7J5.mjs';
|
|
7
7
|
|
|
8
8
|
// src/search/cursor-binding.ts
|
|
9
9
|
var ABSENT_TAG = 0;
|
|
@@ -163,6 +163,169 @@ function listBindingOf(filters) {
|
|
|
163
163
|
return bindingOf(values);
|
|
164
164
|
}
|
|
165
165
|
|
|
166
|
+
// src/search/query-fields.ts
|
|
167
|
+
function requireBoostableFields(boost, flatSchema) {
|
|
168
|
+
for (const [field, weight] of Object.entries(boost)) {
|
|
169
|
+
const fieldType = flatSchema[field];
|
|
170
|
+
if (fieldType === void 0 || !isTextFieldType(fieldType) && fieldType !== "string[]") {
|
|
171
|
+
throw new NarsilError(
|
|
172
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
173
|
+
fieldType === void 0 ? `The engine boosts a text field that the schema declares, and the schema declares no field "${field}"` : `The engine boosts a text field alone, and the schema declares field "${field}" as "${fieldType}"`,
|
|
174
|
+
{ field, ...fieldType === void 0 ? {} : { fieldType } }
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
if (typeof weight !== "number" || !Number.isFinite(weight)) {
|
|
178
|
+
throw new NarsilError(
|
|
179
|
+
ErrorCodes.CONFIG_INVALID,
|
|
180
|
+
`The boost on field "${field}" must be a finite number, and this query sets ${String(weight)}`,
|
|
181
|
+
{ field, boost: String(weight) }
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
function requireFacetableFields(facets, flatSchema, strict) {
|
|
187
|
+
for (const field of Object.keys(facets)) {
|
|
188
|
+
const fieldType = flatSchema[field];
|
|
189
|
+
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(field, "facet");
|
|
190
|
+
if (isGeopointOrVectorType(fieldType)) {
|
|
191
|
+
throw new NarsilError(
|
|
192
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
193
|
+
`The engine counts the values of a text, number, boolean, or enum field, and the schema declares field "${field}" as "${fieldType}"`,
|
|
194
|
+
{ field, fieldType }
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
function requireDeclaredGroupFields(fields, flatSchema) {
|
|
200
|
+
for (const field of fields) {
|
|
201
|
+
if (flatSchema[field] === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
function requireUsableQueryFields(params, schema, strict = false) {
|
|
205
|
+
const checksGroup = strict && Array.isArray(params.group?.fields);
|
|
206
|
+
if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
|
|
207
|
+
const flatSchema = flattenSchema(schema);
|
|
208
|
+
if (params.boost !== void 0) requireBoostableFields(params.boost, flatSchema);
|
|
209
|
+
if (params.facets !== void 0) requireFacetableFields(params.facets, flatSchema, strict);
|
|
210
|
+
if (checksGroup && params.group !== void 0) requireDeclaredGroupFields(params.group.fields, flatSchema);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// src/engine/query/shared.ts
|
|
214
|
+
function partitionsFor(manager, partitionIds) {
|
|
215
|
+
return partitionsIn(manager, partitionIds);
|
|
216
|
+
}
|
|
217
|
+
function coverageFor(manager, partitionIds) {
|
|
218
|
+
if (partitionIds === void 0) {
|
|
219
|
+
return {
|
|
220
|
+
totalPartitions: manager.partitionCount,
|
|
221
|
+
queriedPartitions: manager.partitionCount,
|
|
222
|
+
timedOutPartitions: 0,
|
|
223
|
+
failedPartitions: 0
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
const queriedPartitions = partitionsFor(manager, partitionIds).length;
|
|
227
|
+
return {
|
|
228
|
+
totalPartitions: partitionIds.length,
|
|
229
|
+
queriedPartitions,
|
|
230
|
+
timedOutPartitions: 0,
|
|
231
|
+
failedPartitions: partitionIds.length - queriedPartitions
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
function partitionsForVectorSearch(manager, vecIndex, partitionIds) {
|
|
235
|
+
if (partitionIds === void 0) {
|
|
236
|
+
return void 0;
|
|
237
|
+
}
|
|
238
|
+
if (!vecIndex.partitionsKnown()) {
|
|
239
|
+
vecIndex.assignPartitions((docId) => manager.partitionIdOf(docId));
|
|
240
|
+
}
|
|
241
|
+
return new Set(partitionIds);
|
|
242
|
+
}
|
|
243
|
+
function scoringConfigFor(params, context) {
|
|
244
|
+
const scoringMode = params.scoring ?? context.config.defaultScoring ?? "local";
|
|
245
|
+
if (scoringMode !== "broadcast") {
|
|
246
|
+
return { scoringMode, partitionIds: context.partitionIds };
|
|
247
|
+
}
|
|
248
|
+
return { scoringMode, globalStats: context.broadcastStats?.(context.indexName), partitionIds: context.partitionIds };
|
|
249
|
+
}
|
|
250
|
+
function broadcastStatsForWorker(params, context, scoring) {
|
|
251
|
+
if (scoring.scoringMode !== "broadcast" || scoring.globalStats === void 0) {
|
|
252
|
+
return void 0;
|
|
253
|
+
}
|
|
254
|
+
const term = params.term;
|
|
255
|
+
if (term === void 0 || term.trim().length === 0) {
|
|
256
|
+
return void 0;
|
|
257
|
+
}
|
|
258
|
+
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
|
|
259
|
+
}
|
|
260
|
+
function searchOptionsFor(manager) {
|
|
261
|
+
return {
|
|
262
|
+
bm25Params: manager.config.bm25,
|
|
263
|
+
stopWords: manager.analysis.stopWords,
|
|
264
|
+
customTokenizer: manager.analysis.customTokenizer
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
function collectFilterDocIds(manager, params, schema, partitionIds) {
|
|
268
|
+
const filterDocIds = /* @__PURE__ */ new Set();
|
|
269
|
+
if (!params.filters) return filterDocIds;
|
|
270
|
+
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
271
|
+
const partitionFiltered = partition.applyFilters(params.filters, schema);
|
|
272
|
+
for (const docId of partitionFiltered) {
|
|
273
|
+
filterDocIds.add(docId);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
return filterDocIds;
|
|
277
|
+
}
|
|
278
|
+
function vectorResultsToScored(results) {
|
|
279
|
+
return results.map((r) => ({
|
|
280
|
+
docId: r.docId,
|
|
281
|
+
score: r.score,
|
|
282
|
+
termFrequencies: {},
|
|
283
|
+
fieldLengths: {},
|
|
284
|
+
idf: {}
|
|
285
|
+
}));
|
|
286
|
+
}
|
|
287
|
+
function vectorSearchersOf(context) {
|
|
288
|
+
return context.vectorSearchers ?? context.manager.getVectorIndexes();
|
|
289
|
+
}
|
|
290
|
+
function resolveVectorIndex(context, fieldName) {
|
|
291
|
+
return vectorSearchersOf(context).get(fieldName);
|
|
292
|
+
}
|
|
293
|
+
var SEARCH_MODES = /* @__PURE__ */ new Set(["fulltext", "vector", "hybrid"]);
|
|
294
|
+
function requireKnownMode(params) {
|
|
295
|
+
if (params.mode === void 0 || SEARCH_MODES.has(params.mode)) return;
|
|
296
|
+
throw new NarsilError(
|
|
297
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
298
|
+
`A search takes the mode "fulltext", "vector", or "hybrid", while this query names "${String(params.mode)}"`,
|
|
299
|
+
{ mode: String(params.mode), modes: [...SEARCH_MODES] }
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
function requireValidQueryOptions(params, config) {
|
|
303
|
+
const strict = config.strict === true;
|
|
304
|
+
requireSortableFields(params.sort, config.schema, strict);
|
|
305
|
+
requireValidQueryFilter(params.filters, config.schema, strict);
|
|
306
|
+
requireUsableQueryFields(params, config.schema, strict);
|
|
307
|
+
}
|
|
308
|
+
function requireValidQueryFilter(filters, schema, strict = false) {
|
|
309
|
+
if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema), strict);
|
|
310
|
+
}
|
|
311
|
+
function requireVectorSearchable(params, context, needsVector) {
|
|
312
|
+
if (!needsVector) return;
|
|
313
|
+
const field = params.vector?.field;
|
|
314
|
+
if (field === void 0) {
|
|
315
|
+
throw new NarsilError(
|
|
316
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
317
|
+
`A "${params.mode}" query needs a vector clause naming the field to search`,
|
|
318
|
+
{ mode: params.mode }
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
if (vectorSearchersOf(context).has(field)) return;
|
|
322
|
+
throw new NarsilError(
|
|
323
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
324
|
+
`Field "${field}" has no vector index, so a vector search cannot use it`,
|
|
325
|
+
{ field, vectorFields: [...vectorSearchersOf(context).keys()] }
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
|
|
166
329
|
// src/engine/validation.ts
|
|
167
330
|
function now() {
|
|
168
331
|
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
@@ -351,6 +514,7 @@ function pageInSortOrder(partitions, cursor, filtered, limit, sort, signature, s
|
|
|
351
514
|
fields,
|
|
352
515
|
directions,
|
|
353
516
|
fieldTypes,
|
|
517
|
+
modes: sortModesOf(normalized),
|
|
354
518
|
limit: limit + 1,
|
|
355
519
|
anchorKey: cursor === null ? null : cursor.sortKey,
|
|
356
520
|
anchorId: cursor === null ? null : cursor.anchor
|
|
@@ -382,7 +546,8 @@ function executeListDocuments(params, context) {
|
|
|
382
546
|
const startTime = now();
|
|
383
547
|
const limit = clampListLimit(params.limit);
|
|
384
548
|
const signature = sortSignatureOf(params.sort);
|
|
385
|
-
requireSortableFields(params.sort, schema);
|
|
549
|
+
requireSortableFields(params.sort, schema, context.strict === true);
|
|
550
|
+
requireValidQueryFilter(params.filters, schema, context.strict === true);
|
|
386
551
|
const binding = listBindingOf(params.filters);
|
|
387
552
|
let cursor = null;
|
|
388
553
|
if (params.cursor !== void 0) {
|
|
@@ -430,6 +595,10 @@ function getNestedValue(obj, path) {
|
|
|
430
595
|
}
|
|
431
596
|
return current;
|
|
432
597
|
}
|
|
598
|
+
function hitsKeptPerGroup(maxPerGroup) {
|
|
599
|
+
if (maxPerGroup === void 0 || !Number.isFinite(maxPerGroup) || maxPerGroup < 1) return DEFAULT_MAX_PER_GROUP;
|
|
600
|
+
return Math.floor(maxPerGroup);
|
|
601
|
+
}
|
|
433
602
|
function applyGrouping(hits, group, getDocument) {
|
|
434
603
|
if (!group.fields || group.fields.length === 0) {
|
|
435
604
|
return [{ values: {}, hits }];
|
|
@@ -453,16 +622,14 @@ function applyGrouping(hits, group, getDocument) {
|
|
|
453
622
|
}
|
|
454
623
|
}
|
|
455
624
|
const groups = [];
|
|
625
|
+
const keptPerGroup = hitsKeptPerGroup(group.maxPerGroup);
|
|
456
626
|
for (const entry of groupMap.values()) {
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
groupHits = groupHits.slice(0, group.maxPerGroup);
|
|
460
|
-
}
|
|
461
|
-
const result = { values: entry.values, hits: groupHits };
|
|
627
|
+
const everyHit = entry.hits;
|
|
628
|
+
const result = { values: entry.values, hits: everyHit.slice(0, keptPerGroup) };
|
|
462
629
|
if (group.reduce) {
|
|
463
630
|
const folded = foldGroupReducer(
|
|
464
631
|
group.reduce,
|
|
465
|
-
|
|
632
|
+
everyHit.map((hit) => ({ document: getDocument(hit.id), score: hit.score ?? 0 }))
|
|
466
633
|
);
|
|
467
634
|
if (folded.reducerError !== void 0) {
|
|
468
635
|
result.reducerError = folded.reducerError;
|
|
@@ -513,23 +680,27 @@ function dedupePinnedEntries(pinned) {
|
|
|
513
680
|
}
|
|
514
681
|
return deduped;
|
|
515
682
|
}
|
|
516
|
-
function
|
|
683
|
+
function placePinned(hits, pinned, resolveDoc) {
|
|
517
684
|
const result = hits.slice();
|
|
518
685
|
const deduped = dedupePinnedEntries(pinned);
|
|
519
686
|
const pinnedDocIds = new Set(deduped.map((entry) => entry.docId));
|
|
687
|
+
const listedPins = /* @__PURE__ */ new Set();
|
|
520
688
|
for (let i = result.length - 1; i >= 0; i--) {
|
|
521
689
|
if (pinnedDocIds.has(result[i].id)) {
|
|
690
|
+
listedPins.add(result[i].id);
|
|
522
691
|
result.splice(i, 1);
|
|
523
692
|
}
|
|
524
693
|
}
|
|
525
694
|
const sorted = deduped.sort((a, b) => a.position - b.position);
|
|
695
|
+
let placedFromOutside = 0;
|
|
526
696
|
for (const entry of sorted) {
|
|
527
697
|
const doc = resolveDoc(entry.docId);
|
|
528
698
|
if (!doc) continue;
|
|
699
|
+
if (!listedPins.has(entry.docId)) placedFromOutside++;
|
|
529
700
|
const pos = Math.max(0, Math.min(entry.position, result.length));
|
|
530
701
|
result.splice(pos, 0, doc);
|
|
531
702
|
}
|
|
532
|
-
return result;
|
|
703
|
+
return { hits: result, placedFromOutside };
|
|
533
704
|
}
|
|
534
705
|
|
|
535
706
|
// src/highlighting/highlighter.ts
|
|
@@ -627,19 +798,25 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
627
798
|
});
|
|
628
799
|
const matchedRanges = [];
|
|
629
800
|
const prefixToken = options?.prefixToken;
|
|
801
|
+
const tolerance = options?.tolerance ?? 0;
|
|
802
|
+
const prefixLength = options?.prefixLength ?? 0;
|
|
803
|
+
const matchByFieldToken = /* @__PURE__ */ new Map();
|
|
804
|
+
const matchesQuery = (fieldToken) => {
|
|
805
|
+
const known = matchByFieldToken.get(fieldToken);
|
|
806
|
+
if (known !== void 0) return known;
|
|
807
|
+
const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
|
|
808
|
+
const matches = stemmedQueryTokens.some(
|
|
809
|
+
(stemmedQuery) => fuzzyTermMatches(stemmedQuery, stemmedField, tolerance, prefixLength)
|
|
810
|
+
);
|
|
811
|
+
matchByFieldToken.set(fieldToken, matches);
|
|
812
|
+
return matches;
|
|
813
|
+
};
|
|
630
814
|
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
631
815
|
if (i >= charOffsets.length) break;
|
|
632
816
|
const fieldToken = fieldResult.tokens[i].token;
|
|
633
|
-
const
|
|
634
|
-
if (
|
|
817
|
+
const prefixMatch = prefixToken !== void 0 && fieldToken.startsWith(prefixToken);
|
|
818
|
+
if (prefixMatch || matchesQuery(fieldToken)) {
|
|
635
819
|
matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
|
|
636
|
-
continue;
|
|
637
|
-
}
|
|
638
|
-
for (const stemmedQuery of stemmedQueryTokens) {
|
|
639
|
-
if (stemmedField === stemmedQuery) {
|
|
640
|
-
matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
|
|
641
|
-
break;
|
|
642
|
-
}
|
|
643
820
|
}
|
|
644
821
|
}
|
|
645
822
|
const mergedPositions = mergeOverlapping(matchedRanges);
|
|
@@ -709,7 +886,9 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
709
886
|
preTag: params.highlight.preTag,
|
|
710
887
|
postTag: params.highlight.postTag,
|
|
711
888
|
maxSnippetLength: params.highlight.maxSnippetLength,
|
|
712
|
-
prefixToken
|
|
889
|
+
prefixToken,
|
|
890
|
+
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
891
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
713
892
|
});
|
|
714
893
|
}
|
|
715
894
|
}
|
|
@@ -717,120 +896,6 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
717
896
|
}
|
|
718
897
|
}
|
|
719
898
|
|
|
720
|
-
// src/engine/query/shared.ts
|
|
721
|
-
function partitionsFor(manager, partitionIds) {
|
|
722
|
-
return partitionsIn(manager, partitionIds);
|
|
723
|
-
}
|
|
724
|
-
function coverageFor(manager, partitionIds) {
|
|
725
|
-
if (partitionIds === void 0) {
|
|
726
|
-
return {
|
|
727
|
-
totalPartitions: manager.partitionCount,
|
|
728
|
-
queriedPartitions: manager.partitionCount,
|
|
729
|
-
timedOutPartitions: 0,
|
|
730
|
-
failedPartitions: 0
|
|
731
|
-
};
|
|
732
|
-
}
|
|
733
|
-
const queriedPartitions = partitionsFor(manager, partitionIds).length;
|
|
734
|
-
return {
|
|
735
|
-
totalPartitions: partitionIds.length,
|
|
736
|
-
queriedPartitions,
|
|
737
|
-
timedOutPartitions: 0,
|
|
738
|
-
failedPartitions: partitionIds.length - queriedPartitions
|
|
739
|
-
};
|
|
740
|
-
}
|
|
741
|
-
function partitionsForVectorSearch(manager, vecIndex, partitionIds) {
|
|
742
|
-
if (partitionIds === void 0) {
|
|
743
|
-
return void 0;
|
|
744
|
-
}
|
|
745
|
-
if (!vecIndex.partitionsKnown()) {
|
|
746
|
-
vecIndex.assignPartitions((docId) => manager.partitionIdOf(docId));
|
|
747
|
-
}
|
|
748
|
-
return new Set(partitionIds);
|
|
749
|
-
}
|
|
750
|
-
function scoringConfigFor(params, context) {
|
|
751
|
-
const scoringMode = params.scoring ?? context.config.defaultScoring ?? "local";
|
|
752
|
-
if (scoringMode !== "broadcast") {
|
|
753
|
-
return { scoringMode, partitionIds: context.partitionIds };
|
|
754
|
-
}
|
|
755
|
-
return { scoringMode, globalStats: context.broadcastStats?.(context.indexName), partitionIds: context.partitionIds };
|
|
756
|
-
}
|
|
757
|
-
function broadcastStatsForWorker(params, context, scoring) {
|
|
758
|
-
if (scoring.scoringMode !== "broadcast" || scoring.globalStats === void 0) {
|
|
759
|
-
return void 0;
|
|
760
|
-
}
|
|
761
|
-
const term = params.term;
|
|
762
|
-
if (term === void 0 || term.trim().length === 0) {
|
|
763
|
-
return void 0;
|
|
764
|
-
}
|
|
765
|
-
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
|
|
766
|
-
}
|
|
767
|
-
function searchOptionsFor(manager) {
|
|
768
|
-
return {
|
|
769
|
-
bm25Params: manager.config.bm25,
|
|
770
|
-
stopWords: manager.analysis.stopWords,
|
|
771
|
-
customTokenizer: manager.analysis.customTokenizer
|
|
772
|
-
};
|
|
773
|
-
}
|
|
774
|
-
function collectFilterDocIds(manager, params, schema, partitionIds) {
|
|
775
|
-
const filterDocIds = /* @__PURE__ */ new Set();
|
|
776
|
-
if (!params.filters) return filterDocIds;
|
|
777
|
-
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
778
|
-
const partitionFiltered = partition.applyFilters(params.filters, schema);
|
|
779
|
-
for (const docId of partitionFiltered) {
|
|
780
|
-
filterDocIds.add(docId);
|
|
781
|
-
}
|
|
782
|
-
}
|
|
783
|
-
return filterDocIds;
|
|
784
|
-
}
|
|
785
|
-
function vectorResultsToScored(results) {
|
|
786
|
-
return results.map((r) => ({
|
|
787
|
-
docId: r.docId,
|
|
788
|
-
score: r.score,
|
|
789
|
-
termFrequencies: {},
|
|
790
|
-
fieldLengths: {},
|
|
791
|
-
idf: {}
|
|
792
|
-
}));
|
|
793
|
-
}
|
|
794
|
-
function vectorSearchersOf(context) {
|
|
795
|
-
return context.vectorSearchers ?? context.manager.getVectorIndexes();
|
|
796
|
-
}
|
|
797
|
-
function resolveVectorIndex(context, fieldName) {
|
|
798
|
-
return vectorSearchersOf(context).get(fieldName);
|
|
799
|
-
}
|
|
800
|
-
var SEARCH_MODES = /* @__PURE__ */ new Set(["fulltext", "vector", "hybrid"]);
|
|
801
|
-
function requireKnownMode(params) {
|
|
802
|
-
if (params.mode === void 0 || SEARCH_MODES.has(params.mode)) return;
|
|
803
|
-
throw new NarsilError(
|
|
804
|
-
ErrorCodes.SEARCH_INVALID_MODE,
|
|
805
|
-
`A search takes the mode "fulltext", "vector", or "hybrid", while this query names "${String(params.mode)}"`,
|
|
806
|
-
{ mode: String(params.mode), modes: [...SEARCH_MODES] }
|
|
807
|
-
);
|
|
808
|
-
}
|
|
809
|
-
function requireVectorSearchable(params, context, needsVector) {
|
|
810
|
-
if (!needsVector) return;
|
|
811
|
-
const field = params.vector?.field;
|
|
812
|
-
if (field === void 0) {
|
|
813
|
-
throw new NarsilError(
|
|
814
|
-
ErrorCodes.SEARCH_INVALID_MODE,
|
|
815
|
-
`A "${params.mode}" query needs a vector clause naming the field to search`,
|
|
816
|
-
{ mode: params.mode }
|
|
817
|
-
);
|
|
818
|
-
}
|
|
819
|
-
if (vectorSearchersOf(context).has(field)) return;
|
|
820
|
-
throw new NarsilError(
|
|
821
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
822
|
-
`Field "${field}" has no vector index, so a vector search cannot use it`,
|
|
823
|
-
{ field, vectorFields: [...vectorSearchersOf(context).keys()] }
|
|
824
|
-
);
|
|
825
|
-
}
|
|
826
|
-
function clampAlpha(alpha) {
|
|
827
|
-
if (alpha === void 0) return 0.5;
|
|
828
|
-
if (!Number.isFinite(alpha)) return 0.5;
|
|
829
|
-
if (alpha < 0) return 0;
|
|
830
|
-
if (alpha > 1) return 1;
|
|
831
|
-
return alpha;
|
|
832
|
-
}
|
|
833
|
-
|
|
834
899
|
// src/engine/query/sorted.ts
|
|
835
900
|
function sortsWithoutScores(params) {
|
|
836
901
|
return params.includeScores !== true && params.includeScoreComponents !== true && params.minScore === void 0 && params.group === void 0 && params.pinned === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
|
|
@@ -851,19 +916,21 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
851
916
|
const directions = normalized.map((entry) => entry.direction);
|
|
852
917
|
const flatSchema = flattenSchema(schema);
|
|
853
918
|
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
919
|
+
const modes = sortModesOf(normalized);
|
|
854
920
|
const options = searchOptionsFor(manager);
|
|
855
921
|
const partitions = partitionsFor(manager, context.partitionIds);
|
|
856
922
|
const partitionLimit = offset + limit + 1;
|
|
857
923
|
const merged = [];
|
|
858
924
|
const partitionFacets = [];
|
|
925
|
+
const partitionFacetConfig = params.facets !== void 0 && partitions.length > 1 ? everyValueFacetConfig(params.facets) : params.facets;
|
|
859
926
|
let count = 0;
|
|
860
927
|
for (const partition of partitions) {
|
|
861
928
|
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
862
|
-
if (
|
|
929
|
+
if (partitionFacetConfig !== void 0) {
|
|
863
930
|
partitionFacets.push(
|
|
864
931
|
partition.computeFacets(
|
|
865
932
|
matches === null ? /* @__PURE__ */ new Set() : { ordinalBitset: matches.ordinalBitset() },
|
|
866
|
-
|
|
933
|
+
partitionFacetConfig,
|
|
867
934
|
schema
|
|
868
935
|
)
|
|
869
936
|
);
|
|
@@ -874,6 +941,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
874
941
|
fields,
|
|
875
942
|
directions,
|
|
876
943
|
fieldTypes,
|
|
944
|
+
modes,
|
|
877
945
|
limit: partitionLimit,
|
|
878
946
|
anchorKey,
|
|
879
947
|
anchorId,
|
|
@@ -899,13 +967,42 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
899
967
|
hits: page.map((entry) => ({ id: entry.id, document: void 0 })),
|
|
900
968
|
count,
|
|
901
969
|
cursor,
|
|
902
|
-
facets: partitionFacets.length > 0 ? mergeFacets(partitionFacets) : void 0
|
|
970
|
+
facets: params.facets !== void 0 && partitionFacets.length > 0 ? mergeFacets(partitionFacets, params.facets) : void 0
|
|
903
971
|
};
|
|
904
972
|
}
|
|
905
973
|
|
|
906
974
|
// src/search/fusion.ts
|
|
975
|
+
function rankConstantOf(k) {
|
|
976
|
+
if (k === void 0 || k === null) return DEFAULT_RANK_CONSTANT;
|
|
977
|
+
if (!Number.isInteger(k) || k < MIN_RANK_CONSTANT) {
|
|
978
|
+
throw new NarsilError(
|
|
979
|
+
ErrorCodes.CONFIG_INVALID,
|
|
980
|
+
`hybrid.k must be a whole number of at least ${MIN_RANK_CONSTANT}`,
|
|
981
|
+
{ k }
|
|
982
|
+
);
|
|
983
|
+
}
|
|
984
|
+
return k;
|
|
985
|
+
}
|
|
986
|
+
function linearAlphaOf(alpha) {
|
|
987
|
+
if (alpha === void 0 || alpha === null) return DEFAULT_LINEAR_ALPHA;
|
|
988
|
+
if (!Number.isFinite(alpha) || alpha < MIN_LINEAR_ALPHA || alpha > MAX_LINEAR_ALPHA) {
|
|
989
|
+
throw new NarsilError(
|
|
990
|
+
ErrorCodes.CONFIG_INVALID,
|
|
991
|
+
`hybrid.alpha must be a number from ${MIN_LINEAR_ALPHA} to ${MAX_LINEAR_ALPHA}`,
|
|
992
|
+
{ alpha }
|
|
993
|
+
);
|
|
994
|
+
}
|
|
995
|
+
return alpha;
|
|
996
|
+
}
|
|
997
|
+
function resolveHybridFusion(hybrid) {
|
|
998
|
+
const strategy = hybrid?.strategy ?? "rrf";
|
|
999
|
+
if (strategy !== "rrf" && strategy !== "linear") {
|
|
1000
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `hybrid.strategy must be "rrf" or "linear"`, { strategy });
|
|
1001
|
+
}
|
|
1002
|
+
return { strategy, k: rankConstantOf(hybrid?.k), alpha: linearAlphaOf(hybrid?.alpha) };
|
|
1003
|
+
}
|
|
907
1004
|
function reciprocalRankFusion(lists, options) {
|
|
908
|
-
const k = options.k
|
|
1005
|
+
const k = options.k;
|
|
909
1006
|
const scores = /* @__PURE__ */ new Map();
|
|
910
1007
|
const docData = /* @__PURE__ */ new Map();
|
|
911
1008
|
for (const list of lists) {
|
|
@@ -997,7 +1094,18 @@ function oversampleOf(vectorConfig) {
|
|
|
997
1094
|
}
|
|
998
1095
|
return oversample;
|
|
999
1096
|
}
|
|
1000
|
-
|
|
1097
|
+
function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
1098
|
+
const reach = cursorDepth + offset + limit;
|
|
1099
|
+
if (reach > RESULT_WINDOW) {
|
|
1100
|
+
throw new NarsilError(
|
|
1101
|
+
ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
|
|
1102
|
+
`The engine pages a vector search through the first ${RESULT_WINDOW} results, and this page ends at ${reach}. Narrow the search with a filter or a similarity floor`,
|
|
1103
|
+
{ limit, offset, cursorDepth, window: RESULT_WINDOW }
|
|
1104
|
+
);
|
|
1105
|
+
}
|
|
1106
|
+
return Math.min(reach + 1, RESULT_WINDOW);
|
|
1107
|
+
}
|
|
1108
|
+
async function executeVectorSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1001
1109
|
const { manager, config, partitionIds } = context;
|
|
1002
1110
|
const vectorConfig = params.vector;
|
|
1003
1111
|
if (!vectorConfig || !vectorConfig.value) {
|
|
@@ -1018,8 +1126,8 @@ async function executeVectorSearch(params, context, limit, offset) {
|
|
|
1018
1126
|
filterPartitions = partitionsForVectorSearch(manager, vecIndex, partitionIds);
|
|
1019
1127
|
}
|
|
1020
1128
|
const queryVec = new Float32Array(vectorConfig.value);
|
|
1021
|
-
const k = limit
|
|
1022
|
-
const
|
|
1129
|
+
const k = vectorFetchDepth(limit, offset, cursorDepth);
|
|
1130
|
+
const outcome = await vecIndex.searchParallel(queryVec, k, {
|
|
1023
1131
|
metric: vectorConfig.metric ?? "cosine",
|
|
1024
1132
|
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1025
1133
|
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
@@ -1027,8 +1135,8 @@ async function executeVectorSearch(params, context, limit, offset) {
|
|
|
1027
1135
|
efSearch: vectorConfig.efSearch,
|
|
1028
1136
|
oversample: oversampleOf(vectorConfig)
|
|
1029
1137
|
});
|
|
1030
|
-
const scored = vectorResultsToScored(results);
|
|
1031
|
-
return { scored, totalMatched:
|
|
1138
|
+
const scored = vectorResultsToScored(outcome.results);
|
|
1139
|
+
return { scored, totalMatched: outcome.matched, matchedExact: outcome.matchedExact };
|
|
1032
1140
|
}
|
|
1033
1141
|
async function textLeg(textOnlyParams, context) {
|
|
1034
1142
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
@@ -1047,9 +1155,9 @@ async function textLeg(textOnlyParams, context) {
|
|
|
1047
1155
|
async function vectorLeg(vectorConfig, queryVector, context, filterDocIds, k) {
|
|
1048
1156
|
const { manager } = context;
|
|
1049
1157
|
const vecIndex = resolveVectorIndex(context, vectorConfig.field);
|
|
1050
|
-
if (!vecIndex) return [];
|
|
1158
|
+
if (!vecIndex) return { scored: [], holdsEveryMatch: true };
|
|
1051
1159
|
const filterPartitions = filterDocIds === void 0 ? partitionsForVectorSearch(manager, vecIndex, context.partitionIds) : void 0;
|
|
1052
|
-
const
|
|
1160
|
+
const outcome = await vecIndex.searchParallel(queryVector, k, {
|
|
1053
1161
|
metric: vectorConfig.metric ?? "cosine",
|
|
1054
1162
|
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1055
1163
|
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
@@ -1057,11 +1165,15 @@ async function vectorLeg(vectorConfig, queryVector, context, filterDocIds, k) {
|
|
|
1057
1165
|
efSearch: vectorConfig.efSearch,
|
|
1058
1166
|
oversample: oversampleOf(vectorConfig)
|
|
1059
1167
|
});
|
|
1060
|
-
return
|
|
1168
|
+
return {
|
|
1169
|
+
scored: vectorResultsToScored(outcome.results),
|
|
1170
|
+
holdsEveryMatch: outcome.matchedExact && outcome.results.length === outcome.matched
|
|
1171
|
+
};
|
|
1061
1172
|
}
|
|
1062
|
-
async function executeHybridSearch(params, context, limit, offset) {
|
|
1173
|
+
async function executeHybridSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1063
1174
|
const { manager, config } = context;
|
|
1064
1175
|
const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
|
|
1176
|
+
const fusion = resolveHybridFusion(params.hybrid);
|
|
1065
1177
|
let filterDocIds;
|
|
1066
1178
|
if (params.filters) {
|
|
1067
1179
|
filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds);
|
|
@@ -1078,31 +1190,36 @@ async function executeHybridSearch(params, context, limit, offset) {
|
|
|
1078
1190
|
new Float32Array(vectorConfig.value),
|
|
1079
1191
|
context,
|
|
1080
1192
|
filterDocIds,
|
|
1081
|
-
limit
|
|
1193
|
+
vectorFetchDepth(limit, offset, cursorDepth)
|
|
1082
1194
|
);
|
|
1083
|
-
const [textFanOutResult,
|
|
1084
|
-
const
|
|
1085
|
-
const strategy = hybridConfig.strategy ?? "rrf";
|
|
1195
|
+
const [textFanOutResult, vectorOutcome] = await Promise.all([textLeg(textOnlyParams, context), vectorPending]);
|
|
1196
|
+
const vectorScored = vectorOutcome.scored;
|
|
1086
1197
|
let fusedScored;
|
|
1087
|
-
if (strategy === "rrf") {
|
|
1088
|
-
|
|
1089
|
-
fusedScored = reciprocalRankFusion([textFanOutResult.scored, vectorScored], { k: rrfK });
|
|
1198
|
+
if (fusion.strategy === "rrf") {
|
|
1199
|
+
fusedScored = reciprocalRankFusion([textFanOutResult.scored, vectorScored], { k: fusion.k });
|
|
1090
1200
|
} else {
|
|
1091
|
-
|
|
1092
|
-
fusedScored = linearCombination(textFanOutResult.scored, vectorScored, { alpha });
|
|
1201
|
+
fusedScored = linearCombination(textFanOutResult.scored, vectorScored, { alpha: fusion.alpha });
|
|
1093
1202
|
}
|
|
1094
1203
|
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
1095
1204
|
const threshold = params.minScore;
|
|
1096
1205
|
fusedScored = fusedScored.filter((doc) => doc.score >= threshold);
|
|
1097
1206
|
}
|
|
1207
|
+
const textHoldsEveryMatch = textFanOutResult.scored.length === textFanOutResult.totalMatched;
|
|
1098
1208
|
return {
|
|
1099
1209
|
scored: fusedScored,
|
|
1100
1210
|
totalMatched: fusedScored.length,
|
|
1211
|
+
matchedExact: vectorOutcome.holdsEveryMatch && textHoldsEveryMatch,
|
|
1101
1212
|
facets: textFanOutResult.facets
|
|
1102
1213
|
};
|
|
1103
1214
|
}
|
|
1104
1215
|
|
|
1105
1216
|
// src/engine/query/index.ts
|
|
1217
|
+
function requireCursorDepth(cursor, signature, binding) {
|
|
1218
|
+
if (cursor === void 0) return 0;
|
|
1219
|
+
const decoded = decodePageCursor(cursor);
|
|
1220
|
+
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
1221
|
+
return decoded.depth ?? 0;
|
|
1222
|
+
}
|
|
1106
1223
|
async function executeQuery(params, context) {
|
|
1107
1224
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
1108
1225
|
const startTime = now();
|
|
@@ -1123,11 +1240,12 @@ async function executeQuery(params, context) {
|
|
|
1123
1240
|
{ sort: sortFields.map((entry) => entry.field) }
|
|
1124
1241
|
);
|
|
1125
1242
|
}
|
|
1126
|
-
|
|
1243
|
+
requireValidQueryOptions(params, config);
|
|
1127
1244
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
1128
1245
|
let paginated;
|
|
1129
1246
|
let nextCursor;
|
|
1130
1247
|
let count;
|
|
1248
|
+
let countExact = true;
|
|
1131
1249
|
let facets;
|
|
1132
1250
|
let groups;
|
|
1133
1251
|
if (sortSignature !== null && hasTerm && !isVectorOnly && !isHybridMode && sortsWithoutScores(params)) {
|
|
@@ -1141,9 +1259,11 @@ async function executeQuery(params, context) {
|
|
|
1141
1259
|
const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
|
|
1142
1260
|
let fanOutResult;
|
|
1143
1261
|
if (isVectorOnly && hasGlobalVectorIndex) {
|
|
1144
|
-
|
|
1262
|
+
const depth = requireCursorDepth(params.searchAfter, sortSignature, context.cursorBinding);
|
|
1263
|
+
fanOutResult = await executeVectorSearch(params, context, limit, offset, depth);
|
|
1145
1264
|
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
1146
|
-
|
|
1265
|
+
const depth = requireCursorDepth(params.searchAfter, sortSignature, context.cursorBinding);
|
|
1266
|
+
fanOutResult = await executeHybridSearch(params, context, limit, offset, depth);
|
|
1147
1267
|
} else {
|
|
1148
1268
|
const scoring = scoringConfigFor(params, context);
|
|
1149
1269
|
const workerResult = workerSearch ? await workerSearch(indexName, params, broadcastStatsForWorker(params, context, scoring), context.partitionIds) : null;
|
|
@@ -1178,11 +1298,12 @@ async function executeQuery(params, context) {
|
|
|
1178
1298
|
const sortFieldNames = sortFields.map((entry) => entry.field);
|
|
1179
1299
|
const sortFlatSchema = sortFieldNames.length === 0 ? {} : flattenSchema(config.schema);
|
|
1180
1300
|
const sortFieldTypes = sortFieldNames.map((field) => sortFlatSchema[field]);
|
|
1301
|
+
const sortModes = sortModesOf(sortFields);
|
|
1181
1302
|
const sortKeyCache = /* @__PURE__ */ new Map();
|
|
1182
1303
|
const sortKeyOf = (docId) => {
|
|
1183
1304
|
let key = sortKeyCache.get(docId);
|
|
1184
1305
|
if (key === void 0) {
|
|
1185
|
-
key = manager.sortValues(docId, sortFieldNames, sortFieldTypes);
|
|
1306
|
+
key = manager.sortValues(docId, sortFieldNames, sortFieldTypes, sortModes);
|
|
1186
1307
|
sortKeyCache.set(docId, key);
|
|
1187
1308
|
}
|
|
1188
1309
|
return key;
|
|
@@ -1193,12 +1314,15 @@ async function executeQuery(params, context) {
|
|
|
1193
1314
|
if (params.group) {
|
|
1194
1315
|
groups = applyGrouping(hits, params.group, (docId) => manager.getRef(docId));
|
|
1195
1316
|
}
|
|
1317
|
+
let pinsFromOutside = 0;
|
|
1196
1318
|
if (params.pinned && params.searchAfter === void 0 && context.partitionIds === void 0) {
|
|
1197
|
-
|
|
1319
|
+
const placement = placePinned(hits, params.pinned, (docId) => {
|
|
1198
1320
|
const doc = manager.getRef(docId);
|
|
1199
1321
|
if (!doc) return void 0;
|
|
1200
1322
|
return { id: docId, score: 0, document: doc };
|
|
1201
1323
|
});
|
|
1324
|
+
hits = placement.hits;
|
|
1325
|
+
pinsFromOutside = placement.placedFromOutside;
|
|
1202
1326
|
}
|
|
1203
1327
|
let sortContext;
|
|
1204
1328
|
if (sortSignature !== null) {
|
|
@@ -1220,7 +1344,9 @@ async function executeQuery(params, context) {
|
|
|
1220
1344
|
);
|
|
1221
1345
|
paginated = paged.paginated;
|
|
1222
1346
|
nextCursor = paged.nextCursor;
|
|
1223
|
-
|
|
1347
|
+
const holdsEveryMatch = fanOutResult.matchedExact !== false && fanOutResult.scored.length === fanOutResult.totalMatched;
|
|
1348
|
+
count = fanOutResult.totalMatched + (holdsEveryMatch ? pinsFromOutside : 0);
|
|
1349
|
+
countExact = fanOutResult.matchedExact !== false && (holdsEveryMatch || pinsFromOutside === 0);
|
|
1224
1350
|
facets = fanOutResult.facets;
|
|
1225
1351
|
if (sortSignature !== null && params.includeScores !== true) {
|
|
1226
1352
|
for (const hit of hits) hit.score = void 0;
|
|
@@ -1256,6 +1382,7 @@ async function executeQuery(params, context) {
|
|
|
1256
1382
|
return {
|
|
1257
1383
|
hits: paginated,
|
|
1258
1384
|
count,
|
|
1385
|
+
countExact,
|
|
1259
1386
|
elapsed,
|
|
1260
1387
|
cursor: nextCursor,
|
|
1261
1388
|
facets,
|
|
@@ -1267,6 +1394,7 @@ async function executePreflight(params, context) {
|
|
|
1267
1394
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
1268
1395
|
const startTime = now();
|
|
1269
1396
|
requireKnownMode(params);
|
|
1397
|
+
requireValidQueryOptions(params, config);
|
|
1270
1398
|
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
1271
1399
|
const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
|
|
1272
1400
|
const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
|
|
@@ -1275,14 +1403,17 @@ async function executePreflight(params, context) {
|
|
|
1275
1403
|
const requestedVectorField = params.vector?.field;
|
|
1276
1404
|
const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
|
|
1277
1405
|
let totalMatched;
|
|
1406
|
+
let countExact = true;
|
|
1278
1407
|
const preflightLimit = 1e3;
|
|
1279
1408
|
const preflightOffset = 0;
|
|
1280
1409
|
if (isVectorOnly && hasGlobalVectorIndex) {
|
|
1281
1410
|
const result = await executeVectorSearch(params, context, preflightLimit, preflightOffset);
|
|
1282
1411
|
totalMatched = result.totalMatched;
|
|
1412
|
+
countExact = result.matchedExact !== false;
|
|
1283
1413
|
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
1284
1414
|
const result = await executeHybridSearch(params, context, preflightLimit, preflightOffset);
|
|
1285
1415
|
totalMatched = result.totalMatched;
|
|
1416
|
+
countExact = result.matchedExact !== false;
|
|
1286
1417
|
} else if (countsWithoutScores(params)) {
|
|
1287
1418
|
totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
|
|
1288
1419
|
searchOptions: searchOptionsFor(manager),
|
|
@@ -1306,7 +1437,7 @@ async function executePreflight(params, context) {
|
|
|
1306
1437
|
}
|
|
1307
1438
|
}
|
|
1308
1439
|
const elapsed = now() - startTime;
|
|
1309
|
-
return { count: totalMatched, elapsed };
|
|
1440
|
+
return { count: totalMatched, countExact, elapsed };
|
|
1310
1441
|
}
|
|
1311
1442
|
|
|
1312
1443
|
// src/engine/suggest.ts
|
|
@@ -1356,4 +1487,4 @@ function pickDisplaySurface(surfaceOccurrences) {
|
|
|
1356
1487
|
return best;
|
|
1357
1488
|
}
|
|
1358
1489
|
|
|
1359
|
-
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, listBindingOf, now, queryBindingOf, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validateWorkerConfig };
|
|
1490
|
+
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, resolveHybridFusion, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validateWorkerConfig };
|