@delali/narsil 0.3.0 → 0.3.1-next.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
- package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
- package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
- package/dist/browser/chunk-RHG4O77A.mjs +206 -0
- package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
- package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
- package/dist/browser/module-CBB2ETYP.mjs +3111 -0
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
- package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
- package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
- package/dist/browser/write-7D6VIK6O.mjs +208 -0
- package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
- package/dist/chunk-2RIRGOUL.mjs +113 -0
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-46D6JQIG.mjs +426 -0
- package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
- package/dist/chunk-747OHOUA.mjs +11 -0
- package/dist/chunk-75XIEF6H.mjs +126 -0
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/chunk-BOFBLJFZ.mjs +209 -0
- package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
- package/dist/chunk-CVXLLEQ3.mjs +1011 -0
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-E74YLAIG.mjs +24 -0
- package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
- package/dist/chunk-EWZLIYHV.mjs +57 -0
- package/dist/chunk-GBEEG5AN.mjs +35 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-HMFNFVP7.mjs +14 -0
- package/dist/chunk-J5QP52HA.mjs +1751 -0
- package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
- package/dist/chunk-LKOB5COA.mjs +117 -0
- package/dist/chunk-LPQOEU5M.mjs +96 -0
- package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
- package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
- package/dist/chunk-OM362BRO.mjs +948 -0
- package/dist/chunk-OZDFA4WC.mjs +67 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PC2G76VW.mjs +2540 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
- package/dist/chunk-PNFKUCG3.mjs +1130 -0
- package/dist/chunk-Q6XWVUZX.mjs +44 -0
- package/dist/chunk-R27EMH5M.mjs +617 -0
- package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
- package/dist/chunk-RF7PLG43.mjs +106 -0
- package/dist/chunk-RFBLQLLZ.mjs +16 -0
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
- package/dist/chunk-S7FL6UAH.mjs +521 -0
- package/dist/chunk-S7GNDRPS.mjs +651 -0
- package/dist/chunk-SP3NM3ZG.mjs +3537 -0
- package/dist/chunk-U476IXKQ.mjs +10 -0
- package/dist/chunk-UA3AUGNV.mjs +462 -0
- package/dist/chunk-UTG5YNXV.mjs +507 -0
- package/dist/chunk-VI4R6W25.mjs +8484 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XADZPPZI.mjs +614 -0
- package/dist/chunk-YEEQHT4V.mjs +212 -0
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-YYQIODMF.mjs +275 -0
- package/dist/chunk-ZI34TVOA.mjs +144 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
- package/dist/chunk-ZM7KU4YW.mjs +35 -0
- package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +209 -45
- package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
- package/dist/config-DffQlmRq.d.ts +421 -0
- package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +79 -18
- package/dist/distribution/coordinator/etcd.mjs +553 -109
- package/dist/distribution/coordinator/in-memory.d.ts +4 -3
- package/dist/distribution/coordinator/in-memory.mjs +22 -3
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +79 -14
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +12 -7
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +50 -28
- package/dist/distribution.d.ts +144 -23
- package/dist/distribution.mjs +14479 -8173
- package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
- package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
- package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
- package/dist/index.browser.mjs +4 -32180
- package/dist/index.d.ts +91 -20
- package/dist/index.mjs +33 -16
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/module-3S65MVCM.mjs +2151 -0
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +17 -0
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +12 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
- package/dist/react.d.ts +279 -183
- package/dist/react.mjs +201 -48
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/rerank-K7C4WZFG.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
- package/dist/runtime-6Y4ZYXOG.mjs +177 -0
- package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
- package/dist/server.d.ts +13 -12
- package/dist/server.mjs +611 -80
- package/dist/snapshot-QGZPODM5.mjs +799 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
- package/dist/thread-N3IUGUS3.mjs +697 -0
- package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +46 -4
- package/dist/vector/search-worker.mjs +21 -11
- package/dist/workers/entry.mjs +9 -6
- package/dist/write-U5RI7MAX.mjs +241 -0
- package/package.json +55 -9
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-U7KYW4BA.mjs +0 -135
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/chunk-ZNZBXMP5.mjs +0 -2117
- package/dist/chunk-ZZT3EAUC.mjs +0 -11168
- package/dist/config-BhldZgaK.d.ts +0 -194
- package/dist/direct-executor-ALAVE7FC.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
- package/dist/thread-DYHEQRVY.mjs +0 -402
|
@@ -1,9 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
1
|
+
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-VI4R6W25.mjs';
|
|
2
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-LXWDQLTW.mjs';
|
|
3
|
+
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-GBEEG5AN.mjs';
|
|
4
|
+
import { flattenSchema, partialFieldsOf } from './chunk-OEYI3KRB.mjs';
|
|
5
|
+
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-E74YLAIG.mjs';
|
|
6
|
+
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
+
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-LKOB5COA.mjs';
|
|
8
|
+
import { tokenize } from './chunk-UTG5YNXV.mjs';
|
|
9
|
+
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
|
+
import { compareCodePoints, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
|
+
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
|
|
7
13
|
|
|
8
14
|
// src/search/cursor-binding.ts
|
|
9
15
|
var ABSENT_TAG = 0;
|
|
@@ -14,7 +20,13 @@ var NUMBER_TAG = 4;
|
|
|
14
20
|
var STRING_TAG = 5;
|
|
15
21
|
var LIST_TAG = 6;
|
|
16
22
|
var MAP_TAG = 7;
|
|
23
|
+
var LENGTH_BYTES = 4;
|
|
24
|
+
var LAST_ASCII_UNIT = 127;
|
|
17
25
|
var encoder = new TextEncoder();
|
|
26
|
+
function flush(stream) {
|
|
27
|
+
stream.hash = fnv1aContinue(stream.hash, stream.bytes, stream.length);
|
|
28
|
+
stream.length = 0;
|
|
29
|
+
}
|
|
18
30
|
function ensureCapacity(stream, extra) {
|
|
19
31
|
if (stream.length + extra <= stream.bytes.length) return;
|
|
20
32
|
const grown = new Uint8Array(Math.max(stream.bytes.length * 2, stream.length + extra));
|
|
@@ -43,35 +55,38 @@ function writeNumber(stream, value) {
|
|
|
43
55
|
}
|
|
44
56
|
stream.length += 8;
|
|
45
57
|
}
|
|
46
|
-
function
|
|
58
|
+
function writeEncodedString(stream, value) {
|
|
47
59
|
const encoded = encoder.encode(value);
|
|
48
60
|
writeUint32(stream, encoded.length);
|
|
49
61
|
ensureCapacity(stream, encoded.length);
|
|
50
62
|
stream.bytes.set(encoded, stream.length);
|
|
51
63
|
stream.length += encoded.length;
|
|
52
64
|
}
|
|
65
|
+
function writeString(stream, value) {
|
|
66
|
+
ensureCapacity(stream, LENGTH_BYTES + value.length);
|
|
67
|
+
const start = stream.length + LENGTH_BYTES;
|
|
68
|
+
const bytes = stream.bytes;
|
|
69
|
+
for (let at = 0; at < value.length; at++) {
|
|
70
|
+
const unit = value.charCodeAt(at);
|
|
71
|
+
if (unit > LAST_ASCII_UNIT) {
|
|
72
|
+
writeEncodedString(stream, value);
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
bytes[start + at] = unit;
|
|
76
|
+
}
|
|
77
|
+
writeUint32(stream, value.length);
|
|
78
|
+
stream.length += value.length;
|
|
79
|
+
}
|
|
53
80
|
var NumberList = class {
|
|
54
81
|
constructor(members) {
|
|
55
82
|
this.members = members;
|
|
56
83
|
}
|
|
57
84
|
};
|
|
58
|
-
var NUMBER_MEMBER_BYTES = 9;
|
|
59
|
-
var LIST_HEADER_BYTES = 5;
|
|
60
85
|
function writeNumberList(stream, list) {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
let at = stream.length;
|
|
66
|
-
bytes[at] = LIST_TAG;
|
|
67
|
-
view.setUint32(at + 1, count, false);
|
|
68
|
-
at += LIST_HEADER_BYTES;
|
|
69
|
-
for (let i = 0; i < count; i++) {
|
|
70
|
-
bytes[at] = NUMBER_TAG;
|
|
71
|
-
view.setFloat64(at + 1, Math.fround(list.members[i]), false);
|
|
72
|
-
at += NUMBER_MEMBER_BYTES;
|
|
73
|
-
}
|
|
74
|
-
stream.length = at;
|
|
86
|
+
writeByte(stream, LIST_TAG);
|
|
87
|
+
writeUint32(stream, list.members.length);
|
|
88
|
+
flush(stream);
|
|
89
|
+
stream.hash = fnv1aContinueTaggedNumbers(stream.hash, NUMBER_TAG, list.members);
|
|
75
90
|
}
|
|
76
91
|
function writeValue(stream, value) {
|
|
77
92
|
if (value instanceof NumberList) {
|
|
@@ -129,17 +144,26 @@ function boundVectorOf(vector) {
|
|
|
129
144
|
}
|
|
130
145
|
return bound;
|
|
131
146
|
}
|
|
147
|
+
function boundRerankOf(rerank) {
|
|
148
|
+
if (typeof rerank !== "object" || rerank === null || Array.isArray(rerank)) return rerank;
|
|
149
|
+
const present = {};
|
|
150
|
+
for (const [key, value] of Object.entries(rerank)) {
|
|
151
|
+
if (value !== void 0) present[key] = value;
|
|
152
|
+
}
|
|
153
|
+
return present;
|
|
154
|
+
}
|
|
132
155
|
var FILTERS_SLOT = 2;
|
|
133
|
-
var BOUND_SLOT_COUNT =
|
|
156
|
+
var BOUND_SLOT_COUNT = 16;
|
|
134
157
|
function bindingOf(values) {
|
|
135
|
-
const stream = { bytes: new Uint8Array(256), length: 0 };
|
|
158
|
+
const stream = { bytes: new Uint8Array(256), length: 0, hash: FNV_OFFSET_BASIS };
|
|
136
159
|
for (const value of values) {
|
|
137
160
|
writeValue(stream, value);
|
|
138
161
|
}
|
|
139
|
-
|
|
162
|
+
flush(stream);
|
|
163
|
+
return (stream.hash >>> 0).toString(16).padStart(8, "0");
|
|
140
164
|
}
|
|
141
165
|
function queryBindingOf(params) {
|
|
142
|
-
|
|
166
|
+
const values = [
|
|
143
167
|
params.term,
|
|
144
168
|
params.fields,
|
|
145
169
|
params.filters,
|
|
@@ -149,13 +173,16 @@ function queryBindingOf(params) {
|
|
|
149
173
|
params.tolerance,
|
|
150
174
|
params.prefixLength,
|
|
151
175
|
params.prefix,
|
|
176
|
+
params.partialWords,
|
|
152
177
|
params.exact,
|
|
153
178
|
params.pinned,
|
|
154
179
|
params.mode,
|
|
155
180
|
params.hybrid,
|
|
156
181
|
boundVectorOf(params.vector),
|
|
157
182
|
params.scoring ?? "local"
|
|
158
|
-
]
|
|
183
|
+
];
|
|
184
|
+
if (params.rerank !== void 0) values.push(boundRerankOf(params.rerank));
|
|
185
|
+
return bindingOf(values);
|
|
159
186
|
}
|
|
160
187
|
function listBindingOf(filters) {
|
|
161
188
|
const values = new Array(BOUND_SLOT_COUNT).fill(void 0);
|
|
@@ -167,7 +194,7 @@ function listBindingOf(filters) {
|
|
|
167
194
|
function requireBoostableFields(boost, flatSchema) {
|
|
168
195
|
for (const [field, weight] of Object.entries(boost)) {
|
|
169
196
|
const fieldType = flatSchema[field];
|
|
170
|
-
if (fieldType === void 0 || !
|
|
197
|
+
if (fieldType === void 0 || !isWordIndexedFieldType(fieldType)) {
|
|
171
198
|
throw new NarsilError(
|
|
172
199
|
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
173
200
|
fieldType === void 0 ? `The engine boosts a text field that the schema declares, and the schema declares no field "${field}"` : `The engine boosts a text field alone, and the schema declares field "${field}" as "${fieldType}"`,
|
|
@@ -196,21 +223,47 @@ function requireFacetableFields(facets, flatSchema, strict) {
|
|
|
196
223
|
}
|
|
197
224
|
}
|
|
198
225
|
}
|
|
199
|
-
function
|
|
226
|
+
function requireGroupableFields(fields, flatSchema, strict) {
|
|
200
227
|
for (const field of fields) {
|
|
201
|
-
|
|
228
|
+
const fieldType = flatSchema[field];
|
|
229
|
+
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
|
|
230
|
+
if (fieldType !== void 0 && VECTOR_PATTERN.test(fieldType)) {
|
|
231
|
+
throw new NarsilError(
|
|
232
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
233
|
+
`The engine groups hits by a text, number, boolean, enum, or geopoint field, and the schema declares field "${field}" as "${fieldType}"`,
|
|
234
|
+
{ field, fieldType }
|
|
235
|
+
);
|
|
236
|
+
}
|
|
202
237
|
}
|
|
203
238
|
}
|
|
204
239
|
function requireUsableQueryFields(params, schema, strict = false) {
|
|
205
|
-
const
|
|
240
|
+
const groupFields = params.group?.fields;
|
|
241
|
+
const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
|
|
206
242
|
if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
|
|
207
243
|
const flatSchema = flattenSchema(schema);
|
|
208
244
|
if (params.boost !== void 0) requireBoostableFields(params.boost, flatSchema);
|
|
209
245
|
if (params.facets !== void 0) requireFacetableFields(params.facets, flatSchema, strict);
|
|
210
|
-
if (checksGroup
|
|
246
|
+
if (checksGroup) requireGroupableFields(groupFields, flatSchema, strict);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// src/search/typo-tolerance.ts
|
|
250
|
+
function requireIntegerUpTo(option, value, maximum) {
|
|
251
|
+
if (value === void 0 || Number.isInteger(value) && value >= 0 && value <= maximum) return;
|
|
252
|
+
throw new NarsilError(
|
|
253
|
+
ErrorCodes.CONFIG_INVALID,
|
|
254
|
+
`The query option "${option}" takes an integer from 0 to ${maximum}, and this query sets ${String(value)}`,
|
|
255
|
+
{ option, value: String(value), maximum }
|
|
256
|
+
);
|
|
257
|
+
}
|
|
258
|
+
function requireValidTypoTolerance(params) {
|
|
259
|
+
requireIntegerUpTo("tolerance", params.tolerance, MAX_TOLERANCE);
|
|
260
|
+
requireIntegerUpTo("prefixLength", params.prefixLength, MAX_PREFIX_LENGTH);
|
|
211
261
|
}
|
|
212
262
|
|
|
213
263
|
// src/engine/query/shared.ts
|
|
264
|
+
function withPatternWork(context) {
|
|
265
|
+
return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap) };
|
|
266
|
+
}
|
|
214
267
|
function partitionsFor(manager, partitionIds) {
|
|
215
268
|
return partitionsIn(manager, partitionIds);
|
|
216
269
|
}
|
|
@@ -255,26 +308,22 @@ function broadcastStatsForWorker(params, context, scoring) {
|
|
|
255
308
|
if (term === void 0 || term.trim().length === 0) {
|
|
256
309
|
return void 0;
|
|
257
310
|
}
|
|
258
|
-
|
|
311
|
+
const { manager } = context;
|
|
312
|
+
const analysis = { ...manager.analysis, synonyms: manager.synonyms };
|
|
313
|
+
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, analysis, {
|
|
314
|
+
partitions: partitionsFor(manager, context.partitionIds),
|
|
315
|
+
expansion: { options: params, schema: context.config.schema }
|
|
316
|
+
});
|
|
259
317
|
}
|
|
260
|
-
function searchOptionsFor(manager) {
|
|
318
|
+
function searchOptionsFor(manager, patternWork) {
|
|
261
319
|
return {
|
|
262
320
|
bm25Params: manager.config.bm25,
|
|
263
321
|
stopWords: manager.analysis.stopWords,
|
|
264
|
-
customTokenizer: manager.analysis.customTokenizer
|
|
322
|
+
customTokenizer: manager.analysis.customTokenizer,
|
|
323
|
+
synonyms: manager.synonyms,
|
|
324
|
+
patternWork
|
|
265
325
|
};
|
|
266
326
|
}
|
|
267
|
-
function collectFilterDocIds(manager, params, schema, partitionIds) {
|
|
268
|
-
const filterDocIds = /* @__PURE__ */ new Set();
|
|
269
|
-
if (!params.filters) return filterDocIds;
|
|
270
|
-
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
271
|
-
const partitionFiltered = partition.applyFilters(params.filters, schema);
|
|
272
|
-
for (const docId of partitionFiltered) {
|
|
273
|
-
filterDocIds.add(docId);
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
return filterDocIds;
|
|
277
|
-
}
|
|
278
327
|
function vectorResultsToScored(results) {
|
|
279
328
|
return results.map((r) => ({
|
|
280
329
|
docId: r.docId,
|
|
@@ -299,14 +348,26 @@ function requireKnownMode(params) {
|
|
|
299
348
|
{ mode: String(params.mode), modes: [...SEARCH_MODES] }
|
|
300
349
|
);
|
|
301
350
|
}
|
|
302
|
-
function
|
|
351
|
+
function requireBooleanPartialWords(params) {
|
|
352
|
+
const setting = params.partialWords;
|
|
353
|
+
if (setting === void 0 || typeof setting === "boolean") return;
|
|
354
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The "partialWords" query option takes true or false', {
|
|
355
|
+
option: "partialWords",
|
|
356
|
+
received: setting === null ? "null" : typeof setting
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
function validQueryParams(params, config) {
|
|
303
360
|
const strict = config.strict === true;
|
|
361
|
+
requireValidTypoTolerance(params);
|
|
362
|
+
requireBooleanPartialWords(params);
|
|
304
363
|
requireSortableFields(params.sort, config.schema, strict);
|
|
305
|
-
|
|
364
|
+
const filters = validQueryFilter(params.filters, config.schema);
|
|
306
365
|
requireUsableQueryFields(params, config.schema, strict);
|
|
366
|
+
return filters === params.filters ? params : { ...params, filters };
|
|
307
367
|
}
|
|
308
|
-
function
|
|
309
|
-
if (filters
|
|
368
|
+
function validQueryFilter(filters, schema) {
|
|
369
|
+
if (filters === void 0 || filters === null) return filters;
|
|
370
|
+
return validatedFilterOf(filters, flattenSchema(schema));
|
|
310
371
|
}
|
|
311
372
|
function requireVectorSearchable(params, context, needsVector) {
|
|
312
373
|
if (!needsVector) return;
|
|
@@ -380,6 +441,13 @@ function validateDocId(docId) {
|
|
|
380
441
|
if (docId.includes("\0")) {
|
|
381
442
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
382
443
|
}
|
|
444
|
+
if (holdsUnpairedSurrogate(docId)) {
|
|
445
|
+
throw new NarsilError(
|
|
446
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
447
|
+
"Document ID holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8",
|
|
448
|
+
{ docId }
|
|
449
|
+
);
|
|
450
|
+
}
|
|
383
451
|
}
|
|
384
452
|
function validatePartitionConfig(partitions) {
|
|
385
453
|
const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
|
|
@@ -399,6 +467,26 @@ function validatePartitionConfig(partitions) {
|
|
|
399
467
|
});
|
|
400
468
|
}
|
|
401
469
|
}
|
|
470
|
+
function requireSurfaceFormsForPartialFields(schema, surfaceForms) {
|
|
471
|
+
if (surfaceForms !== false) return;
|
|
472
|
+
const partialFields = partialFieldsOf(schema);
|
|
473
|
+
if (partialFields.length === 0) return;
|
|
474
|
+
throw new NarsilError(
|
|
475
|
+
ErrorCodes.CONFIG_INVALID,
|
|
476
|
+
`Field "${partialFields[0]}" declares "partial", which compares words as the documents wrote them, so its index needs surfaceForms left on`,
|
|
477
|
+
{ fields: [...partialFields], surfaceForms: false }
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
function validatePatternValueLimit(limit) {
|
|
481
|
+
if (limit === void 0) return;
|
|
482
|
+
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < MIN_PATTERN_VALUE_LIMIT || limit > MAX_PATTERN_VALUE_LIMIT) {
|
|
483
|
+
throw new NarsilError(
|
|
484
|
+
ErrorCodes.CONFIG_INVALID,
|
|
485
|
+
`patternValueLimit must be a whole number of code points from ${MIN_PATTERN_VALUE_LIMIT} to ${MAX_PATTERN_VALUE_LIMIT}`,
|
|
486
|
+
{ patternValueLimit: typeof limit === "number" ? limit : String(limit) }
|
|
487
|
+
);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
402
490
|
function requirePositiveSafeInteger(field, value) {
|
|
403
491
|
if (value === void 0) return;
|
|
404
492
|
if (!Number.isSafeInteger(value) || value < 1) {
|
|
@@ -468,8 +556,9 @@ function firstIdAfter(sorted, anchor) {
|
|
|
468
556
|
}
|
|
469
557
|
return low;
|
|
470
558
|
}
|
|
471
|
-
function collectFilterMatches(partitions, filters, schema) {
|
|
472
|
-
const
|
|
559
|
+
function collectFilterMatches(partitions, filters, schema, patternWorkCap) {
|
|
560
|
+
const meter = createPatternWorkMeter(patternWorkCap);
|
|
561
|
+
const perPartition = partitions.map((partition) => partition.filterMatches(filters, schema, meter));
|
|
473
562
|
let total = 0;
|
|
474
563
|
for (const matches of perPartition) total += matches.count;
|
|
475
564
|
return { matchesFor: (partitionIndex) => perPartition[partitionIndex] ?? null, total };
|
|
@@ -547,7 +636,8 @@ function executeListDocuments(params, context) {
|
|
|
547
636
|
const limit = clampListLimit(params.limit);
|
|
548
637
|
const signature = sortSignatureOf(params.sort);
|
|
549
638
|
requireSortableFields(params.sort, schema, context.strict === true);
|
|
550
|
-
|
|
639
|
+
const filters = validQueryFilter(params.filters, schema);
|
|
640
|
+
const projection = resolveProjection(params.document);
|
|
551
641
|
const binding = listBindingOf(params.filters);
|
|
552
642
|
let cursor = null;
|
|
553
643
|
if (params.cursor !== void 0) {
|
|
@@ -555,10 +645,9 @@ function executeListDocuments(params, context) {
|
|
|
555
645
|
requireMatchingCursor(cursor, params.cursor, signature, false, binding);
|
|
556
646
|
}
|
|
557
647
|
const partitions = context.partitionIds === void 0 ? manager.getAllPartitions() : context.partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
|
|
558
|
-
const filtered =
|
|
648
|
+
const filtered = filters === void 0 ? null : collectFilterMatches(partitions, filters, schema, context.patternWorkCap);
|
|
559
649
|
const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
|
|
560
650
|
const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
|
|
561
|
-
const projection = resolveProjection(params.document);
|
|
562
651
|
const documents = [];
|
|
563
652
|
for (const docId of page.ids) {
|
|
564
653
|
if (projection.kind === "none") {
|
|
@@ -581,37 +670,23 @@ function executeListDocuments(params, context) {
|
|
|
581
670
|
}
|
|
582
671
|
|
|
583
672
|
// src/search/grouping.ts
|
|
584
|
-
function getNestedValue(obj, path) {
|
|
585
|
-
const segments = path.split(".");
|
|
586
|
-
let current = obj;
|
|
587
|
-
for (const segment of segments) {
|
|
588
|
-
if (current === null || current === void 0 || typeof current !== "object") {
|
|
589
|
-
return void 0;
|
|
590
|
-
}
|
|
591
|
-
if (!Object.hasOwn(current, segment)) {
|
|
592
|
-
return void 0;
|
|
593
|
-
}
|
|
594
|
-
current = current[segment];
|
|
595
|
-
}
|
|
596
|
-
return current;
|
|
597
|
-
}
|
|
598
673
|
function hitsKeptPerGroup(maxPerGroup) {
|
|
599
674
|
if (maxPerGroup === void 0 || !Number.isFinite(maxPerGroup) || maxPerGroup < 1) return DEFAULT_MAX_PER_GROUP;
|
|
600
675
|
return Math.floor(maxPerGroup);
|
|
601
676
|
}
|
|
602
|
-
function applyGrouping(hits, group,
|
|
677
|
+
function applyGrouping(hits, group, documents) {
|
|
603
678
|
if (!group.fields || group.fields.length === 0) {
|
|
604
679
|
return [{ values: {}, hits }];
|
|
605
680
|
}
|
|
606
681
|
const groupMap = /* @__PURE__ */ new Map();
|
|
607
682
|
for (const hit of hits) {
|
|
608
|
-
const
|
|
683
|
+
const stored = documents.fieldValues(hit.id, group.fields);
|
|
609
684
|
const fieldValues = [];
|
|
610
685
|
const valuesRecord = {};
|
|
611
|
-
for (
|
|
612
|
-
const value =
|
|
686
|
+
for (let at = 0; at < group.fields.length; at++) {
|
|
687
|
+
const value = stored?.[at];
|
|
613
688
|
fieldValues.push(value);
|
|
614
|
-
valuesRecord[
|
|
689
|
+
valuesRecord[group.fields[at]] = value;
|
|
615
690
|
}
|
|
616
691
|
const compositeKey = JSON.stringify(fieldValues);
|
|
617
692
|
const existing = groupMap.get(compositeKey);
|
|
@@ -629,7 +704,7 @@ function applyGrouping(hits, group, getDocument) {
|
|
|
629
704
|
if (group.reduce) {
|
|
630
705
|
const folded = foldGroupReducer(
|
|
631
706
|
group.reduce,
|
|
632
|
-
everyHit.map((hit) => ({ document:
|
|
707
|
+
everyHit.map((hit) => ({ document: documents.document(hit.id), score: hit.score ?? 0 }))
|
|
633
708
|
);
|
|
634
709
|
if (folded.reducerError !== void 0) {
|
|
635
710
|
result.reducerError = folded.reducerError;
|
|
@@ -800,22 +875,27 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
800
875
|
const prefixToken = options?.prefixToken;
|
|
801
876
|
const tolerance = options?.tolerance ?? 0;
|
|
802
877
|
const prefixLength = options?.prefixLength ?? 0;
|
|
878
|
+
const completedToken = options?.completedToken;
|
|
879
|
+
const queryTolerances = queryTokens.map((qt) => qt.token === completedToken ? 0 : tolerance);
|
|
880
|
+
const synonyms = options?.synonyms;
|
|
803
881
|
const matchByFieldToken = /* @__PURE__ */ new Map();
|
|
804
882
|
const matchesQuery = (fieldToken) => {
|
|
805
883
|
const known = matchByFieldToken.get(fieldToken);
|
|
806
884
|
if (known !== void 0) return known;
|
|
807
885
|
const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
|
|
808
|
-
const matches = stemmedQueryTokens.some(
|
|
809
|
-
(stemmedQuery) => fuzzyTermMatches(stemmedQuery, stemmedField,
|
|
886
|
+
const matches = synonyms?.includes(stemmedField) === true || stemmedQueryTokens.some(
|
|
887
|
+
(stemmedQuery, index) => fuzzyTermMatches(stemmedQuery, stemmedField, queryTolerances[index], prefixLength)
|
|
810
888
|
);
|
|
811
889
|
matchByFieldToken.set(fieldToken, matches);
|
|
812
890
|
return matches;
|
|
813
891
|
};
|
|
892
|
+
const matchesInside = options?.partialWords === void 0 ? null : partialWordHighlighter(text, language, options.partialWords);
|
|
814
893
|
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
815
894
|
if (i >= charOffsets.length) break;
|
|
816
895
|
const fieldToken = fieldResult.tokens[i].token;
|
|
817
896
|
const prefixMatch = prefixToken !== void 0 && fieldToken.startsWith(prefixToken);
|
|
818
|
-
|
|
897
|
+
const insideMatch = matchesInside?.(fieldResult.tokens[i].position, fieldToken) === true;
|
|
898
|
+
if (prefixMatch || insideMatch || matchesQuery(fieldToken)) {
|
|
819
899
|
matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
|
|
820
900
|
}
|
|
821
901
|
}
|
|
@@ -847,7 +927,26 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
847
927
|
}
|
|
848
928
|
|
|
849
929
|
// src/engine/query/highlight.ts
|
|
850
|
-
function
|
|
930
|
+
function writtenWordsOf(params, language, analysis) {
|
|
931
|
+
return tokenize(params.term ?? "", language, {
|
|
932
|
+
stem: false,
|
|
933
|
+
removeStopWords: true,
|
|
934
|
+
customTokenizer: analysis.customTokenizer,
|
|
935
|
+
stopWordOverride: analysis.stopWords
|
|
936
|
+
}).tokens;
|
|
937
|
+
}
|
|
938
|
+
function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
939
|
+
if (params.partialWords !== true || params.exact === true) return null;
|
|
940
|
+
const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
|
|
941
|
+
if (fields.length === 0) return null;
|
|
942
|
+
const words = partialWordsToHighlight(queryTokens, written);
|
|
943
|
+
if (words.length === 0) return null;
|
|
944
|
+
return {
|
|
945
|
+
fields: new Set(fields),
|
|
946
|
+
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
947
|
+
};
|
|
948
|
+
}
|
|
949
|
+
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument, synonymLookup) {
|
|
851
950
|
if (!params.highlight) return;
|
|
852
951
|
const queryTokenResult = tokenize(params.term ?? "", language, {
|
|
853
952
|
stem: true,
|
|
@@ -855,22 +954,24 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
855
954
|
customTokenizer: analysis.customTokenizer,
|
|
856
955
|
stopWordOverride: analysis.stopWords
|
|
857
956
|
});
|
|
957
|
+
const queryTokens = queryTokenResult.tokens;
|
|
958
|
+
const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
|
|
959
|
+
const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
|
|
960
|
+
const written = usesPrefix || usesPartialWords ? writtenWordsOf(params, language, analysis) : [];
|
|
858
961
|
let prefixToken;
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
const
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
customTokenizer: analysis.customTokenizer,
|
|
865
|
-
stopWordOverride: analysis.stopWords
|
|
866
|
-
});
|
|
867
|
-
for (const t of unstemmed.tokens) {
|
|
868
|
-
if (t.position === lastPosition) {
|
|
962
|
+
let completedToken;
|
|
963
|
+
if (usesPrefix) {
|
|
964
|
+
const lastToken = queryTokens[queryTokens.length - 1];
|
|
965
|
+
for (const t of written) {
|
|
966
|
+
if (t.position === lastToken.position) {
|
|
869
967
|
prefixToken = t.token;
|
|
968
|
+
completedToken = lastToken.token;
|
|
870
969
|
break;
|
|
871
970
|
}
|
|
872
971
|
}
|
|
873
972
|
}
|
|
973
|
+
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
974
|
+
const synonyms = params.exact === true ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
|
|
874
975
|
for (const hit of hits) {
|
|
875
976
|
const highlights = {};
|
|
876
977
|
const projected = hit.document;
|
|
@@ -882,13 +983,16 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
882
983
|
fieldValue = stored?.[field];
|
|
883
984
|
}
|
|
884
985
|
if (typeof fieldValue === "string") {
|
|
885
|
-
highlights[field] = highlightField(fieldValue,
|
|
986
|
+
highlights[field] = highlightField(fieldValue, queryTokens, language, {
|
|
886
987
|
preTag: params.highlight.preTag,
|
|
887
988
|
postTag: params.highlight.postTag,
|
|
888
989
|
maxSnippetLength: params.highlight.maxSnippetLength,
|
|
889
990
|
prefixToken,
|
|
991
|
+
completedToken,
|
|
890
992
|
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
891
|
-
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
993
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
994
|
+
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0,
|
|
995
|
+
synonyms
|
|
892
996
|
});
|
|
893
997
|
}
|
|
894
998
|
}
|
|
@@ -896,81 +1000,6 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
896
1000
|
}
|
|
897
1001
|
}
|
|
898
1002
|
|
|
899
|
-
// src/engine/query/sorted.ts
|
|
900
|
-
function sortsWithoutScores(params) {
|
|
901
|
-
return params.includeScores !== true && params.includeScoreComponents !== true && params.minScore === void 0 && params.group === void 0 && params.pinned === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
|
|
902
|
-
}
|
|
903
|
-
function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
904
|
-
const { manager, language, config } = context;
|
|
905
|
-
const schema = config.schema;
|
|
906
|
-
let anchorKey = null;
|
|
907
|
-
let anchorId = null;
|
|
908
|
-
if (params.searchAfter !== void 0) {
|
|
909
|
-
const decoded = decodePageCursor(params.searchAfter);
|
|
910
|
-
requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding);
|
|
911
|
-
anchorKey = decoded.sortKey;
|
|
912
|
-
anchorId = decoded.anchor;
|
|
913
|
-
}
|
|
914
|
-
const normalized = normalizeSort(params.sort);
|
|
915
|
-
const fields = normalized.map((entry) => entry.field);
|
|
916
|
-
const directions = normalized.map((entry) => entry.direction);
|
|
917
|
-
const flatSchema = flattenSchema(schema);
|
|
918
|
-
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
919
|
-
const modes = sortModesOf(normalized);
|
|
920
|
-
const options = searchOptionsFor(manager);
|
|
921
|
-
const partitions = partitionsFor(manager, context.partitionIds);
|
|
922
|
-
const partitionLimit = offset + limit + 1;
|
|
923
|
-
const merged = [];
|
|
924
|
-
const partitionFacets = [];
|
|
925
|
-
const partitionFacetConfig = params.facets !== void 0 && partitions.length > 1 ? everyValueFacetConfig(params.facets) : params.facets;
|
|
926
|
-
let count = 0;
|
|
927
|
-
for (const partition of partitions) {
|
|
928
|
-
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
929
|
-
if (partitionFacetConfig !== void 0) {
|
|
930
|
-
partitionFacets.push(
|
|
931
|
-
partition.computeFacets(
|
|
932
|
-
matches === null ? /* @__PURE__ */ new Set() : { ordinalBitset: matches.ordinalBitset() },
|
|
933
|
-
partitionFacetConfig,
|
|
934
|
-
schema
|
|
935
|
-
)
|
|
936
|
-
);
|
|
937
|
-
}
|
|
938
|
-
if (matches === null || matches.count === 0) continue;
|
|
939
|
-
count += matches.count;
|
|
940
|
-
const page2 = partition.sortedPage({
|
|
941
|
-
fields,
|
|
942
|
-
directions,
|
|
943
|
-
fieldTypes,
|
|
944
|
-
modes,
|
|
945
|
-
limit: partitionLimit,
|
|
946
|
-
anchorKey,
|
|
947
|
-
anchorId,
|
|
948
|
-
matches
|
|
949
|
-
});
|
|
950
|
-
for (const entry of page2) merged.push(entry);
|
|
951
|
-
}
|
|
952
|
-
merged.sort((a, b) => compareComparableKeys(a.key, b.key, directions) || compareCodePoints(a.id, b.id));
|
|
953
|
-
const hasMore = merged.length > offset + limit;
|
|
954
|
-
const page = merged.slice(offset, offset + limit);
|
|
955
|
-
const last = page[page.length - 1];
|
|
956
|
-
let cursor;
|
|
957
|
-
if (hasMore && last !== void 0) {
|
|
958
|
-
cursor = encodePageCursor({
|
|
959
|
-
anchor: last.id,
|
|
960
|
-
score: null,
|
|
961
|
-
sortKey: last.key,
|
|
962
|
-
sortSignature,
|
|
963
|
-
binding: context.cursorBinding
|
|
964
|
-
});
|
|
965
|
-
}
|
|
966
|
-
return {
|
|
967
|
-
hits: page.map((entry) => ({ id: entry.id, document: void 0 })),
|
|
968
|
-
count,
|
|
969
|
-
cursor,
|
|
970
|
-
facets: params.facets !== void 0 && partitionFacets.length > 0 ? mergeFacets(partitionFacets, params.facets) : void 0
|
|
971
|
-
};
|
|
972
|
-
}
|
|
973
|
-
|
|
974
1003
|
// src/search/fusion.ts
|
|
975
1004
|
function rankConstantOf(k) {
|
|
976
1005
|
if (k === void 0 || k === null) return DEFAULT_RANK_CONSTANT;
|
|
@@ -1003,30 +1032,27 @@ function resolveHybridFusion(hybrid) {
|
|
|
1003
1032
|
}
|
|
1004
1033
|
function reciprocalRankFusion(lists, options) {
|
|
1005
1034
|
const k = options.k;
|
|
1006
|
-
const
|
|
1007
|
-
const
|
|
1035
|
+
const positions = /* @__PURE__ */ new Map();
|
|
1036
|
+
const result = [];
|
|
1008
1037
|
for (const list of lists) {
|
|
1009
1038
|
for (let rank = 0; rank < list.length; rank++) {
|
|
1010
1039
|
const doc = list[rank];
|
|
1011
1040
|
const rrfContribution = 1 / (k + rank + 1);
|
|
1012
|
-
|
|
1013
|
-
if (
|
|
1014
|
-
|
|
1041
|
+
const position = positions.get(doc.docId);
|
|
1042
|
+
if (position === void 0) {
|
|
1043
|
+
positions.set(doc.docId, result.length);
|
|
1044
|
+
result.push({
|
|
1045
|
+
docId: doc.docId,
|
|
1046
|
+
score: rrfContribution,
|
|
1047
|
+
termFrequencies: doc.termFrequencies,
|
|
1048
|
+
fieldLengths: doc.fieldLengths,
|
|
1049
|
+
idf: doc.idf
|
|
1050
|
+
});
|
|
1051
|
+
} else {
|
|
1052
|
+
result[position].score += rrfContribution;
|
|
1015
1053
|
}
|
|
1016
1054
|
}
|
|
1017
1055
|
}
|
|
1018
|
-
const result = [];
|
|
1019
|
-
for (const [docId, score] of scores) {
|
|
1020
|
-
const original = docData.get(docId);
|
|
1021
|
-
if (!original) continue;
|
|
1022
|
-
result.push({
|
|
1023
|
-
docId,
|
|
1024
|
-
score,
|
|
1025
|
-
termFrequencies: original.termFrequencies,
|
|
1026
|
-
fieldLengths: original.fieldLengths,
|
|
1027
|
-
idf: original.idf
|
|
1028
|
-
});
|
|
1029
|
-
}
|
|
1030
1056
|
result.sort((a, b) => b.score - a.score || compareCodePoints(a.docId, b.docId));
|
|
1031
1057
|
return result;
|
|
1032
1058
|
}
|
|
@@ -1081,6 +1107,47 @@ function minMaxNormalize(docs) {
|
|
|
1081
1107
|
return docs.map((d) => ({ docId: d.docId, score: (d.score - min) / range }));
|
|
1082
1108
|
}
|
|
1083
1109
|
|
|
1110
|
+
// src/engine/query/filter-subsets.ts
|
|
1111
|
+
function collectFilterSubsets(manager, filters, schema, partitionIds, patternWork) {
|
|
1112
|
+
const subsets = [];
|
|
1113
|
+
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
1114
|
+
for (const state of partition.readStates()) {
|
|
1115
|
+
subsets.push({
|
|
1116
|
+
owner: state,
|
|
1117
|
+
bits: applyPartitionFiltersBitset(state, filters, schema, patternWork),
|
|
1118
|
+
numberingGeneration: state.docStore.numberingGeneration()
|
|
1119
|
+
});
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
return subsets;
|
|
1123
|
+
}
|
|
1124
|
+
function filterSubsetsMatchNothing(subsets) {
|
|
1125
|
+
for (const subset of subsets) {
|
|
1126
|
+
for (let wordIndex = 0; wordIndex < subset.bits.length; wordIndex++) {
|
|
1127
|
+
if (subset.bits[wordIndex] !== 0) return false;
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
return true;
|
|
1131
|
+
}
|
|
1132
|
+
function filterSubsetsRenumbered(subsets) {
|
|
1133
|
+
return subsets.some((subset) => subset.owner.docStore.numberingGeneration() !== subset.numberingGeneration);
|
|
1134
|
+
}
|
|
1135
|
+
async function searchCurrentSubsets(fieldName, first, collectAgain, search) {
|
|
1136
|
+
let subsets = first;
|
|
1137
|
+
for (let attempt = 1; ; attempt++) {
|
|
1138
|
+
const answer = await search(subsets);
|
|
1139
|
+
if (!filterSubsetsRenumbered(subsets)) return answer;
|
|
1140
|
+
if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
|
|
1141
|
+
throw new NarsilError(
|
|
1142
|
+
ErrorCodes.WORKER_BUSY,
|
|
1143
|
+
`The index renumbered its documents during each of ${attempt} filtered searches of vector field "${fieldName}"`,
|
|
1144
|
+
{ fieldName, attempts: attempt }
|
|
1145
|
+
);
|
|
1146
|
+
}
|
|
1147
|
+
subsets = collectAgain();
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1084
1151
|
// src/engine/query/vector.ts
|
|
1085
1152
|
function oversampleOf(vectorConfig) {
|
|
1086
1153
|
const oversample = vectorConfig.oversample;
|
|
@@ -1105,8 +1172,37 @@ function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
|
1105
1172
|
}
|
|
1106
1173
|
return Math.min(reach + 1, RESULT_WINDOW);
|
|
1107
1174
|
}
|
|
1175
|
+
function filteredSearchOf(params, context) {
|
|
1176
|
+
const filters = params.filters;
|
|
1177
|
+
if (!filters) return void 0;
|
|
1178
|
+
const { manager, config, partitionIds, patternWork } = context;
|
|
1179
|
+
const collectAgain = () => collectFilterSubsets(manager, filters, config.schema, partitionIds, patternWork);
|
|
1180
|
+
return { first: collectAgain(), collectAgain };
|
|
1181
|
+
}
|
|
1182
|
+
function searchField(vecIndex, vectorConfig, queryVector, k, context, filtered, rescoreWindow) {
|
|
1183
|
+
const options = {
|
|
1184
|
+
metric: vectorConfig.metric ?? "cosine",
|
|
1185
|
+
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1186
|
+
efSearch: vectorConfig.efSearch,
|
|
1187
|
+
oversample: oversampleOf(vectorConfig),
|
|
1188
|
+
...rescoreWindow !== void 0 ? { rescoreWindow } : {}
|
|
1189
|
+
};
|
|
1190
|
+
if (filtered !== void 0) {
|
|
1191
|
+
return searchCurrentSubsets(
|
|
1192
|
+
vecIndex.fieldName,
|
|
1193
|
+
filtered.first,
|
|
1194
|
+
filtered.collectAgain,
|
|
1195
|
+
(filterSubsets) => vecIndex.searchParallel(queryVector, k, { ...options, filterSubsets })
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
const filterPartitions = partitionsForVectorSearch(context.manager, vecIndex, context.partitionIds);
|
|
1199
|
+
return vecIndex.searchParallel(
|
|
1200
|
+
queryVector,
|
|
1201
|
+
k,
|
|
1202
|
+
filterPartitions === void 0 ? options : { ...options, filterPartitions }
|
|
1203
|
+
);
|
|
1204
|
+
}
|
|
1108
1205
|
async function executeVectorSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1109
|
-
const { manager, config, partitionIds } = context;
|
|
1110
1206
|
const vectorConfig = params.vector;
|
|
1111
1207
|
if (!vectorConfig || !vectorConfig.value) {
|
|
1112
1208
|
return { scored: [], totalMatched: 0 };
|
|
@@ -1115,26 +1211,13 @@ async function executeVectorSearch(params, context, limit, offset, cursorDepth =
|
|
|
1115
1211
|
if (!vecIndex) {
|
|
1116
1212
|
return { scored: [], totalMatched: 0 };
|
|
1117
1213
|
}
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds);
|
|
1122
|
-
if (filterDocIds.size === 0) {
|
|
1123
|
-
return { scored: [], totalMatched: 0 };
|
|
1124
|
-
}
|
|
1125
|
-
} else {
|
|
1126
|
-
filterPartitions = partitionsForVectorSearch(manager, vecIndex, partitionIds);
|
|
1214
|
+
const filtered = filteredSearchOf(params, context);
|
|
1215
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
1216
|
+
return { scored: [], totalMatched: 0 };
|
|
1127
1217
|
}
|
|
1128
1218
|
const queryVec = new Float32Array(vectorConfig.value);
|
|
1129
1219
|
const k = vectorFetchDepth(limit, offset, cursorDepth);
|
|
1130
|
-
const outcome = await vecIndex
|
|
1131
|
-
metric: vectorConfig.metric ?? "cosine",
|
|
1132
|
-
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1133
|
-
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
1134
|
-
...filterPartitions !== void 0 ? { filterPartitions } : {},
|
|
1135
|
-
efSearch: vectorConfig.efSearch,
|
|
1136
|
-
oversample: oversampleOf(vectorConfig)
|
|
1137
|
-
});
|
|
1220
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVec, k, context, filtered, context.rescoreWindow);
|
|
1138
1221
|
const scored = vectorResultsToScored(outcome.results);
|
|
1139
1222
|
return { scored, totalMatched: outcome.matched, matchedExact: outcome.matchedExact };
|
|
1140
1223
|
}
|
|
@@ -1146,40 +1229,35 @@ async function textLeg(textOnlyParams, context) {
|
|
|
1146
1229
|
indexName,
|
|
1147
1230
|
textOnlyParams,
|
|
1148
1231
|
broadcastStatsForWorker(textOnlyParams, context, scoring),
|
|
1149
|
-
context.partitionIds
|
|
1232
|
+
context.partitionIds,
|
|
1233
|
+
context.patternWork
|
|
1150
1234
|
);
|
|
1151
1235
|
if (viaWorker) return viaWorker;
|
|
1152
1236
|
}
|
|
1153
|
-
return fanOutQuery(
|
|
1237
|
+
return fanOutQuery(
|
|
1238
|
+
manager,
|
|
1239
|
+
textOnlyParams,
|
|
1240
|
+
language,
|
|
1241
|
+
config.schema,
|
|
1242
|
+
scoring,
|
|
1243
|
+
searchOptionsFor(manager, context.patternWork)
|
|
1244
|
+
);
|
|
1154
1245
|
}
|
|
1155
|
-
async function vectorLeg(vectorConfig, queryVector, context,
|
|
1156
|
-
const { manager } = context;
|
|
1246
|
+
async function vectorLeg(vectorConfig, queryVector, context, filtered, k, rescoreWindow) {
|
|
1157
1247
|
const vecIndex = resolveVectorIndex(context, vectorConfig.field);
|
|
1158
1248
|
if (!vecIndex) return { scored: [], holdsEveryMatch: true };
|
|
1159
|
-
const
|
|
1160
|
-
const outcome = await vecIndex.searchParallel(queryVector, k, {
|
|
1161
|
-
metric: vectorConfig.metric ?? "cosine",
|
|
1162
|
-
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1163
|
-
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
1164
|
-
...filterPartitions !== void 0 ? { filterPartitions } : {},
|
|
1165
|
-
efSearch: vectorConfig.efSearch,
|
|
1166
|
-
oversample: oversampleOf(vectorConfig)
|
|
1167
|
-
});
|
|
1249
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVector, k, context, filtered, rescoreWindow);
|
|
1168
1250
|
return {
|
|
1169
1251
|
scored: vectorResultsToScored(outcome.results),
|
|
1170
1252
|
holdsEveryMatch: outcome.matchedExact && outcome.results.length === outcome.matched
|
|
1171
1253
|
};
|
|
1172
1254
|
}
|
|
1173
1255
|
async function executeHybridSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1174
|
-
const { manager, config } = context;
|
|
1175
1256
|
const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
|
|
1176
1257
|
const fusion = resolveHybridFusion(params.hybrid);
|
|
1177
|
-
|
|
1178
|
-
if (
|
|
1179
|
-
|
|
1180
|
-
if (filterDocIds.size === 0) {
|
|
1181
|
-
return { scored: [], totalMatched: 0 };
|
|
1182
|
-
}
|
|
1258
|
+
const filtered = filteredSearchOf(params, context);
|
|
1259
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
1260
|
+
return { scored: [], totalMatched: 0 };
|
|
1183
1261
|
}
|
|
1184
1262
|
if (!vectorConfig || !vectorConfig.value) {
|
|
1185
1263
|
const textOnly = await textLeg(textOnlyParams, context);
|
|
@@ -1189,7 +1267,7 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
|
|
|
1189
1267
|
vectorConfig,
|
|
1190
1268
|
new Float32Array(vectorConfig.value),
|
|
1191
1269
|
context,
|
|
1192
|
-
|
|
1270
|
+
filtered,
|
|
1193
1271
|
vectorFetchDepth(limit, offset, cursorDepth)
|
|
1194
1272
|
);
|
|
1195
1273
|
const [textFanOutResult, vectorOutcome] = await Promise.all([textLeg(textOnlyParams, context), vectorPending]);
|
|
@@ -1213,6 +1291,107 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
|
|
|
1213
1291
|
};
|
|
1214
1292
|
}
|
|
1215
1293
|
|
|
1294
|
+
// src/engine/query/runtime-host.ts
|
|
1295
|
+
var runtimeHost = {
|
|
1296
|
+
now,
|
|
1297
|
+
compareCodePoints,
|
|
1298
|
+
clampRowCount,
|
|
1299
|
+
resolveProjection,
|
|
1300
|
+
projectionKeepsField,
|
|
1301
|
+
decodePageCursor,
|
|
1302
|
+
encodePageCursor,
|
|
1303
|
+
requireMatchingCursor,
|
|
1304
|
+
dedupePinnedEntries,
|
|
1305
|
+
placePinned,
|
|
1306
|
+
normalizeSort,
|
|
1307
|
+
flattenSchema,
|
|
1308
|
+
parsedTypeOf,
|
|
1309
|
+
isStringValuedBase,
|
|
1310
|
+
applyHighlights,
|
|
1311
|
+
textLeg,
|
|
1312
|
+
vectorLeg,
|
|
1313
|
+
filteredSearchOf,
|
|
1314
|
+
filterSubsetsMatchNothing,
|
|
1315
|
+
reciprocalRankFusion,
|
|
1316
|
+
linearCombination,
|
|
1317
|
+
resolveHybridFusion
|
|
1318
|
+
};
|
|
1319
|
+
|
|
1320
|
+
// src/engine/query/sorted.ts
|
|
1321
|
+
function sortsWithoutScores(params) {
|
|
1322
|
+
return params.includeScores !== true && params.includeScoreComponents !== true && params.minScore === void 0 && params.group === void 0 && params.pinned === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
|
|
1323
|
+
}
|
|
1324
|
+
function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
1325
|
+
const { manager, language, config } = context;
|
|
1326
|
+
const schema = config.schema;
|
|
1327
|
+
let anchorKey = null;
|
|
1328
|
+
let anchorId = null;
|
|
1329
|
+
if (params.searchAfter !== void 0) {
|
|
1330
|
+
const decoded = decodePageCursor(params.searchAfter);
|
|
1331
|
+
requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding);
|
|
1332
|
+
anchorKey = decoded.sortKey;
|
|
1333
|
+
anchorId = decoded.anchor;
|
|
1334
|
+
}
|
|
1335
|
+
const normalized = normalizeSort(params.sort);
|
|
1336
|
+
const fields = normalized.map((entry) => entry.field);
|
|
1337
|
+
const directions = normalized.map((entry) => entry.direction);
|
|
1338
|
+
const flatSchema = flattenSchema(schema);
|
|
1339
|
+
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
1340
|
+
const modes = sortModesOf(normalized);
|
|
1341
|
+
const options = searchOptionsFor(manager, context.patternWork);
|
|
1342
|
+
const partitions = partitionsFor(manager, context.partitionIds);
|
|
1343
|
+
const partitionLimit = offset + limit + 1;
|
|
1344
|
+
const merged = [];
|
|
1345
|
+
const partitionFacets = [];
|
|
1346
|
+
const partitionFacetConfig = params.facets !== void 0 && partitions.length > 1 ? everyValueFacetConfig(params.facets) : params.facets;
|
|
1347
|
+
let count = 0;
|
|
1348
|
+
for (const partition of partitions) {
|
|
1349
|
+
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
1350
|
+
if (partitionFacetConfig !== void 0) {
|
|
1351
|
+
partitionFacets.push(
|
|
1352
|
+
partition.computeFacets(
|
|
1353
|
+
matches === null ? /* @__PURE__ */ new Set() : { ordinalBitset: matches.ordinalBitset() },
|
|
1354
|
+
partitionFacetConfig,
|
|
1355
|
+
schema
|
|
1356
|
+
)
|
|
1357
|
+
);
|
|
1358
|
+
}
|
|
1359
|
+
if (matches === null || matches.count === 0) continue;
|
|
1360
|
+
count += matches.count;
|
|
1361
|
+
const page2 = partition.sortedPage({
|
|
1362
|
+
fields,
|
|
1363
|
+
directions,
|
|
1364
|
+
fieldTypes,
|
|
1365
|
+
modes,
|
|
1366
|
+
limit: partitionLimit,
|
|
1367
|
+
anchorKey,
|
|
1368
|
+
anchorId,
|
|
1369
|
+
matches
|
|
1370
|
+
});
|
|
1371
|
+
for (const entry of page2) merged.push(entry);
|
|
1372
|
+
}
|
|
1373
|
+
merged.sort((a, b) => compareComparableKeys(a.key, b.key, directions) || compareCodePoints(a.id, b.id));
|
|
1374
|
+
const hasMore = merged.length > offset + limit;
|
|
1375
|
+
const page = merged.slice(offset, offset + limit);
|
|
1376
|
+
const last = page[page.length - 1];
|
|
1377
|
+
let cursor;
|
|
1378
|
+
if (hasMore && last !== void 0) {
|
|
1379
|
+
cursor = encodePageCursor({
|
|
1380
|
+
anchor: last.id,
|
|
1381
|
+
score: null,
|
|
1382
|
+
sortKey: last.key,
|
|
1383
|
+
sortSignature,
|
|
1384
|
+
binding: context.cursorBinding
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
return {
|
|
1388
|
+
hits: page.map((entry) => ({ id: entry.id, document: void 0 })),
|
|
1389
|
+
count,
|
|
1390
|
+
cursor,
|
|
1391
|
+
facets: params.facets !== void 0 && partitionFacets.length > 0 ? mergeFacets(partitionFacets, params.facets) : void 0
|
|
1392
|
+
};
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1216
1395
|
// src/engine/query/index.ts
|
|
1217
1396
|
function requireCursorDepth(cursor, signature, binding) {
|
|
1218
1397
|
if (cursor === void 0) return 0;
|
|
@@ -1220,28 +1399,42 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
1220
1399
|
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
1221
1400
|
return decoded.depth ?? 0;
|
|
1222
1401
|
}
|
|
1223
|
-
|
|
1402
|
+
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
1403
|
+
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
1404
|
+
return import('./runtime-6Y4ZYXOG.mjs').then(
|
|
1405
|
+
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
function executeQuery(request, callerContext) {
|
|
1409
|
+
if (request.rerank === void 0) return runQuery(request, callerContext);
|
|
1410
|
+
return import('./runtime-6Y4ZYXOG.mjs').then(
|
|
1411
|
+
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
1412
|
+
);
|
|
1413
|
+
}
|
|
1414
|
+
async function runQuery(request, callerContext) {
|
|
1415
|
+
const context = withPatternWork(callerContext);
|
|
1224
1416
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
1225
1417
|
const startTime = now();
|
|
1226
|
-
const limit = clampLimit(
|
|
1227
|
-
const offset = clampOffset(
|
|
1418
|
+
const limit = clampLimit(request.limit);
|
|
1419
|
+
const offset = clampOffset(request.offset);
|
|
1228
1420
|
requireWithinResultWindow(limit, offset);
|
|
1229
|
-
const sortFields = normalizeSort(
|
|
1230
|
-
requireKnownMode(
|
|
1231
|
-
const sortSignature = sortSignatureOf(
|
|
1232
|
-
const hasTerm =
|
|
1233
|
-
const hasVector =
|
|
1234
|
-
const isHybridMode =
|
|
1235
|
-
const isVectorOnly = (
|
|
1236
|
-
if (sortSignature !== null && (isHybridMode ||
|
|
1421
|
+
const sortFields = normalizeSort(request.sort);
|
|
1422
|
+
requireKnownMode(request);
|
|
1423
|
+
const sortSignature = sortSignatureOf(request.sort);
|
|
1424
|
+
const hasTerm = request.term !== void 0 && request.term.trim().length > 0;
|
|
1425
|
+
const hasVector = request.vector !== void 0 && request.vector.value !== void 0;
|
|
1426
|
+
const isHybridMode = request.mode === "hybrid" || hasTerm && hasVector;
|
|
1427
|
+
const isVectorOnly = (request.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
|
|
1428
|
+
if (sortSignature !== null && (isHybridMode || request.hybrid !== void 0)) {
|
|
1237
1429
|
throw new NarsilError(
|
|
1238
1430
|
ErrorCodes.SEARCH_INVALID_MODE,
|
|
1239
1431
|
"A hybrid query cannot carry a sort, because fusion defines the order of hybrid results",
|
|
1240
1432
|
{ sort: sortFields.map((entry) => entry.field) }
|
|
1241
1433
|
);
|
|
1242
1434
|
}
|
|
1243
|
-
|
|
1435
|
+
const params = validQueryParams(request, config);
|
|
1244
1436
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
1437
|
+
const projection = resolveProjection(params.document);
|
|
1245
1438
|
let paginated;
|
|
1246
1439
|
let nextCursor;
|
|
1247
1440
|
let count;
|
|
@@ -1263,14 +1456,27 @@ async function executeQuery(params, context) {
|
|
|
1263
1456
|
fanOutResult = await executeVectorSearch(params, context, limit, offset, depth);
|
|
1264
1457
|
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
1265
1458
|
const depth = requireCursorDepth(params.searchAfter, sortSignature, context.cursorBinding);
|
|
1266
|
-
fanOutResult = await
|
|
1459
|
+
fanOutResult = await hybridSearchOf(params, context, limit, offset, depth);
|
|
1267
1460
|
} else {
|
|
1268
1461
|
const scoring = scoringConfigFor(params, context);
|
|
1269
|
-
const workerResult = workerSearch ? await workerSearch(
|
|
1462
|
+
const workerResult = workerSearch ? await workerSearch(
|
|
1463
|
+
indexName,
|
|
1464
|
+
params,
|
|
1465
|
+
broadcastStatsForWorker(params, context, scoring),
|
|
1466
|
+
context.partitionIds,
|
|
1467
|
+
context.patternWork
|
|
1468
|
+
) : null;
|
|
1270
1469
|
if (workerResult) {
|
|
1271
1470
|
fanOutResult = workerResult;
|
|
1272
1471
|
} else {
|
|
1273
|
-
fanOutResult = await fanOutQuery(
|
|
1472
|
+
fanOutResult = await fanOutQuery(
|
|
1473
|
+
manager,
|
|
1474
|
+
params,
|
|
1475
|
+
language,
|
|
1476
|
+
config.schema,
|
|
1477
|
+
scoring,
|
|
1478
|
+
searchOptionsFor(manager, context.patternWork)
|
|
1479
|
+
);
|
|
1274
1480
|
}
|
|
1275
1481
|
}
|
|
1276
1482
|
const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
@@ -1312,7 +1518,10 @@ async function executeQuery(params, context) {
|
|
|
1312
1518
|
hits = applySorting(hits, params.sort, sortKeyOf);
|
|
1313
1519
|
}
|
|
1314
1520
|
if (params.group) {
|
|
1315
|
-
groups = applyGrouping(hits, params.group,
|
|
1521
|
+
groups = applyGrouping(hits, params.group, {
|
|
1522
|
+
fieldValues: (docId, fieldPaths) => manager.fieldValues(docId, fieldPaths),
|
|
1523
|
+
document: (docId) => manager.getRef(docId)
|
|
1524
|
+
});
|
|
1316
1525
|
}
|
|
1317
1526
|
let pinsFromOutside = 0;
|
|
1318
1527
|
if (params.pinned && params.searchAfter === void 0 && context.partitionIds === void 0) {
|
|
@@ -1333,7 +1542,7 @@ async function executeQuery(params, context) {
|
|
|
1333
1542
|
};
|
|
1334
1543
|
}
|
|
1335
1544
|
const pinnedIds = params.pinned !== void 0 && params.searchAfter === void 0 ? new Set(params.pinned.map((entry) => entry.docId)) : void 0;
|
|
1336
|
-
const paged = applyPagination(
|
|
1545
|
+
const paged = (fanOutResult.paginate ?? applyPagination)(
|
|
1337
1546
|
hits,
|
|
1338
1547
|
limit,
|
|
1339
1548
|
offset,
|
|
@@ -1352,7 +1561,6 @@ async function executeQuery(params, context) {
|
|
|
1352
1561
|
for (const hit of hits) hit.score = void 0;
|
|
1353
1562
|
}
|
|
1354
1563
|
}
|
|
1355
|
-
const projection = resolveProjection(params.document);
|
|
1356
1564
|
if (projection.kind === "none") {
|
|
1357
1565
|
for (const hit of paginated) {
|
|
1358
1566
|
hit.document = {};
|
|
@@ -1376,7 +1584,7 @@ async function executeQuery(params, context) {
|
|
|
1376
1584
|
if (params.highlight) {
|
|
1377
1585
|
const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
|
|
1378
1586
|
const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
|
|
1379
|
-
applyHighlights(paginated, params, language, manager.analysis, readStoredDocument);
|
|
1587
|
+
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument, manager.synonyms);
|
|
1380
1588
|
}
|
|
1381
1589
|
const elapsed = now() - startTime;
|
|
1382
1590
|
return {
|
|
@@ -1390,11 +1598,12 @@ async function executeQuery(params, context) {
|
|
|
1390
1598
|
coverage: coverageFor(manager, context.partitionIds)
|
|
1391
1599
|
};
|
|
1392
1600
|
}
|
|
1393
|
-
async function executePreflight(
|
|
1601
|
+
async function executePreflight(request, callerContext) {
|
|
1602
|
+
const context = withPatternWork(callerContext);
|
|
1394
1603
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
1395
1604
|
const startTime = now();
|
|
1396
|
-
requireKnownMode(
|
|
1397
|
-
|
|
1605
|
+
requireKnownMode(request);
|
|
1606
|
+
const params = validQueryParams(request, config);
|
|
1398
1607
|
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
1399
1608
|
const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
|
|
1400
1609
|
const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
|
|
@@ -1411,17 +1620,23 @@ async function executePreflight(params, context) {
|
|
|
1411
1620
|
totalMatched = result.totalMatched;
|
|
1412
1621
|
countExact = result.matchedExact !== false;
|
|
1413
1622
|
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
1414
|
-
const result = await
|
|
1623
|
+
const result = await hybridSearchOf(params, context, preflightLimit, preflightOffset);
|
|
1415
1624
|
totalMatched = result.totalMatched;
|
|
1416
1625
|
countExact = result.matchedExact !== false;
|
|
1417
1626
|
} else if (countsWithoutScores(params)) {
|
|
1418
1627
|
totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
|
|
1419
|
-
searchOptions: searchOptionsFor(manager),
|
|
1628
|
+
searchOptions: searchOptionsFor(manager, context.patternWork),
|
|
1420
1629
|
partitionIds: context.partitionIds
|
|
1421
1630
|
});
|
|
1422
1631
|
} else {
|
|
1423
1632
|
const scoring = scoringConfigFor(params, context);
|
|
1424
|
-
const workerResult = workerSearch ? await workerSearch(
|
|
1633
|
+
const workerResult = workerSearch ? await workerSearch(
|
|
1634
|
+
indexName,
|
|
1635
|
+
params,
|
|
1636
|
+
broadcastStatsForWorker(params, context, scoring),
|
|
1637
|
+
context.partitionIds,
|
|
1638
|
+
context.patternWork
|
|
1639
|
+
) : null;
|
|
1425
1640
|
if (workerResult) {
|
|
1426
1641
|
totalMatched = workerResult.totalMatched;
|
|
1427
1642
|
} else {
|
|
@@ -1431,7 +1646,7 @@ async function executePreflight(params, context) {
|
|
|
1431
1646
|
language,
|
|
1432
1647
|
config.schema,
|
|
1433
1648
|
scoring,
|
|
1434
|
-
searchOptionsFor(manager)
|
|
1649
|
+
searchOptionsFor(manager, context.patternWork)
|
|
1435
1650
|
);
|
|
1436
1651
|
totalMatched = fanOutResult.totalMatched;
|
|
1437
1652
|
}
|
|
@@ -1487,4 +1702,4 @@ function pickDisplaySurface(surfaceOccurrences) {
|
|
|
1487
1702
|
return best;
|
|
1488
1703
|
}
|
|
1489
1704
|
|
|
1490
|
-
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, resolveHybridFusion, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validateWorkerConfig };
|
|
1705
|
+
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, runtimeHost, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|