@delali/narsil 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/dist/adapters/filesystem.d.ts +8 -0
- package/dist/adapters/filesystem.mjs +107 -0
- package/dist/adapters/indexeddb.d.ts +9 -0
- package/dist/adapters/indexeddb.mjs +153 -0
- package/dist/adapters/memory.d.ts +5 -0
- package/dist/adapters/memory.mjs +30 -0
- package/dist/adapters-XODT2F3h.d.ts +30 -0
- package/dist/chunk-5HXMVMSM.mjs +3147 -0
- package/dist/chunk-JKIBUPAY.mjs +42 -0
- package/dist/chunk-N37VMC62.mjs +18 -0
- package/dist/chunk-WYC635GI.mjs +335 -0
- package/dist/direct-executor-MQD7JRD4.mjs +3 -0
- package/dist/index.d.ts +386 -0
- package/dist/index.mjs +1099 -0
- package/dist/invalidation/broadcast-channel.d.ts +8 -0
- package/dist/invalidation/broadcast-channel.mjs +46 -0
- package/dist/invalidation/filesystem.d.ts +10 -0
- package/dist/invalidation/filesystem.mjs +159 -0
- package/dist/invalidation/noop.d.ts +5 -0
- package/dist/invalidation/noop.mjs +13 -0
- package/dist/language-ClUGLuoC.d.ts +13 -0
- package/dist/languages/arabic.d.ts +5 -0
- package/dist/languages/arabic.mjs +631 -0
- package/dist/languages/armenian.d.ts +5 -0
- package/dist/languages/armenian.mjs +306 -0
- package/dist/languages/bulgarian.d.ts +5 -0
- package/dist/languages/bulgarian.mjs +330 -0
- package/dist/languages/chinese.d.ts +5 -0
- package/dist/languages/chinese.mjs +250 -0
- package/dist/languages/dagbani.d.ts +5 -0
- package/dist/languages/dagbani.mjs +85 -0
- package/dist/languages/danish.d.ts +5 -0
- package/dist/languages/danish.mjs +224 -0
- package/dist/languages/dutch.d.ts +5 -0
- package/dist/languages/dutch.mjs +296 -0
- package/dist/languages/english.d.ts +5 -0
- package/dist/languages/english.mjs +1 -0
- package/dist/languages/ewe.d.ts +5 -0
- package/dist/languages/ewe.mjs +85 -0
- package/dist/languages/finnish.d.ts +5 -0
- package/dist/languages/finnish.mjs +536 -0
- package/dist/languages/french.d.ts +5 -0
- package/dist/languages/french.mjs +635 -0
- package/dist/languages/ga.d.ts +5 -0
- package/dist/languages/ga.mjs +69 -0
- package/dist/languages/german.d.ts +5 -0
- package/dist/languages/german.mjs +414 -0
- package/dist/languages/greek.d.ts +5 -0
- package/dist/languages/greek.mjs +888 -0
- package/dist/languages/hausa.d.ts +5 -0
- package/dist/languages/hausa.mjs +243 -0
- package/dist/languages/hindi.d.ts +5 -0
- package/dist/languages/hindi.mjs +333 -0
- package/dist/languages/hungarian.d.ts +5 -0
- package/dist/languages/hungarian.mjs +600 -0
- package/dist/languages/igbo.d.ts +5 -0
- package/dist/languages/igbo.mjs +96 -0
- package/dist/languages/indonesian.d.ts +5 -0
- package/dist/languages/indonesian.mjs +896 -0
- package/dist/languages/irish.d.ts +5 -0
- package/dist/languages/irish.mjs +268 -0
- package/dist/languages/italian.d.ts +5 -0
- package/dist/languages/italian.mjs +713 -0
- package/dist/languages/japanese.d.ts +5 -0
- package/dist/languages/japanese.mjs +210 -0
- package/dist/languages/nepali.d.ts +5 -0
- package/dist/languages/nepali.mjs +262 -0
- package/dist/languages/norwegian.d.ts +5 -0
- package/dist/languages/norwegian.mjs +274 -0
- package/dist/languages/portuguese.d.ts +5 -0
- package/dist/languages/portuguese.mjs +606 -0
- package/dist/languages/romanian.d.ts +5 -0
- package/dist/languages/romanian.mjs +634 -0
- package/dist/languages/russian.d.ts +5 -0
- package/dist/languages/russian.mjs +424 -0
- package/dist/languages/sanskrit.d.ts +5 -0
- package/dist/languages/sanskrit.mjs +155 -0
- package/dist/languages/serbian.d.ts +5 -0
- package/dist/languages/serbian.mjs +277 -0
- package/dist/languages/slovenian.d.ts +5 -0
- package/dist/languages/slovenian.mjs +522 -0
- package/dist/languages/spanish.d.ts +5 -0
- package/dist/languages/spanish.mjs +606 -0
- package/dist/languages/swahili.d.ts +5 -0
- package/dist/languages/swahili.mjs +212 -0
- package/dist/languages/swedish.d.ts +5 -0
- package/dist/languages/swedish.mjs +224 -0
- package/dist/languages/tamil.d.ts +5 -0
- package/dist/languages/tamil.mjs +186 -0
- package/dist/languages/turkish.d.ts +5 -0
- package/dist/languages/turkish.mjs +675 -0
- package/dist/languages/twi.d.ts +5 -0
- package/dist/languages/twi.mjs +84 -0
- package/dist/languages/ukrainian.d.ts +5 -0
- package/dist/languages/ukrainian.mjs +444 -0
- package/dist/languages/yoruba.d.ts +5 -0
- package/dist/languages/yoruba.mjs +103 -0
- package/dist/languages/zulu.d.ts +5 -0
- package/dist/languages/zulu.mjs +113 -0
- package/dist/workers/entry.d.ts +3 -0
- package/dist/workers/entry.mjs +106 -0
- package/package.json +87 -0
|
@@ -0,0 +1,3147 @@
|
|
|
1
|
+
import { english } from './chunk-WYC635GI.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-JKIBUPAY.mjs';
|
|
3
|
+
|
|
4
|
+
// src/languages/registry.ts
|
|
5
|
+
var languages = /* @__PURE__ */ new Map();
|
|
6
|
+
languages.set("english", english);
|
|
7
|
+
function registerLanguage(module) {
|
|
8
|
+
languages.set(module.name, module);
|
|
9
|
+
}
|
|
10
|
+
function getLanguage(name) {
|
|
11
|
+
const lang = languages.get(name);
|
|
12
|
+
if (!lang) {
|
|
13
|
+
throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, `Language "${name}" is not registered`, { language: name });
|
|
14
|
+
}
|
|
15
|
+
return lang;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// src/search/facets.ts
|
|
19
|
+
function mergeFacets(partitionFacets) {
|
|
20
|
+
const merged = /* @__PURE__ */ new Map();
|
|
21
|
+
for (const partition of partitionFacets) {
|
|
22
|
+
for (const [field, facetResult] of Object.entries(partition)) {
|
|
23
|
+
let fieldMap = merged.get(field);
|
|
24
|
+
if (!fieldMap) {
|
|
25
|
+
fieldMap = /* @__PURE__ */ new Map();
|
|
26
|
+
merged.set(field, fieldMap);
|
|
27
|
+
}
|
|
28
|
+
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
29
|
+
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const result = {};
|
|
34
|
+
for (const [field, valueMap] of merged) {
|
|
35
|
+
const values = {};
|
|
36
|
+
for (const [value, count] of valueMap) {
|
|
37
|
+
values[value] = count;
|
|
38
|
+
}
|
|
39
|
+
result[field] = { values, count: valueMap.size };
|
|
40
|
+
}
|
|
41
|
+
return result;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// src/core/fuzzy.ts
|
|
45
|
+
function boundedLevenshtein(a, b, tolerance) {
|
|
46
|
+
if (tolerance < 0) return { distance: -1, withinTolerance: false };
|
|
47
|
+
if (a === b) return { distance: 0, withinTolerance: true };
|
|
48
|
+
const m = a.length;
|
|
49
|
+
const n = b.length;
|
|
50
|
+
if (m === 0) {
|
|
51
|
+
return n <= tolerance ? { distance: n, withinTolerance: true } : { distance: n, withinTolerance: false };
|
|
52
|
+
}
|
|
53
|
+
if (n === 0) {
|
|
54
|
+
return m <= tolerance ? { distance: m, withinTolerance: true } : { distance: m, withinTolerance: false };
|
|
55
|
+
}
|
|
56
|
+
if (a.startsWith(b)) {
|
|
57
|
+
const diff2 = m - n;
|
|
58
|
+
return diff2 <= tolerance ? { distance: diff2, withinTolerance: true } : { distance: diff2, withinTolerance: false };
|
|
59
|
+
}
|
|
60
|
+
if (b.startsWith(a)) {
|
|
61
|
+
return { distance: 0, withinTolerance: true };
|
|
62
|
+
}
|
|
63
|
+
const diff = Math.abs(m - n);
|
|
64
|
+
if (diff > tolerance) return { distance: -1, withinTolerance: false };
|
|
65
|
+
const matrix = [];
|
|
66
|
+
for (let i = 0; i <= m; i++) {
|
|
67
|
+
matrix[i] = [i];
|
|
68
|
+
for (let j = 1; j <= n; j++) {
|
|
69
|
+
matrix[i][j] = i === 0 ? j : 0;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
for (let i = 1; i <= m; i++) {
|
|
73
|
+
let rowMin = Infinity;
|
|
74
|
+
for (let j = 1; j <= n; j++) {
|
|
75
|
+
if (a[i - 1] === b[j - 1]) {
|
|
76
|
+
matrix[i][j] = matrix[i - 1][j - 1];
|
|
77
|
+
} else {
|
|
78
|
+
matrix[i][j] = Math.min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + 1);
|
|
79
|
+
}
|
|
80
|
+
rowMin = Math.min(rowMin, matrix[i][j]);
|
|
81
|
+
}
|
|
82
|
+
if (rowMin > tolerance) {
|
|
83
|
+
return { distance: -1, withinTolerance: false };
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const finalDistance = matrix[m][n];
|
|
87
|
+
return {
|
|
88
|
+
distance: finalDistance,
|
|
89
|
+
withinTolerance: finalDistance <= tolerance
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// src/core/tokenizer.ts
|
|
94
|
+
var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{N}_'-]+/u;
|
|
95
|
+
var DEFAULT_MIN_TOKEN_LENGTH = 1;
|
|
96
|
+
var normalizationCache = /* @__PURE__ */ new Map();
|
|
97
|
+
var MAX_CACHE_SIZE = 65536;
|
|
98
|
+
function stripDiacritics(text) {
|
|
99
|
+
return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
|
|
100
|
+
}
|
|
101
|
+
function resolveStopWords(language, override) {
|
|
102
|
+
if (!override) return language.stopWords;
|
|
103
|
+
if (override instanceof Set) return override;
|
|
104
|
+
return override(language.stopWords);
|
|
105
|
+
}
|
|
106
|
+
function transformToken(raw, language, stem, removeDiacritics) {
|
|
107
|
+
const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
|
|
108
|
+
const cacheKey = `${language.name}:${flags}:${raw}`;
|
|
109
|
+
const cached = normalizationCache.get(cacheKey);
|
|
110
|
+
if (cached !== void 0) return cached;
|
|
111
|
+
let normalized = raw;
|
|
112
|
+
if (removeDiacritics) {
|
|
113
|
+
normalized = stripDiacritics(normalized);
|
|
114
|
+
}
|
|
115
|
+
if (stem && language.stemmer) {
|
|
116
|
+
normalized = language.stemmer(normalized);
|
|
117
|
+
}
|
|
118
|
+
cacheAndReturn(cacheKey, normalized);
|
|
119
|
+
return normalized;
|
|
120
|
+
}
|
|
121
|
+
function cacheAndReturn(key, value) {
|
|
122
|
+
if (normalizationCache.size >= MAX_CACHE_SIZE) {
|
|
123
|
+
const firstKey = normalizationCache.keys().next().value;
|
|
124
|
+
normalizationCache.delete(firstKey);
|
|
125
|
+
}
|
|
126
|
+
normalizationCache.set(key, value);
|
|
127
|
+
}
|
|
128
|
+
function splitText(text, language) {
|
|
129
|
+
const pattern = language.tokenizer?.splitPattern ?? DEFAULT_SPLIT_PATTERN;
|
|
130
|
+
return text.split(pattern);
|
|
131
|
+
}
|
|
132
|
+
function tokenize(text, language, options) {
|
|
133
|
+
const {
|
|
134
|
+
stem = true,
|
|
135
|
+
removeStopWords = true,
|
|
136
|
+
removeDiacritics = false,
|
|
137
|
+
customTokenizer,
|
|
138
|
+
stopWordOverride
|
|
139
|
+
} = options ?? {};
|
|
140
|
+
if (customTokenizer) {
|
|
141
|
+
const customResult = customTokenizer.tokenize(text);
|
|
142
|
+
return {
|
|
143
|
+
tokens: customResult,
|
|
144
|
+
originalTokens: customResult.map((t) => t.token)
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
const normalized = text.normalize("NFC").toLowerCase();
|
|
148
|
+
const rawParts = splitText(normalized, language);
|
|
149
|
+
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
150
|
+
const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
|
|
151
|
+
const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
|
|
152
|
+
const tokens = [];
|
|
153
|
+
const originalTokens = [];
|
|
154
|
+
let position = 0;
|
|
155
|
+
for (const part of rawParts) {
|
|
156
|
+
if (part.length < minLength) {
|
|
157
|
+
position++;
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
if (stopWords.has(part)) {
|
|
161
|
+
position++;
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
const processed = transformToken(part, language, stem, effectiveDiacritics);
|
|
165
|
+
if (processed.length > 0) {
|
|
166
|
+
tokens.push({ token: processed, position });
|
|
167
|
+
originalTokens.push(part);
|
|
168
|
+
}
|
|
169
|
+
position++;
|
|
170
|
+
}
|
|
171
|
+
return { tokens, originalTokens };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// src/schema/validator.ts
|
|
175
|
+
var SCALAR_FIELD_TYPES = /* @__PURE__ */ new Set([
|
|
176
|
+
"string",
|
|
177
|
+
"number",
|
|
178
|
+
"boolean",
|
|
179
|
+
"enum",
|
|
180
|
+
"geopoint",
|
|
181
|
+
"string[]",
|
|
182
|
+
"number[]",
|
|
183
|
+
"boolean[]",
|
|
184
|
+
"enum[]"
|
|
185
|
+
]);
|
|
186
|
+
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
187
|
+
var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
|
|
188
|
+
var MAX_NESTING_DEPTH = 4;
|
|
189
|
+
function isPlainObject(value) {
|
|
190
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
191
|
+
}
|
|
192
|
+
function validateSchemaFields(schema, depth, prefix) {
|
|
193
|
+
if (depth > MAX_NESTING_DEPTH) {
|
|
194
|
+
throw new NarsilError(
|
|
195
|
+
ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
|
|
196
|
+
`Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
|
|
197
|
+
{ path: prefix, maxDepth: MAX_NESTING_DEPTH }
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
201
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
202
|
+
if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
|
|
203
|
+
throw new NarsilError(
|
|
204
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
205
|
+
`Field "${field}" is reserved and cannot be defined in a schema`,
|
|
206
|
+
{ field, path }
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
if (isPlainObject(type)) {
|
|
210
|
+
validateSchemaFields(type, depth + 1, path);
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
if (typeof type !== "string") {
|
|
214
|
+
throw new NarsilError(
|
|
215
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
216
|
+
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
217
|
+
{
|
|
218
|
+
field: path,
|
|
219
|
+
type: String(type)
|
|
220
|
+
}
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
if (SCALAR_FIELD_TYPES.has(type)) continue;
|
|
224
|
+
const vectorMatch = VECTOR_PATTERN.exec(type);
|
|
225
|
+
if (vectorMatch) {
|
|
226
|
+
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
227
|
+
if (dimension <= 0) {
|
|
228
|
+
throw new NarsilError(
|
|
229
|
+
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
230
|
+
`Vector field "${path}" must have a positive dimension, got ${dimension}`,
|
|
231
|
+
{ field: path, dimension }
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field "${path}" has an unsupported type: "${type}"`, {
|
|
237
|
+
field: path,
|
|
238
|
+
type
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
function validateSchema(schema) {
|
|
243
|
+
if (!isPlainObject(schema)) {
|
|
244
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
|
|
245
|
+
received: typeof schema
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
if (Object.keys(schema).length === 0) {
|
|
249
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
250
|
+
}
|
|
251
|
+
validateSchemaFields(schema, 1, "");
|
|
252
|
+
}
|
|
253
|
+
function validateGeopoint(path, value) {
|
|
254
|
+
if (!isPlainObject(value)) {
|
|
255
|
+
throw new NarsilError(
|
|
256
|
+
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
257
|
+
`Field "${path}" expected a geopoint object with lat and lon`,
|
|
258
|
+
{
|
|
259
|
+
field: path,
|
|
260
|
+
received: Array.isArray(value) ? "array" : typeof value
|
|
261
|
+
}
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
if (typeof value.lat !== "number" || typeof value.lon !== "number" || Number.isNaN(value.lat) || Number.isNaN(value.lon)) {
|
|
265
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_GEOPOINT, `Field "${path}" geopoint requires numeric lat and lon`, {
|
|
266
|
+
field: path,
|
|
267
|
+
lat: Number.isNaN(value.lat) ? "NaN" : typeof value.lat,
|
|
268
|
+
lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
if (value.lat < -90 || value.lat > 90) {
|
|
272
|
+
throw new NarsilError(
|
|
273
|
+
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
274
|
+
`Field "${path}" geopoint lat must be between -90 and 90`,
|
|
275
|
+
{
|
|
276
|
+
field: path,
|
|
277
|
+
lat: value.lat
|
|
278
|
+
}
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
if (value.lon < -180 || value.lon > 180) {
|
|
282
|
+
throw new NarsilError(
|
|
283
|
+
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
284
|
+
`Field "${path}" geopoint lon must be between -180 and 180`,
|
|
285
|
+
{
|
|
286
|
+
field: path,
|
|
287
|
+
lon: value.lon
|
|
288
|
+
}
|
|
289
|
+
);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
function validateVector(path, value, expectedDimension) {
|
|
293
|
+
const isArray = Array.isArray(value);
|
|
294
|
+
const isFloat32 = value instanceof Float32Array;
|
|
295
|
+
if (!isArray && !isFloat32) {
|
|
296
|
+
throw new NarsilError(
|
|
297
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
298
|
+
`Field "${path}" expected a vector (Array or Float32Array)`,
|
|
299
|
+
{ field: path, received: typeof value }
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
const arr = value;
|
|
303
|
+
if (arr.length !== expectedDimension) {
|
|
304
|
+
throw new NarsilError(
|
|
305
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
306
|
+
`Field "${path}" expected vector of dimension ${expectedDimension}, got ${arr.length}`,
|
|
307
|
+
{ field: path, expected: expectedDimension, received: arr.length }
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
for (let i = 0; i < arr.length; i++) {
|
|
311
|
+
if (typeof arr[i] !== "number" || Number.isNaN(arr[i])) {
|
|
312
|
+
throw new NarsilError(
|
|
313
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
314
|
+
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
315
|
+
{ field: path, index: i, value: arr[i] }
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
function validateTypedArray(path, value, elementType) {
|
|
321
|
+
if (!Array.isArray(value)) {
|
|
322
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an array`, {
|
|
323
|
+
field: path,
|
|
324
|
+
received: typeof value
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
for (let i = 0; i < value.length; i++) {
|
|
328
|
+
const el = value[i];
|
|
329
|
+
if (typeof el !== elementType) {
|
|
330
|
+
throw new NarsilError(
|
|
331
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
332
|
+
`Field "${path}" array element at index ${i} expected ${elementType}, got ${typeof el}`,
|
|
333
|
+
{ field: path, index: i, expected: elementType, received: typeof el }
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
if (elementType === "number" && Number.isNaN(el)) {
|
|
337
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" array element at index ${i} is NaN`, {
|
|
338
|
+
field: path,
|
|
339
|
+
index: i
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
function validateFieldValue(path, value, type) {
|
|
345
|
+
switch (type) {
|
|
346
|
+
case "string":
|
|
347
|
+
if (typeof value !== "string") {
|
|
348
|
+
throw new NarsilError(
|
|
349
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
350
|
+
`Field "${path}" expected string, got ${typeof value}`,
|
|
351
|
+
{
|
|
352
|
+
field: path,
|
|
353
|
+
expected: "string",
|
|
354
|
+
received: typeof value
|
|
355
|
+
}
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
return;
|
|
359
|
+
case "number":
|
|
360
|
+
if (typeof value !== "number" || Number.isNaN(value)) {
|
|
361
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a valid number`, {
|
|
362
|
+
field: path,
|
|
363
|
+
expected: "number",
|
|
364
|
+
received: Number.isNaN(value) ? "NaN" : typeof value
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
return;
|
|
368
|
+
case "boolean":
|
|
369
|
+
if (typeof value !== "boolean") {
|
|
370
|
+
throw new NarsilError(
|
|
371
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
372
|
+
`Field "${path}" expected boolean, got ${typeof value}`,
|
|
373
|
+
{
|
|
374
|
+
field: path,
|
|
375
|
+
expected: "boolean",
|
|
376
|
+
received: typeof value
|
|
377
|
+
}
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
return;
|
|
381
|
+
case "enum":
|
|
382
|
+
if (typeof value !== "string") {
|
|
383
|
+
throw new NarsilError(
|
|
384
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
385
|
+
`Field "${path}" expected enum (string), got ${typeof value}`,
|
|
386
|
+
{
|
|
387
|
+
field: path,
|
|
388
|
+
expected: "enum",
|
|
389
|
+
received: typeof value
|
|
390
|
+
}
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
return;
|
|
394
|
+
case "geopoint":
|
|
395
|
+
validateGeopoint(path, value);
|
|
396
|
+
return;
|
|
397
|
+
case "string[]":
|
|
398
|
+
validateTypedArray(path, value, "string");
|
|
399
|
+
return;
|
|
400
|
+
case "number[]":
|
|
401
|
+
validateTypedArray(path, value, "number");
|
|
402
|
+
return;
|
|
403
|
+
case "boolean[]":
|
|
404
|
+
validateTypedArray(path, value, "boolean");
|
|
405
|
+
return;
|
|
406
|
+
case "enum[]":
|
|
407
|
+
validateTypedArray(path, value, "string");
|
|
408
|
+
return;
|
|
409
|
+
default: {
|
|
410
|
+
const vectorMatch = VECTOR_PATTERN.exec(type);
|
|
411
|
+
if (vectorMatch) {
|
|
412
|
+
validateVector(path, value, Number.parseInt(vectorMatch[1], 10));
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
function validateDocumentFields(doc, schema, prefix) {
|
|
418
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
419
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
420
|
+
const value = doc[field];
|
|
421
|
+
if (value === void 0 || value === null) continue;
|
|
422
|
+
if (isPlainObject(type)) {
|
|
423
|
+
if (!isPlainObject(value)) {
|
|
424
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an object`, {
|
|
425
|
+
field: path,
|
|
426
|
+
received: Array.isArray(value) ? "array" : typeof value
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
validateDocumentFields(value, type, path);
|
|
430
|
+
continue;
|
|
431
|
+
}
|
|
432
|
+
validateFieldValue(path, value, type);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
function validateDocument(document, schema) {
|
|
436
|
+
if (!isPlainObject(document)) {
|
|
437
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document must be a plain object", {
|
|
438
|
+
received: typeof document
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
validateDocumentFields(document, schema, "");
|
|
442
|
+
}
|
|
443
|
+
function flattenRecursive(schema, prefix, result) {
|
|
444
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
445
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
446
|
+
if (isPlainObject(type)) {
|
|
447
|
+
flattenRecursive(type, path, result);
|
|
448
|
+
} else {
|
|
449
|
+
result[path] = type;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
function flattenSchema(schema) {
|
|
454
|
+
const result = {};
|
|
455
|
+
flattenRecursive(schema, "", result);
|
|
456
|
+
return result;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
// src/search/fulltext.ts
|
|
460
|
+
function fulltextSearch(partition, params, language, schema, options) {
|
|
461
|
+
if (!params.term || params.term.trim().length === 0) {
|
|
462
|
+
return { scored: [], totalMatched: 0 };
|
|
463
|
+
}
|
|
464
|
+
if (params.fields && params.fields.length === 0) {
|
|
465
|
+
return { scored: [], totalMatched: 0 };
|
|
466
|
+
}
|
|
467
|
+
const flatSchema = flattenSchema(schema);
|
|
468
|
+
if (params.fields) {
|
|
469
|
+
validateSearchFields(params.fields, flatSchema);
|
|
470
|
+
}
|
|
471
|
+
const queryTokenResult = tokenize(params.term, language, {
|
|
472
|
+
stem: true,
|
|
473
|
+
removeStopWords: true,
|
|
474
|
+
customTokenizer: options?.customTokenizer,
|
|
475
|
+
stopWordOverride: options?.stopWords
|
|
476
|
+
});
|
|
477
|
+
if (queryTokenResult.tokens.length === 0) {
|
|
478
|
+
return { scored: [], totalMatched: 0 };
|
|
479
|
+
}
|
|
480
|
+
const queryTokens = deduplicateTokens(queryTokenResult.tokens);
|
|
481
|
+
const rawResult = partition.searchFulltext({
|
|
482
|
+
queryTokens,
|
|
483
|
+
fields: params.fields,
|
|
484
|
+
boost: params.boost,
|
|
485
|
+
tolerance: params.tolerance ?? 0,
|
|
486
|
+
prefixLength: params.prefixLength ?? 2,
|
|
487
|
+
exact: params.exact ?? false,
|
|
488
|
+
bm25Params: options?.bm25Params,
|
|
489
|
+
globalStats: options?.globalStats
|
|
490
|
+
});
|
|
491
|
+
let scored = rawResult.scored;
|
|
492
|
+
const termMatch = params.termMatch ?? "any";
|
|
493
|
+
if (termMatch !== "any") {
|
|
494
|
+
scored = filterByTermCoverage(scored, queryTokens, termMatch, params.tolerance ?? 0, params.exact ?? false);
|
|
495
|
+
}
|
|
496
|
+
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
497
|
+
const threshold = params.minScore;
|
|
498
|
+
scored = scored.filter((doc) => doc.score >= threshold);
|
|
499
|
+
}
|
|
500
|
+
if (params.filters) {
|
|
501
|
+
const matchingDocIds = partition.applyFilters(params.filters, schema);
|
|
502
|
+
scored = scored.filter((doc) => matchingDocIds.has(doc.docId));
|
|
503
|
+
}
|
|
504
|
+
return { scored, totalMatched: scored.length };
|
|
505
|
+
}
|
|
506
|
+
function validateSearchFields(fields, flatSchema) {
|
|
507
|
+
for (const field of fields) {
|
|
508
|
+
const fieldType = flatSchema[field];
|
|
509
|
+
if (!fieldType) {
|
|
510
|
+
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FIELD, `Field "${field}" does not exist in the schema`, { field });
|
|
511
|
+
}
|
|
512
|
+
if (fieldType !== "string" && fieldType !== "string[]") {
|
|
513
|
+
throw new NarsilError(
|
|
514
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
515
|
+
`Field "${field}" has type "${fieldType}" which cannot be used for full-text search`,
|
|
516
|
+
{ field, fieldType }
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
function deduplicateTokens(tokens) {
|
|
522
|
+
const seen = /* @__PURE__ */ new Set();
|
|
523
|
+
const result = [];
|
|
524
|
+
for (const t of tokens) {
|
|
525
|
+
if (!seen.has(t.token)) {
|
|
526
|
+
seen.add(t.token);
|
|
527
|
+
result.push(t);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
return result;
|
|
531
|
+
}
|
|
532
|
+
function filterByTermCoverage(scored, queryTokens, policy, tolerance, exact) {
|
|
533
|
+
const requiredCount = policy === "all" ? queryTokens.length : policy;
|
|
534
|
+
if (requiredCount <= 0) return scored;
|
|
535
|
+
if (requiredCount > queryTokens.length) return [];
|
|
536
|
+
return scored.filter((doc) => {
|
|
537
|
+
const matched = countDocTermMatches(doc, queryTokens, tolerance, exact);
|
|
538
|
+
return matched >= requiredCount;
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
function countDocTermMatches(doc, queryTokens, tolerance, exact) {
|
|
542
|
+
const indexTokens = Object.keys(doc.idf);
|
|
543
|
+
if (indexTokens.length === 0) return 0;
|
|
544
|
+
let count = 0;
|
|
545
|
+
for (const qt of queryTokens) {
|
|
546
|
+
if (queryTermSatisfied(qt.token, indexTokens, tolerance, exact)) {
|
|
547
|
+
count++;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
return count;
|
|
551
|
+
}
|
|
552
|
+
function queryTermSatisfied(queryToken, indexTokens, tolerance, exact) {
|
|
553
|
+
if (exact || tolerance === 0) {
|
|
554
|
+
return indexTokens.includes(queryToken);
|
|
555
|
+
}
|
|
556
|
+
for (const indexToken of indexTokens) {
|
|
557
|
+
if (boundedLevenshtein(queryToken, indexToken, tolerance).withinTolerance) {
|
|
558
|
+
return true;
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
return false;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// src/search/hybrid.ts
|
|
565
|
+
function hybridSearch(partition, params, language, schema, options) {
|
|
566
|
+
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
567
|
+
const hasVector = params.vector !== void 0;
|
|
568
|
+
if (!hasTerm && !hasVector) {
|
|
569
|
+
return { scored: [], totalMatched: 0 };
|
|
570
|
+
}
|
|
571
|
+
if (hasTerm && !hasVector) {
|
|
572
|
+
return fulltextSearch(partition, params, language, schema, options);
|
|
573
|
+
}
|
|
574
|
+
if (!hasTerm && hasVector) {
|
|
575
|
+
return runVectorOnly(partition, params);
|
|
576
|
+
}
|
|
577
|
+
const textResult = fulltextSearch(partition, params, language, schema, options);
|
|
578
|
+
const vectorConfig = params.vector;
|
|
579
|
+
const vectorResult = partition.searchVector({
|
|
580
|
+
field: vectorConfig.field,
|
|
581
|
+
value: vectorConfig.value,
|
|
582
|
+
k: params.limit ?? 10,
|
|
583
|
+
similarity: vectorConfig.similarity,
|
|
584
|
+
metric: vectorConfig.metric
|
|
585
|
+
});
|
|
586
|
+
const normalizedText = normalizeScores(textResult.scored);
|
|
587
|
+
const normalizedVector = normalizeScores(vectorResult.scored);
|
|
588
|
+
const textDocMap = /* @__PURE__ */ new Map();
|
|
589
|
+
for (const doc of textResult.scored) {
|
|
590
|
+
textDocMap.set(doc.docId, doc);
|
|
591
|
+
}
|
|
592
|
+
const mergedMap = /* @__PURE__ */ new Map();
|
|
593
|
+
for (const doc of normalizedText) {
|
|
594
|
+
mergedMap.set(doc.docId, {
|
|
595
|
+
textScore: doc.score,
|
|
596
|
+
vectorScore: 0,
|
|
597
|
+
textDoc: textDocMap.get(doc.docId) ?? null
|
|
598
|
+
});
|
|
599
|
+
}
|
|
600
|
+
for (const doc of normalizedVector) {
|
|
601
|
+
const existing = mergedMap.get(doc.docId);
|
|
602
|
+
if (existing) {
|
|
603
|
+
existing.vectorScore = doc.score;
|
|
604
|
+
} else {
|
|
605
|
+
mergedMap.set(doc.docId, { textScore: 0, vectorScore: doc.score, textDoc: null });
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
const alpha = clampAlpha(params.hybrid?.alpha);
|
|
609
|
+
const scored = [];
|
|
610
|
+
for (const [docId, entry] of mergedMap) {
|
|
611
|
+
const combinedScore = alpha * entry.vectorScore + (1 - alpha) * entry.textScore;
|
|
612
|
+
scored.push({
|
|
613
|
+
docId,
|
|
614
|
+
score: combinedScore,
|
|
615
|
+
termFrequencies: entry.textDoc?.termFrequencies ?? {},
|
|
616
|
+
fieldLengths: entry.textDoc?.fieldLengths ?? {},
|
|
617
|
+
idf: entry.textDoc?.idf ?? {}
|
|
618
|
+
});
|
|
619
|
+
}
|
|
620
|
+
scored.sort((a, b) => b.score - a.score || a.docId.localeCompare(b.docId));
|
|
621
|
+
let filtered = scored;
|
|
622
|
+
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
623
|
+
const threshold = params.minScore;
|
|
624
|
+
filtered = filtered.filter((doc) => doc.score >= threshold);
|
|
625
|
+
}
|
|
626
|
+
if (params.filters) {
|
|
627
|
+
const matchingDocIds = partition.applyFilters(params.filters, schema);
|
|
628
|
+
filtered = filtered.filter((doc) => matchingDocIds.has(doc.docId));
|
|
629
|
+
}
|
|
630
|
+
return { scored: filtered, totalMatched: filtered.length };
|
|
631
|
+
}
|
|
632
|
+
function runVectorOnly(partition, params) {
|
|
633
|
+
const vectorConfig = params.vector;
|
|
634
|
+
const result = partition.searchVector({
|
|
635
|
+
field: vectorConfig.field,
|
|
636
|
+
value: vectorConfig.value,
|
|
637
|
+
k: params.limit ?? 10,
|
|
638
|
+
similarity: vectorConfig.similarity,
|
|
639
|
+
metric: vectorConfig.metric
|
|
640
|
+
});
|
|
641
|
+
return result;
|
|
642
|
+
}
|
|
643
|
+
function normalizeScores(docs) {
|
|
644
|
+
if (docs.length === 0) return [];
|
|
645
|
+
let min = docs[0].score;
|
|
646
|
+
let max = docs[0].score;
|
|
647
|
+
for (let i = 1; i < docs.length; i++) {
|
|
648
|
+
if (docs[i].score < min) min = docs[i].score;
|
|
649
|
+
if (docs[i].score > max) max = docs[i].score;
|
|
650
|
+
}
|
|
651
|
+
const range = max - min;
|
|
652
|
+
if (range === 0) {
|
|
653
|
+
return docs.map((d) => ({ docId: d.docId, score: 1 }));
|
|
654
|
+
}
|
|
655
|
+
return docs.map((d) => ({ docId: d.docId, score: (d.score - min) / range }));
|
|
656
|
+
}
|
|
657
|
+
function clampAlpha(alpha) {
|
|
658
|
+
if (alpha === void 0) return 0.5;
|
|
659
|
+
if (!Number.isFinite(alpha)) return 0.5;
|
|
660
|
+
if (alpha < 0) return 0;
|
|
661
|
+
if (alpha > 1) return 1;
|
|
662
|
+
return alpha;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
// src/partitioning/distributed-scoring.ts
|
|
666
|
+
function collectGlobalStats(manager) {
|
|
667
|
+
const aggregate = manager.getAggregateStats();
|
|
668
|
+
const averageFieldLengths = {};
|
|
669
|
+
if (aggregate.totalDocuments > 0) {
|
|
670
|
+
for (const [field, totalLength] of Object.entries(aggregate.totalFieldLengths)) {
|
|
671
|
+
averageFieldLengths[field] = totalLength / aggregate.totalDocuments;
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
return {
|
|
675
|
+
totalDocuments: aggregate.totalDocuments,
|
|
676
|
+
docFrequencies: aggregate.docFrequencies,
|
|
677
|
+
totalFieldLengths: aggregate.totalFieldLengths,
|
|
678
|
+
averageFieldLengths
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// src/partitioning/fan-out.ts
|
|
683
|
+
function fanOutQuery(manager, params, language, schema, config, searchOptions) {
|
|
684
|
+
const partitions = manager.getAllPartitions();
|
|
685
|
+
if (partitions.length === 0) {
|
|
686
|
+
return { scored: [], totalMatched: 0 };
|
|
687
|
+
}
|
|
688
|
+
let globalStats;
|
|
689
|
+
const effectiveMode = resolveEffectiveMode(config);
|
|
690
|
+
if (effectiveMode === "dfs") {
|
|
691
|
+
globalStats = collectGlobalStats(manager);
|
|
692
|
+
} else if (effectiveMode === "broadcast") {
|
|
693
|
+
globalStats = config.globalStats;
|
|
694
|
+
}
|
|
695
|
+
const options = buildSearchOptions(searchOptions, globalStats);
|
|
696
|
+
const outcomes = dispatchToAllPartitions(partitions, params, language, schema, options);
|
|
697
|
+
const allScoredArrays = outcomes.map((o) => o.result.scored);
|
|
698
|
+
const merged = kWayMerge(allScoredArrays);
|
|
699
|
+
let totalMatched = 0;
|
|
700
|
+
for (const outcome of outcomes) {
|
|
701
|
+
totalMatched += outcome.result.totalMatched;
|
|
702
|
+
}
|
|
703
|
+
let facets;
|
|
704
|
+
if (params.facets) {
|
|
705
|
+
facets = collectAndMergeFacets(outcomes, params, schema);
|
|
706
|
+
}
|
|
707
|
+
return { scored: merged, totalMatched, facets };
|
|
708
|
+
}
|
|
709
|
+
function resolveEffectiveMode(config) {
|
|
710
|
+
if (config.scoringMode === "broadcast" && !config.globalStats) {
|
|
711
|
+
return "dfs";
|
|
712
|
+
}
|
|
713
|
+
return config.scoringMode;
|
|
714
|
+
}
|
|
715
|
+
function buildSearchOptions(base, globalStats) {
|
|
716
|
+
return {
|
|
717
|
+
...base,
|
|
718
|
+
globalStats: globalStats ?? base?.globalStats
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
function dispatchToAllPartitions(partitions, params, language, schema, options) {
|
|
722
|
+
const outcomes = [];
|
|
723
|
+
for (const partition of partitions) {
|
|
724
|
+
const result = dispatchSinglePartition(partition, params, language, schema, options);
|
|
725
|
+
outcomes.push({ result, partition });
|
|
726
|
+
}
|
|
727
|
+
return outcomes;
|
|
728
|
+
}
|
|
729
|
+
function dispatchSinglePartition(partition, params, language, schema, options) {
|
|
730
|
+
const isHybridMode = params.mode === "hybrid";
|
|
731
|
+
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
732
|
+
const hasVector = params.vector !== void 0;
|
|
733
|
+
if (isHybridMode || hasTerm && hasVector) {
|
|
734
|
+
return hybridSearch(partition, params, language, schema, options);
|
|
735
|
+
}
|
|
736
|
+
if (params.mode === "vector" || hasVector && !hasTerm) {
|
|
737
|
+
if (!params.vector) return { scored: [], totalMatched: 0 };
|
|
738
|
+
const vectorConfig = params.vector;
|
|
739
|
+
return partition.searchVector({
|
|
740
|
+
field: vectorConfig.field,
|
|
741
|
+
value: vectorConfig.value,
|
|
742
|
+
k: params.limit ?? 10,
|
|
743
|
+
similarity: vectorConfig.similarity,
|
|
744
|
+
metric: vectorConfig.metric
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
return fulltextSearch(partition, params, language, schema, options);
|
|
748
|
+
}
|
|
749
|
+
function collectAndMergeFacets(outcomes, params, schema) {
|
|
750
|
+
const partitionFacets = [];
|
|
751
|
+
for (const outcome of outcomes) {
|
|
752
|
+
const matchingDocIds = new Set(outcome.result.scored.map((doc) => doc.docId));
|
|
753
|
+
if (!params.facets) continue;
|
|
754
|
+
const facetResult = outcome.partition.computeFacets(matchingDocIds, params.facets, schema);
|
|
755
|
+
partitionFacets.push(facetResult);
|
|
756
|
+
}
|
|
757
|
+
return mergeFacets(partitionFacets);
|
|
758
|
+
}
|
|
759
|
+
function kWayMerge(arrays) {
|
|
760
|
+
const nonEmpty = arrays.filter((a) => a.length > 0);
|
|
761
|
+
if (nonEmpty.length === 0) return [];
|
|
762
|
+
if (nonEmpty.length === 1) return nonEmpty[0];
|
|
763
|
+
if (nonEmpty.length <= 4) {
|
|
764
|
+
return sequentialMerge(nonEmpty);
|
|
765
|
+
}
|
|
766
|
+
return heapMerge(nonEmpty);
|
|
767
|
+
}
|
|
768
|
+
function sequentialMerge(arrays) {
|
|
769
|
+
let merged = arrays[0];
|
|
770
|
+
for (let i = 1; i < arrays.length; i++) {
|
|
771
|
+
merged = mergeTwoSorted(merged, arrays[i]);
|
|
772
|
+
}
|
|
773
|
+
return merged;
|
|
774
|
+
}
|
|
775
|
+
function mergeTwoSorted(a, b) {
|
|
776
|
+
const result = new Array(a.length + b.length);
|
|
777
|
+
let ai = 0;
|
|
778
|
+
let bi = 0;
|
|
779
|
+
let ri = 0;
|
|
780
|
+
while (ai < a.length && bi < b.length) {
|
|
781
|
+
if (a[ai].score > b[bi].score || a[ai].score === b[bi].score && a[ai].docId <= b[bi].docId) {
|
|
782
|
+
result[ri++] = a[ai++];
|
|
783
|
+
} else {
|
|
784
|
+
result[ri++] = b[bi++];
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
while (ai < a.length) {
|
|
788
|
+
result[ri++] = a[ai++];
|
|
789
|
+
}
|
|
790
|
+
while (bi < b.length) {
|
|
791
|
+
result[ri++] = b[bi++];
|
|
792
|
+
}
|
|
793
|
+
return result;
|
|
794
|
+
}
|
|
795
|
+
function heapMerge(arrays) {
|
|
796
|
+
const heap = [];
|
|
797
|
+
let totalSize = 0;
|
|
798
|
+
for (let i = 0; i < arrays.length; i++) {
|
|
799
|
+
if (arrays[i].length > 0) {
|
|
800
|
+
totalSize += arrays[i].length;
|
|
801
|
+
heapPush(heap, {
|
|
802
|
+
score: arrays[i][0].score,
|
|
803
|
+
docId: arrays[i][0].docId,
|
|
804
|
+
partitionIdx: i,
|
|
805
|
+
resultIdx: 0
|
|
806
|
+
});
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
const result = new Array(totalSize);
|
|
810
|
+
let writeIdx = 0;
|
|
811
|
+
while (heap.length > 0) {
|
|
812
|
+
const top = heapPop(heap);
|
|
813
|
+
result[writeIdx++] = arrays[top.partitionIdx][top.resultIdx];
|
|
814
|
+
const nextIdx = top.resultIdx + 1;
|
|
815
|
+
if (nextIdx < arrays[top.partitionIdx].length) {
|
|
816
|
+
const nextDoc = arrays[top.partitionIdx][nextIdx];
|
|
817
|
+
heapPush(heap, {
|
|
818
|
+
score: nextDoc.score,
|
|
819
|
+
docId: nextDoc.docId,
|
|
820
|
+
partitionIdx: top.partitionIdx,
|
|
821
|
+
resultIdx: nextIdx
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
return result;
|
|
826
|
+
}
|
|
827
|
+
function heapNodeGreater(a, b) {
|
|
828
|
+
if (a.score !== b.score) return a.score > b.score;
|
|
829
|
+
return a.docId < b.docId;
|
|
830
|
+
}
|
|
831
|
+
function heapPush(heap, node) {
|
|
832
|
+
heap.push(node);
|
|
833
|
+
let idx = heap.length - 1;
|
|
834
|
+
while (idx > 0) {
|
|
835
|
+
const parentIdx = idx - 1 >> 1;
|
|
836
|
+
if (heapNodeGreater(heap[idx], heap[parentIdx])) {
|
|
837
|
+
const tmp = heap[idx];
|
|
838
|
+
heap[idx] = heap[parentIdx];
|
|
839
|
+
heap[parentIdx] = tmp;
|
|
840
|
+
idx = parentIdx;
|
|
841
|
+
} else {
|
|
842
|
+
break;
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
function heapPop(heap) {
|
|
847
|
+
const top = heap[0];
|
|
848
|
+
const last = heap.pop();
|
|
849
|
+
if (heap.length > 0) {
|
|
850
|
+
heap[0] = last;
|
|
851
|
+
let idx = 0;
|
|
852
|
+
while (true) {
|
|
853
|
+
const left = 2 * idx + 1;
|
|
854
|
+
const right = 2 * idx + 2;
|
|
855
|
+
let largest = idx;
|
|
856
|
+
if (left < heap.length && heapNodeGreater(heap[left], heap[largest])) {
|
|
857
|
+
largest = left;
|
|
858
|
+
}
|
|
859
|
+
if (right < heap.length && heapNodeGreater(heap[right], heap[largest])) {
|
|
860
|
+
largest = right;
|
|
861
|
+
}
|
|
862
|
+
if (largest !== idx) {
|
|
863
|
+
const tmp = heap[idx];
|
|
864
|
+
heap[idx] = heap[largest];
|
|
865
|
+
heap[largest] = tmp;
|
|
866
|
+
idx = largest;
|
|
867
|
+
} else {
|
|
868
|
+
break;
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
return top;
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
// src/core/document-store.ts
|
|
876
|
+
function createDocumentStore() {
|
|
877
|
+
const docs = /* @__PURE__ */ new Map();
|
|
878
|
+
return {
|
|
879
|
+
store(docId, document, fieldLengths) {
|
|
880
|
+
docs.set(docId, { fields: structuredClone(document), fieldLengths });
|
|
881
|
+
},
|
|
882
|
+
get(docId) {
|
|
883
|
+
return docs.get(docId);
|
|
884
|
+
},
|
|
885
|
+
remove(docId) {
|
|
886
|
+
return docs.delete(docId);
|
|
887
|
+
},
|
|
888
|
+
has(docId) {
|
|
889
|
+
return docs.has(docId);
|
|
890
|
+
},
|
|
891
|
+
count() {
|
|
892
|
+
return docs.size;
|
|
893
|
+
},
|
|
894
|
+
all() {
|
|
895
|
+
return docs.entries();
|
|
896
|
+
},
|
|
897
|
+
clear() {
|
|
898
|
+
docs.clear();
|
|
899
|
+
},
|
|
900
|
+
serialize() {
|
|
901
|
+
const result = {};
|
|
902
|
+
for (const [docId, stored] of docs) {
|
|
903
|
+
result[docId] = stored;
|
|
904
|
+
}
|
|
905
|
+
return result;
|
|
906
|
+
},
|
|
907
|
+
deserialize(data) {
|
|
908
|
+
docs.clear();
|
|
909
|
+
for (const docId of Object.keys(data)) {
|
|
910
|
+
docs.set(docId, data[docId]);
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
};
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
// src/core/inverted-index.ts
|
|
917
|
+
function createInvertedIndex() {
|
|
918
|
+
const index = /* @__PURE__ */ new Map();
|
|
919
|
+
const charBuckets = /* @__PURE__ */ new Map();
|
|
920
|
+
function trackToken(token) {
|
|
921
|
+
if (token.length === 0) return;
|
|
922
|
+
const ch = token[0];
|
|
923
|
+
let bucket = charBuckets.get(ch);
|
|
924
|
+
if (!bucket) {
|
|
925
|
+
bucket = /* @__PURE__ */ new Set();
|
|
926
|
+
charBuckets.set(ch, bucket);
|
|
927
|
+
}
|
|
928
|
+
bucket.add(token);
|
|
929
|
+
}
|
|
930
|
+
function untrackToken(token) {
|
|
931
|
+
if (token.length === 0) return;
|
|
932
|
+
const ch = token[0];
|
|
933
|
+
const bucket = charBuckets.get(ch);
|
|
934
|
+
if (bucket) {
|
|
935
|
+
bucket.delete(token);
|
|
936
|
+
if (bucket.size === 0) charBuckets.delete(ch);
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
function candidatesForPrefix(queryToken, prefixLength) {
|
|
940
|
+
if (prefixLength <= 0 || queryToken.length < prefixLength) {
|
|
941
|
+
return index.keys();
|
|
942
|
+
}
|
|
943
|
+
const firstChar = queryToken[0];
|
|
944
|
+
const bucket = charBuckets.get(firstChar);
|
|
945
|
+
if (!bucket) return [];
|
|
946
|
+
if (prefixLength === 1) return bucket;
|
|
947
|
+
const prefix = queryToken.slice(0, prefixLength);
|
|
948
|
+
const filtered = [];
|
|
949
|
+
for (const t of bucket) {
|
|
950
|
+
if (t.length >= prefixLength && t.startsWith(prefix)) {
|
|
951
|
+
filtered.push(t);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
return filtered;
|
|
955
|
+
}
|
|
956
|
+
return {
|
|
957
|
+
insert(token, entry) {
|
|
958
|
+
let list = index.get(token);
|
|
959
|
+
if (!list) {
|
|
960
|
+
list = { docFrequency: 0, postings: [] };
|
|
961
|
+
index.set(token, list);
|
|
962
|
+
trackToken(token);
|
|
963
|
+
}
|
|
964
|
+
const dupeIdx = list.postings.findIndex((p) => p.docId === entry.docId && p.fieldName === entry.fieldName);
|
|
965
|
+
if (dupeIdx !== -1) {
|
|
966
|
+
list.postings[dupeIdx] = entry;
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
const isNewDoc = !list.postings.some((p) => p.docId === entry.docId);
|
|
970
|
+
list.postings.push(entry);
|
|
971
|
+
if (isNewDoc) list.docFrequency++;
|
|
972
|
+
},
|
|
973
|
+
remove(token, docId) {
|
|
974
|
+
const list = index.get(token);
|
|
975
|
+
if (!list) return;
|
|
976
|
+
const lengthBefore = list.postings.length;
|
|
977
|
+
list.postings = list.postings.filter((p) => p.docId !== docId);
|
|
978
|
+
const removed = lengthBefore - list.postings.length;
|
|
979
|
+
if (removed > 0) list.docFrequency--;
|
|
980
|
+
if (list.postings.length === 0) {
|
|
981
|
+
index.delete(token);
|
|
982
|
+
untrackToken(token);
|
|
983
|
+
}
|
|
984
|
+
},
|
|
985
|
+
lookup(token) {
|
|
986
|
+
return index.get(token);
|
|
987
|
+
},
|
|
988
|
+
fuzzyLookup(token, tolerance, prefixLength) {
|
|
989
|
+
if (tolerance === 0) {
|
|
990
|
+
const exact = index.get(token);
|
|
991
|
+
return exact ? [{ token, postingList: exact }] : [];
|
|
992
|
+
}
|
|
993
|
+
const results = [];
|
|
994
|
+
const candidates = candidatesForPrefix(token, prefixLength);
|
|
995
|
+
for (const candidate of candidates) {
|
|
996
|
+
const { withinTolerance } = boundedLevenshtein(token, candidate, tolerance);
|
|
997
|
+
if (withinTolerance) {
|
|
998
|
+
const postingList = index.get(candidate);
|
|
999
|
+
if (postingList) results.push({ token: candidate, postingList });
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
return results;
|
|
1003
|
+
},
|
|
1004
|
+
has(token) {
|
|
1005
|
+
return index.has(token);
|
|
1006
|
+
},
|
|
1007
|
+
tokens() {
|
|
1008
|
+
return index.keys();
|
|
1009
|
+
},
|
|
1010
|
+
size() {
|
|
1011
|
+
return index.size;
|
|
1012
|
+
},
|
|
1013
|
+
clear() {
|
|
1014
|
+
index.clear();
|
|
1015
|
+
charBuckets.clear();
|
|
1016
|
+
},
|
|
1017
|
+
serialize() {
|
|
1018
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
1019
|
+
for (const [token, list] of index) {
|
|
1020
|
+
result[token] = list;
|
|
1021
|
+
}
|
|
1022
|
+
return result;
|
|
1023
|
+
},
|
|
1024
|
+
deserialize(data) {
|
|
1025
|
+
index.clear();
|
|
1026
|
+
charBuckets.clear();
|
|
1027
|
+
for (const token of Object.keys(data)) {
|
|
1028
|
+
index.set(token, data[token]);
|
|
1029
|
+
trackToken(token);
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
};
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
// src/core/statistics.ts
|
|
1036
|
+
function collectUniqueTokens(tokens) {
|
|
1037
|
+
const unique = /* @__PURE__ */ new Set();
|
|
1038
|
+
const fields = Object.keys(tokens);
|
|
1039
|
+
for (let i = 0; i < fields.length; i++) {
|
|
1040
|
+
const fieldTokens = tokens[fields[i]];
|
|
1041
|
+
for (let j = 0; j < fieldTokens.length; j++) {
|
|
1042
|
+
unique.add(fieldTokens[j]);
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
return unique;
|
|
1046
|
+
}
|
|
1047
|
+
function createPartitionStats() {
|
|
1048
|
+
const stats = {
|
|
1049
|
+
totalDocuments: 0,
|
|
1050
|
+
totalFieldLengths: {},
|
|
1051
|
+
averageFieldLengths: {},
|
|
1052
|
+
docFrequencies: {},
|
|
1053
|
+
addDocument(fieldLengths, tokens) {
|
|
1054
|
+
stats.totalDocuments++;
|
|
1055
|
+
const fields = Object.keys(fieldLengths);
|
|
1056
|
+
for (let i = 0; i < fields.length; i++) {
|
|
1057
|
+
const field = fields[i];
|
|
1058
|
+
stats.totalFieldLengths[field] = (stats.totalFieldLengths[field] ?? 0) + fieldLengths[field];
|
|
1059
|
+
}
|
|
1060
|
+
const uniqueTokens = collectUniqueTokens(tokens);
|
|
1061
|
+
for (const token of uniqueTokens) {
|
|
1062
|
+
stats.docFrequencies[token] = (stats.docFrequencies[token] ?? 0) + 1;
|
|
1063
|
+
}
|
|
1064
|
+
stats.recalculateAverages();
|
|
1065
|
+
},
|
|
1066
|
+
removeDocument(fieldLengths, tokens) {
|
|
1067
|
+
if (stats.totalDocuments === 0) return;
|
|
1068
|
+
stats.totalDocuments--;
|
|
1069
|
+
const fields = Object.keys(fieldLengths);
|
|
1070
|
+
for (let i = 0; i < fields.length; i++) {
|
|
1071
|
+
const field = fields[i];
|
|
1072
|
+
if (field in stats.totalFieldLengths) {
|
|
1073
|
+
stats.totalFieldLengths[field] -= fieldLengths[field];
|
|
1074
|
+
if (stats.totalFieldLengths[field] <= 0) {
|
|
1075
|
+
delete stats.totalFieldLengths[field];
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
const uniqueTokens = collectUniqueTokens(tokens);
|
|
1080
|
+
for (const token of uniqueTokens) {
|
|
1081
|
+
if (token in stats.docFrequencies) {
|
|
1082
|
+
stats.docFrequencies[token]--;
|
|
1083
|
+
if (stats.docFrequencies[token] <= 0) {
|
|
1084
|
+
delete stats.docFrequencies[token];
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
stats.recalculateAverages();
|
|
1089
|
+
},
|
|
1090
|
+
recalculateAverages() {
|
|
1091
|
+
const fields = Object.keys(stats.totalFieldLengths);
|
|
1092
|
+
const newAverages = {};
|
|
1093
|
+
for (let i = 0; i < fields.length; i++) {
|
|
1094
|
+
const field = fields[i];
|
|
1095
|
+
newAverages[field] = stats.totalDocuments > 0 ? stats.totalFieldLengths[field] / stats.totalDocuments : 0;
|
|
1096
|
+
}
|
|
1097
|
+
stats.averageFieldLengths = newAverages;
|
|
1098
|
+
},
|
|
1099
|
+
serialize() {
|
|
1100
|
+
return {
|
|
1101
|
+
totalDocuments: stats.totalDocuments,
|
|
1102
|
+
totalFieldLengths: { ...stats.totalFieldLengths },
|
|
1103
|
+
averageFieldLengths: { ...stats.averageFieldLengths },
|
|
1104
|
+
docFrequencies: { ...stats.docFrequencies }
|
|
1105
|
+
};
|
|
1106
|
+
},
|
|
1107
|
+
deserialize(data) {
|
|
1108
|
+
stats.totalDocuments = data.totalDocuments;
|
|
1109
|
+
stats.totalFieldLengths = { ...data.totalFieldLengths };
|
|
1110
|
+
stats.averageFieldLengths = { ...data.averageFieldLengths };
|
|
1111
|
+
stats.docFrequencies = { ...data.docFrequencies };
|
|
1112
|
+
}
|
|
1113
|
+
};
|
|
1114
|
+
return stats;
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
// src/geo/haversine.ts
|
|
1118
|
+
var EARTH_RADIUS_METERS = 63710088e-1;
|
|
1119
|
+
var DEG_TO_RAD = Math.PI / 180;
|
|
1120
|
+
function haversineDistance(lat1, lon1, lat2, lon2) {
|
|
1121
|
+
const phi1 = lat1 * DEG_TO_RAD;
|
|
1122
|
+
const phi2 = lat2 * DEG_TO_RAD;
|
|
1123
|
+
const deltaPhi = (lat2 - lat1) * DEG_TO_RAD;
|
|
1124
|
+
const deltaLambda = (lon2 - lon1) * DEG_TO_RAD;
|
|
1125
|
+
const halfDeltaPhi = Math.sin(deltaPhi / 2);
|
|
1126
|
+
const halfDeltaLambda = Math.sin(deltaLambda / 2);
|
|
1127
|
+
const a = halfDeltaPhi * halfDeltaPhi + Math.cos(phi1) * Math.cos(phi2) * halfDeltaLambda * halfDeltaLambda;
|
|
1128
|
+
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
|
1129
|
+
return EARTH_RADIUS_METERS * c;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
// src/geo/polygon.ts
|
|
1133
|
+
function isPointInPolygon(lat, lon, polygon) {
|
|
1134
|
+
if (polygon.length < 3) return false;
|
|
1135
|
+
let inside = false;
|
|
1136
|
+
const n = polygon.length;
|
|
1137
|
+
for (let i = 0, j = n - 1; i < n; j = i++) {
|
|
1138
|
+
const lonI = polygon[i].lon;
|
|
1139
|
+
const latI = polygon[i].lat;
|
|
1140
|
+
const lonJ = polygon[j].lon;
|
|
1141
|
+
const latJ = polygon[j].lat;
|
|
1142
|
+
if (lonI > lon !== lonJ > lon) {
|
|
1143
|
+
const slope = (latJ - latI) / (lonJ - lonI);
|
|
1144
|
+
const intersectLat = latI + slope * (lon - lonI);
|
|
1145
|
+
if (lat < intersectLat) {
|
|
1146
|
+
inside = !inside;
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
return inside;
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
// src/geo/vincenty.ts
|
|
1154
|
+
var WGS84_A = 6378137;
|
|
1155
|
+
var WGS84_F = 1 / 298.257223563;
|
|
1156
|
+
var WGS84_B = WGS84_A * (1 - WGS84_F);
|
|
1157
|
+
var DEG_TO_RAD2 = Math.PI / 180;
|
|
1158
|
+
var MAX_ITERATIONS = 200;
|
|
1159
|
+
var CONVERGENCE_THRESHOLD = 1e-12;
|
|
1160
|
+
function vincentyDistance(lat1, lon1, lat2, lon2) {
|
|
1161
|
+
const phi1 = lat1 * DEG_TO_RAD2;
|
|
1162
|
+
const phi2 = lat2 * DEG_TO_RAD2;
|
|
1163
|
+
const L = (lon2 - lon1) * DEG_TO_RAD2;
|
|
1164
|
+
const U1 = Math.atan((1 - WGS84_F) * Math.tan(phi1));
|
|
1165
|
+
const U2 = Math.atan((1 - WGS84_F) * Math.tan(phi2));
|
|
1166
|
+
const sinU1 = Math.sin(U1);
|
|
1167
|
+
const cosU1 = Math.cos(U1);
|
|
1168
|
+
const sinU2 = Math.sin(U2);
|
|
1169
|
+
const cosU2 = Math.cos(U2);
|
|
1170
|
+
let lambda = L;
|
|
1171
|
+
let prevLambda;
|
|
1172
|
+
let sinSigma = 0;
|
|
1173
|
+
let cosSigma = 0;
|
|
1174
|
+
let sigma = 0;
|
|
1175
|
+
let sinAlpha = 0;
|
|
1176
|
+
let cos2Alpha = 0;
|
|
1177
|
+
let cos2SigmaM = 0;
|
|
1178
|
+
let iterations = MAX_ITERATIONS;
|
|
1179
|
+
do {
|
|
1180
|
+
const sinLambda = Math.sin(lambda);
|
|
1181
|
+
const cosLambda = Math.cos(lambda);
|
|
1182
|
+
const term1 = cosU2 * sinLambda;
|
|
1183
|
+
const term2 = cosU1 * sinU2 - sinU1 * cosU2 * cosLambda;
|
|
1184
|
+
sinSigma = Math.sqrt(term1 * term1 + term2 * term2);
|
|
1185
|
+
if (sinSigma === 0) return 0;
|
|
1186
|
+
cosSigma = sinU1 * sinU2 + cosU1 * cosU2 * cosLambda;
|
|
1187
|
+
sigma = Math.atan2(sinSigma, cosSigma);
|
|
1188
|
+
sinAlpha = cosU1 * cosU2 * sinLambda / sinSigma;
|
|
1189
|
+
cos2Alpha = 1 - sinAlpha * sinAlpha;
|
|
1190
|
+
cos2SigmaM = cos2Alpha === 0 ? 0 : cosSigma - 2 * sinU1 * sinU2 / cos2Alpha;
|
|
1191
|
+
const C = WGS84_F / 16 * cos2Alpha * (4 + WGS84_F * (4 - 3 * cos2Alpha));
|
|
1192
|
+
prevLambda = lambda;
|
|
1193
|
+
lambda = L + (1 - C) * WGS84_F * sinAlpha * (sigma + C * sinSigma * (cos2SigmaM + C * cosSigma * (-1 + 2 * cos2SigmaM * cos2SigmaM)));
|
|
1194
|
+
} while (Math.abs(lambda - prevLambda) > CONVERGENCE_THRESHOLD && --iterations > 0);
|
|
1195
|
+
if (iterations === 0) {
|
|
1196
|
+
return haversineDistance(lat1, lon1, lat2, lon2);
|
|
1197
|
+
}
|
|
1198
|
+
const uSq = cos2Alpha * (WGS84_A * WGS84_A - WGS84_B * WGS84_B) / (WGS84_B * WGS84_B);
|
|
1199
|
+
const A = 1 + uSq / 16384 * (4096 + uSq * (-768 + uSq * (320 - 175 * uSq)));
|
|
1200
|
+
const B = uSq / 1024 * (256 + uSq * (-128 + uSq * (74 - 47 * uSq)));
|
|
1201
|
+
const deltaSigma = B * sinSigma * (cos2SigmaM + B / 4 * (cosSigma * (-1 + 2 * cos2SigmaM * cos2SigmaM) - B / 6 * cos2SigmaM * (-3 + 4 * sinSigma * sinSigma) * (-3 + 4 * cos2SigmaM * cos2SigmaM)));
|
|
1202
|
+
return WGS84_B * A * (sigma - deltaSigma);
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
// src/geo/geo-index.ts
|
|
1206
|
+
function createGeoIndex() {
|
|
1207
|
+
const entries = [];
|
|
1208
|
+
return {
|
|
1209
|
+
get entries() {
|
|
1210
|
+
return entries;
|
|
1211
|
+
},
|
|
1212
|
+
insert(docId, lat, lon) {
|
|
1213
|
+
entries.push({ lat, lon, docId });
|
|
1214
|
+
},
|
|
1215
|
+
remove(docId) {
|
|
1216
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
1217
|
+
if (entries[i].docId === docId) {
|
|
1218
|
+
entries.splice(i, 1);
|
|
1219
|
+
return;
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
},
|
|
1223
|
+
radiusQuery(lat, lon, distanceMeters, inside, highPrecision) {
|
|
1224
|
+
const result = /* @__PURE__ */ new Set();
|
|
1225
|
+
const distanceFn = highPrecision ? vincentyDistance : haversineDistance;
|
|
1226
|
+
for (let i = 0; i < entries.length; i++) {
|
|
1227
|
+
const entry = entries[i];
|
|
1228
|
+
const dist = distanceFn(lat, lon, entry.lat, entry.lon);
|
|
1229
|
+
const withinRadius = dist <= distanceMeters;
|
|
1230
|
+
if (inside ? withinRadius : !withinRadius) {
|
|
1231
|
+
result.add(entry.docId);
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
return result;
|
|
1235
|
+
},
|
|
1236
|
+
polygonQuery(points, inside) {
|
|
1237
|
+
const result = /* @__PURE__ */ new Set();
|
|
1238
|
+
for (let i = 0; i < entries.length; i++) {
|
|
1239
|
+
const entry = entries[i];
|
|
1240
|
+
const withinPolygon = isPointInPolygon(entry.lat, entry.lon, points);
|
|
1241
|
+
if (inside ? withinPolygon : !withinPolygon) {
|
|
1242
|
+
result.add(entry.docId);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
return result;
|
|
1246
|
+
},
|
|
1247
|
+
clear() {
|
|
1248
|
+
entries.length = 0;
|
|
1249
|
+
},
|
|
1250
|
+
serialize() {
|
|
1251
|
+
return entries.map((e) => ({ ...e }));
|
|
1252
|
+
},
|
|
1253
|
+
deserialize(data) {
|
|
1254
|
+
entries.length = 0;
|
|
1255
|
+
for (const entry of data) {
|
|
1256
|
+
entries.push({ lat: entry.lat, lon: entry.lon, docId: entry.docId });
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
};
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
// src/vector/similarity.ts
|
|
1263
|
+
function magnitude(v) {
|
|
1264
|
+
let sum = 0;
|
|
1265
|
+
for (let i = 0; i < v.length; i++) {
|
|
1266
|
+
sum += v[i] * v[i];
|
|
1267
|
+
}
|
|
1268
|
+
return Math.sqrt(sum);
|
|
1269
|
+
}
|
|
1270
|
+
function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
|
|
1271
|
+
if (magA === 0 || magB === 0) return 0;
|
|
1272
|
+
return dotProduct(a, b) / (magA * magB);
|
|
1273
|
+
}
|
|
1274
|
+
function dotProduct(a, b) {
|
|
1275
|
+
let sum = 0;
|
|
1276
|
+
for (let i = 0; i < a.length; i++) {
|
|
1277
|
+
sum += a[i] * b[i];
|
|
1278
|
+
}
|
|
1279
|
+
return sum;
|
|
1280
|
+
}
|
|
1281
|
+
function euclideanDistance(a, b) {
|
|
1282
|
+
let sum = 0;
|
|
1283
|
+
for (let i = 0; i < a.length; i++) {
|
|
1284
|
+
const diff = a[i] - b[i];
|
|
1285
|
+
sum += diff * diff;
|
|
1286
|
+
}
|
|
1287
|
+
return Math.sqrt(sum);
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
// src/vector/brute-force.ts
|
|
1291
|
+
function createBruteForceVectorStore(dimension) {
|
|
1292
|
+
const vectors = /* @__PURE__ */ new Map();
|
|
1293
|
+
return {
|
|
1294
|
+
get dimension() {
|
|
1295
|
+
return dimension;
|
|
1296
|
+
},
|
|
1297
|
+
get size() {
|
|
1298
|
+
return vectors.size;
|
|
1299
|
+
},
|
|
1300
|
+
insert(docId, vector) {
|
|
1301
|
+
if (vector.length !== dimension) {
|
|
1302
|
+
throw new Error(`Vector dimension mismatch: expected ${dimension}, got ${vector.length}`);
|
|
1303
|
+
}
|
|
1304
|
+
const mag = magnitude(vector);
|
|
1305
|
+
vectors.set(docId, { docId, vector, magnitude: mag });
|
|
1306
|
+
},
|
|
1307
|
+
remove(docId) {
|
|
1308
|
+
vectors.delete(docId);
|
|
1309
|
+
},
|
|
1310
|
+
has(docId) {
|
|
1311
|
+
return vectors.has(docId);
|
|
1312
|
+
},
|
|
1313
|
+
search(query, k, metric, minSimilarity, filterDocIds) {
|
|
1314
|
+
if (query.length !== dimension) {
|
|
1315
|
+
throw new Error(`Query dimension mismatch: expected ${dimension}, got ${query.length}`);
|
|
1316
|
+
}
|
|
1317
|
+
const queryMag = magnitude(query);
|
|
1318
|
+
const results = [];
|
|
1319
|
+
for (const [docId, entry] of vectors) {
|
|
1320
|
+
if (filterDocIds && !filterDocIds.has(docId)) continue;
|
|
1321
|
+
let score;
|
|
1322
|
+
switch (metric) {
|
|
1323
|
+
case "cosine":
|
|
1324
|
+
score = cosineSimilarityWithMagnitudes(query, entry.vector, queryMag, entry.magnitude);
|
|
1325
|
+
break;
|
|
1326
|
+
case "dotProduct":
|
|
1327
|
+
score = dotProduct(query, entry.vector);
|
|
1328
|
+
break;
|
|
1329
|
+
case "euclidean": {
|
|
1330
|
+
const dist = euclideanDistance(query, entry.vector);
|
|
1331
|
+
score = 1 / (1 + dist);
|
|
1332
|
+
break;
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
if (score >= minSimilarity) {
|
|
1336
|
+
results.push({ docId, score });
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
results.sort((a, b) => b.score - a.score);
|
|
1340
|
+
const topK = results.slice(0, k);
|
|
1341
|
+
return topK.map((r) => ({
|
|
1342
|
+
docId: r.docId,
|
|
1343
|
+
score: r.score,
|
|
1344
|
+
termFrequencies: {},
|
|
1345
|
+
fieldLengths: {},
|
|
1346
|
+
idf: {}
|
|
1347
|
+
}));
|
|
1348
|
+
},
|
|
1349
|
+
clear() {
|
|
1350
|
+
vectors.clear();
|
|
1351
|
+
},
|
|
1352
|
+
entries() {
|
|
1353
|
+
return vectors.entries();
|
|
1354
|
+
}
|
|
1355
|
+
};
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
// src/core/field-index.ts
|
|
1359
|
+
function lowerBound(entries, value) {
|
|
1360
|
+
let lo = 0;
|
|
1361
|
+
let hi = entries.length;
|
|
1362
|
+
while (lo < hi) {
|
|
1363
|
+
const mid = lo + hi >>> 1;
|
|
1364
|
+
if (entries[mid].value < value) lo = mid + 1;
|
|
1365
|
+
else hi = mid;
|
|
1366
|
+
}
|
|
1367
|
+
return lo;
|
|
1368
|
+
}
|
|
1369
|
+
function upperBound(entries, value) {
|
|
1370
|
+
let lo = 0;
|
|
1371
|
+
let hi = entries.length;
|
|
1372
|
+
while (lo < hi) {
|
|
1373
|
+
const mid = lo + hi >>> 1;
|
|
1374
|
+
if (entries[mid].value <= value) lo = mid + 1;
|
|
1375
|
+
else hi = mid;
|
|
1376
|
+
}
|
|
1377
|
+
return lo;
|
|
1378
|
+
}
|
|
1379
|
+
function collectDocIds(entries, from, to) {
|
|
1380
|
+
const result = /* @__PURE__ */ new Set();
|
|
1381
|
+
for (let i = from; i < to; i++) {
|
|
1382
|
+
result.add(entries[i].docId);
|
|
1383
|
+
}
|
|
1384
|
+
return result;
|
|
1385
|
+
}
|
|
1386
|
+
function createNumericIndex() {
|
|
1387
|
+
let entries = [];
|
|
1388
|
+
return {
|
|
1389
|
+
insert(docId, value) {
|
|
1390
|
+
const pos = upperBound(entries, value);
|
|
1391
|
+
entries.splice(pos, 0, { value, docId });
|
|
1392
|
+
},
|
|
1393
|
+
remove(docId, value) {
|
|
1394
|
+
const start = lowerBound(entries, value);
|
|
1395
|
+
const end = upperBound(entries, value);
|
|
1396
|
+
for (let i = start; i < end; i++) {
|
|
1397
|
+
if (entries[i].docId === docId) {
|
|
1398
|
+
entries.splice(i, 1);
|
|
1399
|
+
return;
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
},
|
|
1403
|
+
queryEq(value) {
|
|
1404
|
+
return collectDocIds(entries, lowerBound(entries, value), upperBound(entries, value));
|
|
1405
|
+
},
|
|
1406
|
+
queryNe(value) {
|
|
1407
|
+
const excluded = this.queryEq(value);
|
|
1408
|
+
const result = /* @__PURE__ */ new Set();
|
|
1409
|
+
for (const entry of entries) {
|
|
1410
|
+
if (!excluded.has(entry.docId)) result.add(entry.docId);
|
|
1411
|
+
}
|
|
1412
|
+
return result;
|
|
1413
|
+
},
|
|
1414
|
+
queryGt(value) {
|
|
1415
|
+
return collectDocIds(entries, upperBound(entries, value), entries.length);
|
|
1416
|
+
},
|
|
1417
|
+
queryGte(value) {
|
|
1418
|
+
return collectDocIds(entries, lowerBound(entries, value), entries.length);
|
|
1419
|
+
},
|
|
1420
|
+
queryLt(value) {
|
|
1421
|
+
return collectDocIds(entries, 0, lowerBound(entries, value));
|
|
1422
|
+
},
|
|
1423
|
+
queryLte(value) {
|
|
1424
|
+
return collectDocIds(entries, 0, upperBound(entries, value));
|
|
1425
|
+
},
|
|
1426
|
+
queryBetween(min, max) {
|
|
1427
|
+
return collectDocIds(entries, lowerBound(entries, min), upperBound(entries, max));
|
|
1428
|
+
},
|
|
1429
|
+
getAllDocIds() {
|
|
1430
|
+
return collectDocIds(entries, 0, entries.length);
|
|
1431
|
+
},
|
|
1432
|
+
count() {
|
|
1433
|
+
return entries.length;
|
|
1434
|
+
},
|
|
1435
|
+
clear() {
|
|
1436
|
+
entries = [];
|
|
1437
|
+
},
|
|
1438
|
+
serialize() {
|
|
1439
|
+
return entries.slice();
|
|
1440
|
+
},
|
|
1441
|
+
deserialize(data) {
|
|
1442
|
+
entries = data.slice();
|
|
1443
|
+
}
|
|
1444
|
+
};
|
|
1445
|
+
}
|
|
1446
|
+
function createBooleanIndex() {
|
|
1447
|
+
const trueDocs = /* @__PURE__ */ new Set();
|
|
1448
|
+
const falseDocs = /* @__PURE__ */ new Set();
|
|
1449
|
+
return {
|
|
1450
|
+
insert(docId, value) {
|
|
1451
|
+
if (value) trueDocs.add(docId);
|
|
1452
|
+
else falseDocs.add(docId);
|
|
1453
|
+
},
|
|
1454
|
+
remove(docId, value) {
|
|
1455
|
+
if (value) trueDocs.delete(docId);
|
|
1456
|
+
else falseDocs.delete(docId);
|
|
1457
|
+
},
|
|
1458
|
+
queryEq(value) {
|
|
1459
|
+
return new Set(value ? trueDocs : falseDocs);
|
|
1460
|
+
},
|
|
1461
|
+
queryNe(value) {
|
|
1462
|
+
return new Set(value ? falseDocs : trueDocs);
|
|
1463
|
+
},
|
|
1464
|
+
getAllDocIds() {
|
|
1465
|
+
const all = new Set(trueDocs);
|
|
1466
|
+
for (const id of falseDocs) all.add(id);
|
|
1467
|
+
return all;
|
|
1468
|
+
},
|
|
1469
|
+
count() {
|
|
1470
|
+
return trueDocs.size + falseDocs.size;
|
|
1471
|
+
},
|
|
1472
|
+
clear() {
|
|
1473
|
+
trueDocs.clear();
|
|
1474
|
+
falseDocs.clear();
|
|
1475
|
+
},
|
|
1476
|
+
serialize() {
|
|
1477
|
+
return {
|
|
1478
|
+
trueDocs: Array.from(trueDocs),
|
|
1479
|
+
falseDocs: Array.from(falseDocs)
|
|
1480
|
+
};
|
|
1481
|
+
},
|
|
1482
|
+
deserialize(data) {
|
|
1483
|
+
trueDocs.clear();
|
|
1484
|
+
falseDocs.clear();
|
|
1485
|
+
for (const id of data.trueDocs) trueDocs.add(id);
|
|
1486
|
+
for (const id of data.falseDocs) falseDocs.add(id);
|
|
1487
|
+
}
|
|
1488
|
+
};
|
|
1489
|
+
}
|
|
1490
|
+
function createEnumIndex() {
|
|
1491
|
+
const valueMap = /* @__PURE__ */ new Map();
|
|
1492
|
+
function allDocs() {
|
|
1493
|
+
const result = /* @__PURE__ */ new Set();
|
|
1494
|
+
for (const docSet of valueMap.values()) {
|
|
1495
|
+
for (const id of docSet) result.add(id);
|
|
1496
|
+
}
|
|
1497
|
+
return result;
|
|
1498
|
+
}
|
|
1499
|
+
return {
|
|
1500
|
+
insert(docId, value) {
|
|
1501
|
+
let docSet = valueMap.get(value);
|
|
1502
|
+
if (!docSet) {
|
|
1503
|
+
docSet = /* @__PURE__ */ new Set();
|
|
1504
|
+
valueMap.set(value, docSet);
|
|
1505
|
+
}
|
|
1506
|
+
docSet.add(docId);
|
|
1507
|
+
},
|
|
1508
|
+
remove(docId, value) {
|
|
1509
|
+
const docSet = valueMap.get(value);
|
|
1510
|
+
if (!docSet) return;
|
|
1511
|
+
docSet.delete(docId);
|
|
1512
|
+
if (docSet.size === 0) valueMap.delete(value);
|
|
1513
|
+
},
|
|
1514
|
+
queryEq(value) {
|
|
1515
|
+
const docSet = valueMap.get(value);
|
|
1516
|
+
return docSet ? new Set(docSet) : /* @__PURE__ */ new Set();
|
|
1517
|
+
},
|
|
1518
|
+
queryNe(value) {
|
|
1519
|
+
const excluded = valueMap.get(value);
|
|
1520
|
+
const result = allDocs();
|
|
1521
|
+
if (excluded) {
|
|
1522
|
+
for (const id of excluded) result.delete(id);
|
|
1523
|
+
}
|
|
1524
|
+
return result;
|
|
1525
|
+
},
|
|
1526
|
+
queryIn(values) {
|
|
1527
|
+
const result = /* @__PURE__ */ new Set();
|
|
1528
|
+
for (const val of values) {
|
|
1529
|
+
const docSet = valueMap.get(val);
|
|
1530
|
+
if (docSet) {
|
|
1531
|
+
for (const id of docSet) result.add(id);
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
return result;
|
|
1535
|
+
},
|
|
1536
|
+
queryNin(values) {
|
|
1537
|
+
const excluded = this.queryIn(values);
|
|
1538
|
+
const result = allDocs();
|
|
1539
|
+
for (const id of excluded) result.delete(id);
|
|
1540
|
+
return result;
|
|
1541
|
+
},
|
|
1542
|
+
getAllDocIds() {
|
|
1543
|
+
return allDocs();
|
|
1544
|
+
},
|
|
1545
|
+
count() {
|
|
1546
|
+
let total = 0;
|
|
1547
|
+
for (const docSet of valueMap.values()) total += docSet.size;
|
|
1548
|
+
return total;
|
|
1549
|
+
},
|
|
1550
|
+
clear() {
|
|
1551
|
+
valueMap.clear();
|
|
1552
|
+
},
|
|
1553
|
+
serialize() {
|
|
1554
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
1555
|
+
for (const [value, docSet] of valueMap) {
|
|
1556
|
+
result[value] = Array.from(docSet);
|
|
1557
|
+
}
|
|
1558
|
+
return result;
|
|
1559
|
+
},
|
|
1560
|
+
deserialize(data) {
|
|
1561
|
+
valueMap.clear();
|
|
1562
|
+
for (const value of Object.keys(data)) {
|
|
1563
|
+
valueMap.set(value, new Set(data[value]));
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
};
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
// src/core/partition/utils.ts
|
|
1570
|
+
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
1571
|
+
function getNestedValue(obj, path) {
|
|
1572
|
+
const parts = path.split(".");
|
|
1573
|
+
let current = obj;
|
|
1574
|
+
for (let i = 0; i < parts.length; i++) {
|
|
1575
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
1576
|
+
current = current[parts[i]];
|
|
1577
|
+
}
|
|
1578
|
+
return current;
|
|
1579
|
+
}
|
|
1580
|
+
function stringArraysEqual(a, b) {
|
|
1581
|
+
if (a === b) return true;
|
|
1582
|
+
if (!a || !b) return false;
|
|
1583
|
+
if (a.length !== b.length) return false;
|
|
1584
|
+
for (let i = 0; i < a.length; i++) {
|
|
1585
|
+
if (a[i] !== b[i]) return false;
|
|
1586
|
+
}
|
|
1587
|
+
return true;
|
|
1588
|
+
}
|
|
1589
|
+
function textFieldsChanged(oldDoc, newDoc, flatSchema) {
|
|
1590
|
+
for (const [path, fieldType] of Object.entries(flatSchema)) {
|
|
1591
|
+
if (fieldType !== "string" && fieldType !== "string[]") continue;
|
|
1592
|
+
const oldVal = getNestedValue(oldDoc, path);
|
|
1593
|
+
const newVal = getNestedValue(newDoc, path);
|
|
1594
|
+
if (fieldType === "string") {
|
|
1595
|
+
if (oldVal !== newVal) return true;
|
|
1596
|
+
} else {
|
|
1597
|
+
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
return false;
|
|
1601
|
+
}
|
|
1602
|
+
function parseVectorDimension(fieldType) {
|
|
1603
|
+
const match = VECTOR_PATTERN2.exec(fieldType);
|
|
1604
|
+
return match ? Number.parseInt(match[1], 10) : null;
|
|
1605
|
+
}
|
|
1606
|
+
function tokenizeOptions(options) {
|
|
1607
|
+
return {
|
|
1608
|
+
stem: true,
|
|
1609
|
+
removeStopWords: true,
|
|
1610
|
+
stopWordOverride: options?.stopWordOverride,
|
|
1611
|
+
customTokenizer: options?.customTokenizer
|
|
1612
|
+
};
|
|
1613
|
+
}
|
|
1614
|
+
function getFlatSchema(state, schema) {
|
|
1615
|
+
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
1616
|
+
state.flatSchemaCache = flattenSchema(schema);
|
|
1617
|
+
state.lastSchemaRef = schema;
|
|
1618
|
+
return state.flatSchemaCache;
|
|
1619
|
+
}
|
|
1620
|
+
function getFieldValueForDoc(docStore, docId, fieldPath) {
|
|
1621
|
+
const stored = docStore.get(docId);
|
|
1622
|
+
if (!stored) return void 0;
|
|
1623
|
+
return getNestedValue(stored.fields, fieldPath);
|
|
1624
|
+
}
|
|
1625
|
+
function getAllDocIds(docStore) {
|
|
1626
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1627
|
+
for (const [id] of docStore.all()) {
|
|
1628
|
+
ids.add(id);
|
|
1629
|
+
}
|
|
1630
|
+
return ids;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
// src/core/partition/indexing.ts
|
|
1634
|
+
function ensureFieldIndex(state, fieldPath, fieldType) {
|
|
1635
|
+
if ((fieldType === "number" || fieldType === "number[]") && !state.numericIndexes.has(fieldPath)) {
|
|
1636
|
+
state.numericIndexes.set(fieldPath, createNumericIndex());
|
|
1637
|
+
} else if ((fieldType === "boolean" || fieldType === "boolean[]") && !state.booleanIndexes.has(fieldPath)) {
|
|
1638
|
+
state.booleanIndexes.set(fieldPath, createBooleanIndex());
|
|
1639
|
+
} else if ((fieldType === "enum" || fieldType === "enum[]") && !state.enumIndexes.has(fieldPath)) {
|
|
1640
|
+
state.enumIndexes.set(fieldPath, createEnumIndex());
|
|
1641
|
+
} else if (fieldType === "geopoint" && !state.geoIndexes.has(fieldPath)) {
|
|
1642
|
+
state.geoIndexes.set(fieldPath, createGeoIndex());
|
|
1643
|
+
} else {
|
|
1644
|
+
const dim = parseVectorDimension(fieldType);
|
|
1645
|
+
if (dim !== null && !state.vectorStores.has(fieldPath)) {
|
|
1646
|
+
state.vectorStores.set(fieldPath, createBruteForceVectorStore(dim));
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
function getOrCreateNumericIndex(state, fieldPath) {
|
|
1651
|
+
let idx = state.numericIndexes.get(fieldPath);
|
|
1652
|
+
if (!idx) {
|
|
1653
|
+
idx = createNumericIndex();
|
|
1654
|
+
state.numericIndexes.set(fieldPath, idx);
|
|
1655
|
+
}
|
|
1656
|
+
return idx;
|
|
1657
|
+
}
|
|
1658
|
+
function getOrCreateBooleanIndex(state, fieldPath) {
|
|
1659
|
+
let idx = state.booleanIndexes.get(fieldPath);
|
|
1660
|
+
if (!idx) {
|
|
1661
|
+
idx = createBooleanIndex();
|
|
1662
|
+
state.booleanIndexes.set(fieldPath, idx);
|
|
1663
|
+
}
|
|
1664
|
+
return idx;
|
|
1665
|
+
}
|
|
1666
|
+
function indexStringField(state, docId, fieldPath, text, language, options, fieldLengths, tokensByField) {
|
|
1667
|
+
const result = tokenize(text, language, tokenizeOptions(options));
|
|
1668
|
+
fieldLengths[fieldPath] = result.tokens.length;
|
|
1669
|
+
const fieldTokenList = [];
|
|
1670
|
+
const tokenFreqs = /* @__PURE__ */ new Map();
|
|
1671
|
+
for (const t of result.tokens) {
|
|
1672
|
+
const existing = tokenFreqs.get(t.token);
|
|
1673
|
+
if (existing) {
|
|
1674
|
+
existing.count++;
|
|
1675
|
+
existing.positions.push(t.position);
|
|
1676
|
+
} else {
|
|
1677
|
+
tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
|
|
1678
|
+
}
|
|
1679
|
+
fieldTokenList.push(t.token);
|
|
1680
|
+
}
|
|
1681
|
+
for (const [token, freq] of tokenFreqs) {
|
|
1682
|
+
state.invertedIdx.insert(token, {
|
|
1683
|
+
docId,
|
|
1684
|
+
termFrequency: freq.count,
|
|
1685
|
+
fieldName: fieldPath,
|
|
1686
|
+
positions: freq.positions
|
|
1687
|
+
});
|
|
1688
|
+
}
|
|
1689
|
+
tokensByField[fieldPath] = fieldTokenList;
|
|
1690
|
+
}
|
|
1691
|
+
function indexStringArrayField(state, docId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
|
|
1692
|
+
const tokenFreqs = /* @__PURE__ */ new Map();
|
|
1693
|
+
const fieldTokenList = [];
|
|
1694
|
+
let positionOffset = 0;
|
|
1695
|
+
for (const item of arr) {
|
|
1696
|
+
const result = tokenize(item, language, tokenizeOptions(options));
|
|
1697
|
+
for (const t of result.tokens) {
|
|
1698
|
+
const existing = tokenFreqs.get(t.token);
|
|
1699
|
+
if (existing) {
|
|
1700
|
+
existing.count++;
|
|
1701
|
+
existing.positions.push(positionOffset + t.position);
|
|
1702
|
+
} else {
|
|
1703
|
+
tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
|
|
1704
|
+
}
|
|
1705
|
+
fieldTokenList.push(t.token);
|
|
1706
|
+
}
|
|
1707
|
+
positionOffset += result.tokens.length;
|
|
1708
|
+
}
|
|
1709
|
+
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
1710
|
+
for (const [token, freq] of tokenFreqs) {
|
|
1711
|
+
state.invertedIdx.insert(token, {
|
|
1712
|
+
docId,
|
|
1713
|
+
termFrequency: freq.count,
|
|
1714
|
+
fieldName: fieldPath,
|
|
1715
|
+
positions: freq.positions
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1718
|
+
tokensByField[fieldPath] = fieldTokenList;
|
|
1719
|
+
}
|
|
1720
|
+
function indexDocument(state, docId, document, flatSchema, language, options) {
|
|
1721
|
+
const fieldLengths = {};
|
|
1722
|
+
const tokensByField = {};
|
|
1723
|
+
for (const [fieldPath, fieldType] of Object.entries(flatSchema)) {
|
|
1724
|
+
const value = getNestedValue(document, fieldPath);
|
|
1725
|
+
if (value === void 0 || value === null) continue;
|
|
1726
|
+
ensureFieldIndex(state, fieldPath, fieldType);
|
|
1727
|
+
if (fieldType === "string") {
|
|
1728
|
+
indexStringField(state, docId, fieldPath, value, language, options, fieldLengths, tokensByField);
|
|
1729
|
+
} else if (fieldType === "number") {
|
|
1730
|
+
getOrCreateNumericIndex(state, fieldPath).insert(docId, value);
|
|
1731
|
+
} else if (fieldType === "boolean") {
|
|
1732
|
+
getOrCreateBooleanIndex(state, fieldPath).insert(docId, value);
|
|
1733
|
+
} else if (fieldType === "enum") {
|
|
1734
|
+
state.enumIndexes.get(fieldPath)?.insert(docId, value);
|
|
1735
|
+
} else if (fieldType === "geopoint") {
|
|
1736
|
+
const geo = value;
|
|
1737
|
+
state.geoIndexes.get(fieldPath)?.insert(docId, geo.lat, geo.lon);
|
|
1738
|
+
} else if (fieldType === "string[]") {
|
|
1739
|
+
indexStringArrayField(state, docId, fieldPath, value, language, options, fieldLengths, tokensByField);
|
|
1740
|
+
} else if (fieldType === "number[]") {
|
|
1741
|
+
const numIdx = getOrCreateNumericIndex(state, fieldPath);
|
|
1742
|
+
for (const num of value) numIdx.insert(docId, num);
|
|
1743
|
+
} else if (fieldType === "boolean[]") {
|
|
1744
|
+
const boolIdx = getOrCreateBooleanIndex(state, fieldPath);
|
|
1745
|
+
for (const b of value) boolIdx.insert(docId, b);
|
|
1746
|
+
} else if (fieldType === "enum[]") {
|
|
1747
|
+
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
1748
|
+
if (enumIdx) {
|
|
1749
|
+
for (const e of value) enumIdx.insert(docId, e);
|
|
1750
|
+
}
|
|
1751
|
+
} else {
|
|
1752
|
+
const dim = parseVectorDimension(fieldType);
|
|
1753
|
+
if (dim !== null) {
|
|
1754
|
+
const vec = value instanceof Float32Array ? value : new Float32Array(value);
|
|
1755
|
+
state.vectorStores.get(fieldPath)?.insert(docId, vec);
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
return { fieldLengths, tokensByField };
|
|
1760
|
+
}
|
|
1761
|
+
function removeFromIndexes(state, docId, storedDoc, flatSchema, language, options) {
|
|
1762
|
+
const fieldLengths = {};
|
|
1763
|
+
const tokensByField = {};
|
|
1764
|
+
const fields = storedDoc.fields;
|
|
1765
|
+
for (const [fieldPath, fieldType] of Object.entries(flatSchema)) {
|
|
1766
|
+
const value = getNestedValue(fields, fieldPath);
|
|
1767
|
+
if (value === void 0 || value === null) continue;
|
|
1768
|
+
if (fieldType === "string") {
|
|
1769
|
+
const result = tokenize(value, language, tokenizeOptions(options));
|
|
1770
|
+
fieldLengths[fieldPath] = result.tokens.length;
|
|
1771
|
+
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
1772
|
+
const fieldTokenList = [];
|
|
1773
|
+
for (const t of result.tokens) {
|
|
1774
|
+
uniqueTokens.add(t.token);
|
|
1775
|
+
fieldTokenList.push(t.token);
|
|
1776
|
+
}
|
|
1777
|
+
for (const token of uniqueTokens) state.invertedIdx.remove(token, docId);
|
|
1778
|
+
tokensByField[fieldPath] = fieldTokenList;
|
|
1779
|
+
} else if (fieldType === "number") {
|
|
1780
|
+
state.numericIndexes.get(fieldPath)?.remove(docId, value);
|
|
1781
|
+
} else if (fieldType === "boolean") {
|
|
1782
|
+
state.booleanIndexes.get(fieldPath)?.remove(docId, value);
|
|
1783
|
+
} else if (fieldType === "enum") {
|
|
1784
|
+
state.enumIndexes.get(fieldPath)?.remove(docId, value);
|
|
1785
|
+
} else if (fieldType === "geopoint") {
|
|
1786
|
+
state.geoIndexes.get(fieldPath)?.remove(docId);
|
|
1787
|
+
} else if (fieldType === "string[]") {
|
|
1788
|
+
const arr = value;
|
|
1789
|
+
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
1790
|
+
const fieldTokenList = [];
|
|
1791
|
+
for (const item of arr) {
|
|
1792
|
+
const result = tokenize(item, language, tokenizeOptions(options));
|
|
1793
|
+
for (const t of result.tokens) {
|
|
1794
|
+
uniqueTokens.add(t.token);
|
|
1795
|
+
fieldTokenList.push(t.token);
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
1799
|
+
for (const token of uniqueTokens) state.invertedIdx.remove(token, docId);
|
|
1800
|
+
tokensByField[fieldPath] = fieldTokenList;
|
|
1801
|
+
} else if (fieldType === "number[]") {
|
|
1802
|
+
const numIdx = state.numericIndexes.get(fieldPath);
|
|
1803
|
+
if (numIdx) {
|
|
1804
|
+
for (const num of value) numIdx.remove(docId, num);
|
|
1805
|
+
}
|
|
1806
|
+
} else if (fieldType === "boolean[]") {
|
|
1807
|
+
const boolIdx = state.booleanIndexes.get(fieldPath);
|
|
1808
|
+
if (boolIdx) {
|
|
1809
|
+
for (const b of value) boolIdx.remove(docId, b);
|
|
1810
|
+
}
|
|
1811
|
+
} else if (fieldType === "enum[]") {
|
|
1812
|
+
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
1813
|
+
if (enumIdx) {
|
|
1814
|
+
for (const e of value) enumIdx.remove(docId, e);
|
|
1815
|
+
}
|
|
1816
|
+
} else {
|
|
1817
|
+
if (parseVectorDimension(fieldType) !== null) {
|
|
1818
|
+
state.vectorStores.get(fieldPath)?.remove(docId);
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
return { fieldLengths, tokensByField };
|
|
1823
|
+
}
|
|
1824
|
+
function updateFieldIndexOnly(state, docId, oldFields, newDoc, flatSchema) {
|
|
1825
|
+
for (const [fieldPath, fieldType] of Object.entries(flatSchema)) {
|
|
1826
|
+
if (fieldType === "string" || fieldType === "string[]") continue;
|
|
1827
|
+
const oldVal = getNestedValue(oldFields, fieldPath);
|
|
1828
|
+
const newVal = getNestedValue(newDoc, fieldPath);
|
|
1829
|
+
if (oldVal === newVal) continue;
|
|
1830
|
+
if (oldVal !== void 0 && oldVal !== null && newVal !== void 0 && newVal !== null && typeof oldVal === typeof newVal && JSON.stringify(oldVal) === JSON.stringify(newVal))
|
|
1831
|
+
continue;
|
|
1832
|
+
if (newVal !== void 0 && newVal !== null) {
|
|
1833
|
+
ensureFieldIndex(state, fieldPath, fieldType);
|
|
1834
|
+
}
|
|
1835
|
+
if (fieldType === "number") {
|
|
1836
|
+
const numIdx = state.numericIndexes.get(fieldPath);
|
|
1837
|
+
if (numIdx) {
|
|
1838
|
+
if (oldVal !== void 0 && oldVal !== null) numIdx.remove(docId, oldVal);
|
|
1839
|
+
if (newVal !== void 0 && newVal !== null) numIdx.insert(docId, newVal);
|
|
1840
|
+
}
|
|
1841
|
+
} else if (fieldType === "boolean") {
|
|
1842
|
+
const boolIdx = state.booleanIndexes.get(fieldPath);
|
|
1843
|
+
if (boolIdx) {
|
|
1844
|
+
if (oldVal !== void 0 && oldVal !== null) boolIdx.remove(docId, oldVal);
|
|
1845
|
+
if (newVal !== void 0 && newVal !== null) boolIdx.insert(docId, newVal);
|
|
1846
|
+
}
|
|
1847
|
+
} else if (fieldType === "enum") {
|
|
1848
|
+
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
1849
|
+
if (enumIdx) {
|
|
1850
|
+
if (oldVal !== void 0 && oldVal !== null) enumIdx.remove(docId, oldVal);
|
|
1851
|
+
if (newVal !== void 0 && newVal !== null) enumIdx.insert(docId, newVal);
|
|
1852
|
+
}
|
|
1853
|
+
} else if (fieldType === "geopoint") {
|
|
1854
|
+
const geoIdx = state.geoIndexes.get(fieldPath);
|
|
1855
|
+
if (geoIdx) {
|
|
1856
|
+
if (oldVal !== void 0 && oldVal !== null) geoIdx.remove(docId);
|
|
1857
|
+
if (newVal !== void 0 && newVal !== null) {
|
|
1858
|
+
const geo = newVal;
|
|
1859
|
+
geoIdx.insert(docId, geo.lat, geo.lon);
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1862
|
+
} else if (fieldType === "number[]") {
|
|
1863
|
+
const numIdx = state.numericIndexes.get(fieldPath);
|
|
1864
|
+
if (numIdx) {
|
|
1865
|
+
if (oldVal !== void 0 && oldVal !== null) {
|
|
1866
|
+
for (const n of oldVal) numIdx.remove(docId, n);
|
|
1867
|
+
}
|
|
1868
|
+
if (newVal !== void 0 && newVal !== null) {
|
|
1869
|
+
for (const n of newVal) numIdx.insert(docId, n);
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
} else if (fieldType === "boolean[]") {
|
|
1873
|
+
const boolIdx = state.booleanIndexes.get(fieldPath);
|
|
1874
|
+
if (boolIdx) {
|
|
1875
|
+
if (oldVal !== void 0 && oldVal !== null) {
|
|
1876
|
+
for (const b of oldVal) boolIdx.remove(docId, b);
|
|
1877
|
+
}
|
|
1878
|
+
if (newVal !== void 0 && newVal !== null) {
|
|
1879
|
+
for (const b of newVal) boolIdx.insert(docId, b);
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
} else if (fieldType === "enum[]") {
|
|
1883
|
+
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
1884
|
+
if (enumIdx) {
|
|
1885
|
+
if (oldVal !== void 0 && oldVal !== null) {
|
|
1886
|
+
for (const e of oldVal) enumIdx.remove(docId, e);
|
|
1887
|
+
}
|
|
1888
|
+
if (newVal !== void 0 && newVal !== null) {
|
|
1889
|
+
for (const e of newVal) enumIdx.insert(docId, e);
|
|
1890
|
+
}
|
|
1891
|
+
}
|
|
1892
|
+
} else if (parseVectorDimension(fieldType) !== null) {
|
|
1893
|
+
const vecStore = state.vectorStores.get(fieldPath);
|
|
1894
|
+
if (vecStore) {
|
|
1895
|
+
vecStore.remove(docId);
|
|
1896
|
+
if (newVal !== void 0 && newVal !== null) {
|
|
1897
|
+
const vec = newVal instanceof Float32Array ? newVal : new Float32Array(newVal);
|
|
1898
|
+
vecStore.insert(docId, vec);
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
// src/filters/combinators.ts
|
|
1906
|
+
function applyAnd(sets) {
|
|
1907
|
+
if (sets.length === 0) return /* @__PURE__ */ new Set();
|
|
1908
|
+
if (sets.length === 1) return sets[0];
|
|
1909
|
+
const sorted = [...sets].sort((a, b) => a.size - b.size);
|
|
1910
|
+
let result = sorted[0];
|
|
1911
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1912
|
+
const next = sorted[i];
|
|
1913
|
+
const intersection = /* @__PURE__ */ new Set();
|
|
1914
|
+
for (const item of result) {
|
|
1915
|
+
if (next.has(item)) intersection.add(item);
|
|
1916
|
+
}
|
|
1917
|
+
result = intersection;
|
|
1918
|
+
if (result.size === 0) break;
|
|
1919
|
+
}
|
|
1920
|
+
return result;
|
|
1921
|
+
}
|
|
1922
|
+
function applyOr(sets) {
|
|
1923
|
+
if (sets.length === 0) return /* @__PURE__ */ new Set();
|
|
1924
|
+
if (sets.length === 1) return sets[0];
|
|
1925
|
+
const result = /* @__PURE__ */ new Set();
|
|
1926
|
+
for (const set of sets) {
|
|
1927
|
+
for (const item of set) {
|
|
1928
|
+
result.add(item);
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
return result;
|
|
1932
|
+
}
|
|
1933
|
+
function applyNot(universe, excluded) {
|
|
1934
|
+
const result = /* @__PURE__ */ new Set();
|
|
1935
|
+
for (const item of universe) {
|
|
1936
|
+
if (!excluded.has(item)) result.add(item);
|
|
1937
|
+
}
|
|
1938
|
+
return result;
|
|
1939
|
+
}
|
|
1940
|
+
|
|
1941
|
+
// src/filters/operators.ts
|
|
1942
|
+
var METERS_PER_KM = 1e3;
|
|
1943
|
+
var METERS_PER_MI = 1609.344;
|
|
1944
|
+
function convertToMeters(distance, unit) {
|
|
1945
|
+
if (unit === "km") return distance * METERS_PER_KM;
|
|
1946
|
+
if (unit === "mi") return distance * METERS_PER_MI;
|
|
1947
|
+
return distance;
|
|
1948
|
+
}
|
|
1949
|
+
function setDifference(a, b) {
|
|
1950
|
+
const result = /* @__PURE__ */ new Set();
|
|
1951
|
+
for (const item of a) {
|
|
1952
|
+
if (!b.has(item)) result.add(item);
|
|
1953
|
+
}
|
|
1954
|
+
return result;
|
|
1955
|
+
}
|
|
1956
|
+
function applyEq(value, docIds, getValue, fieldIndex) {
|
|
1957
|
+
if (fieldIndex?.type === "numeric" && typeof value === "number") {
|
|
1958
|
+
return fieldIndex.index.eq(value);
|
|
1959
|
+
}
|
|
1960
|
+
if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
|
|
1961
|
+
return value ? fieldIndex.index.getTrue() : fieldIndex.index.getFalse();
|
|
1962
|
+
}
|
|
1963
|
+
if (fieldIndex?.type === "enum" && typeof value === "string") {
|
|
1964
|
+
return fieldIndex.index.getDocIds(value);
|
|
1965
|
+
}
|
|
1966
|
+
const result = /* @__PURE__ */ new Set();
|
|
1967
|
+
for (const docId of docIds) {
|
|
1968
|
+
if (getValue(docId) === value) result.add(docId);
|
|
1969
|
+
}
|
|
1970
|
+
return result;
|
|
1971
|
+
}
|
|
1972
|
+
function applyNe(value, docIds, getValue, fieldIndex) {
|
|
1973
|
+
if (fieldIndex?.type === "numeric" && typeof value === "number") {
|
|
1974
|
+
return setDifference(fieldIndex.index.allDocIds(), fieldIndex.index.eq(value));
|
|
1975
|
+
}
|
|
1976
|
+
if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
|
|
1977
|
+
return value ? fieldIndex.index.getFalse() : fieldIndex.index.getTrue();
|
|
1978
|
+
}
|
|
1979
|
+
if (fieldIndex?.type === "enum" && typeof value === "string") {
|
|
1980
|
+
return setDifference(fieldIndex.index.allDocIds(), fieldIndex.index.getDocIds(value));
|
|
1981
|
+
}
|
|
1982
|
+
const result = /* @__PURE__ */ new Set();
|
|
1983
|
+
for (const docId of docIds) {
|
|
1984
|
+
const v = getValue(docId);
|
|
1985
|
+
if (v !== void 0 && v !== null && v !== value) result.add(docId);
|
|
1986
|
+
}
|
|
1987
|
+
return result;
|
|
1988
|
+
}
|
|
1989
|
+
function applyGt(value, docIds, getValue, fieldIndex) {
|
|
1990
|
+
if (fieldIndex?.type === "numeric") {
|
|
1991
|
+
return fieldIndex.index.gt(value);
|
|
1992
|
+
}
|
|
1993
|
+
const result = /* @__PURE__ */ new Set();
|
|
1994
|
+
for (const docId of docIds) {
|
|
1995
|
+
const v = getValue(docId);
|
|
1996
|
+
if (typeof v === "number" && v > value) result.add(docId);
|
|
1997
|
+
}
|
|
1998
|
+
return result;
|
|
1999
|
+
}
|
|
2000
|
+
function applyLt(value, docIds, getValue, fieldIndex) {
|
|
2001
|
+
if (fieldIndex?.type === "numeric") {
|
|
2002
|
+
return fieldIndex.index.lt(value);
|
|
2003
|
+
}
|
|
2004
|
+
const result = /* @__PURE__ */ new Set();
|
|
2005
|
+
for (const docId of docIds) {
|
|
2006
|
+
const v = getValue(docId);
|
|
2007
|
+
if (typeof v === "number" && v < value) result.add(docId);
|
|
2008
|
+
}
|
|
2009
|
+
return result;
|
|
2010
|
+
}
|
|
2011
|
+
function applyGte(value, docIds, getValue, fieldIndex) {
|
|
2012
|
+
if (fieldIndex?.type === "numeric") {
|
|
2013
|
+
return fieldIndex.index.gte(value);
|
|
2014
|
+
}
|
|
2015
|
+
const result = /* @__PURE__ */ new Set();
|
|
2016
|
+
for (const docId of docIds) {
|
|
2017
|
+
const v = getValue(docId);
|
|
2018
|
+
if (typeof v === "number" && v >= value) result.add(docId);
|
|
2019
|
+
}
|
|
2020
|
+
return result;
|
|
2021
|
+
}
|
|
2022
|
+
function applyLte(value, docIds, getValue, fieldIndex) {
|
|
2023
|
+
if (fieldIndex?.type === "numeric") {
|
|
2024
|
+
return fieldIndex.index.lte(value);
|
|
2025
|
+
}
|
|
2026
|
+
const result = /* @__PURE__ */ new Set();
|
|
2027
|
+
for (const docId of docIds) {
|
|
2028
|
+
const v = getValue(docId);
|
|
2029
|
+
if (typeof v === "number" && v <= value) result.add(docId);
|
|
2030
|
+
}
|
|
2031
|
+
return result;
|
|
2032
|
+
}
|
|
2033
|
+
function applyBetween(range, docIds, getValue, fieldIndex) {
|
|
2034
|
+
if (fieldIndex?.type === "numeric") {
|
|
2035
|
+
return fieldIndex.index.between(range[0], range[1]);
|
|
2036
|
+
}
|
|
2037
|
+
const result = /* @__PURE__ */ new Set();
|
|
2038
|
+
for (const docId of docIds) {
|
|
2039
|
+
const v = getValue(docId);
|
|
2040
|
+
if (typeof v === "number" && v >= range[0] && v <= range[1]) result.add(docId);
|
|
2041
|
+
}
|
|
2042
|
+
return result;
|
|
2043
|
+
}
|
|
2044
|
+
function applyIn(values, docIds, getValue, fieldIndex) {
|
|
2045
|
+
if (fieldIndex?.type === "enum") {
|
|
2046
|
+
const result2 = /* @__PURE__ */ new Set();
|
|
2047
|
+
for (const val of values) {
|
|
2048
|
+
for (const docId of fieldIndex.index.getDocIds(val)) {
|
|
2049
|
+
result2.add(docId);
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
return result2;
|
|
2053
|
+
}
|
|
2054
|
+
const valSet = new Set(values);
|
|
2055
|
+
const result = /* @__PURE__ */ new Set();
|
|
2056
|
+
for (const docId of docIds) {
|
|
2057
|
+
const v = getValue(docId);
|
|
2058
|
+
if (typeof v === "string" && valSet.has(v)) result.add(docId);
|
|
2059
|
+
}
|
|
2060
|
+
return result;
|
|
2061
|
+
}
|
|
2062
|
+
function applyNin(values, docIds, getValue, fieldIndex) {
|
|
2063
|
+
if (fieldIndex?.type === "enum") {
|
|
2064
|
+
const matched = /* @__PURE__ */ new Set();
|
|
2065
|
+
for (const val of values) {
|
|
2066
|
+
for (const docId of fieldIndex.index.getDocIds(val)) {
|
|
2067
|
+
matched.add(docId);
|
|
2068
|
+
}
|
|
2069
|
+
}
|
|
2070
|
+
return setDifference(fieldIndex.index.allDocIds(), matched);
|
|
2071
|
+
}
|
|
2072
|
+
const valSet = new Set(values);
|
|
2073
|
+
const result = /* @__PURE__ */ new Set();
|
|
2074
|
+
for (const docId of docIds) {
|
|
2075
|
+
const v = getValue(docId);
|
|
2076
|
+
if (v !== void 0 && v !== null && typeof v === "string" && !valSet.has(v)) result.add(docId);
|
|
2077
|
+
}
|
|
2078
|
+
return result;
|
|
2079
|
+
}
|
|
2080
|
+
function applyStartsWith(prefix, docIds, getValue) {
|
|
2081
|
+
const result = /* @__PURE__ */ new Set();
|
|
2082
|
+
for (const docId of docIds) {
|
|
2083
|
+
const v = getValue(docId);
|
|
2084
|
+
if (typeof v === "string" && v.startsWith(prefix)) result.add(docId);
|
|
2085
|
+
}
|
|
2086
|
+
return result;
|
|
2087
|
+
}
|
|
2088
|
+
function applyEndsWith(suffix, docIds, getValue) {
|
|
2089
|
+
const result = /* @__PURE__ */ new Set();
|
|
2090
|
+
for (const docId of docIds) {
|
|
2091
|
+
const v = getValue(docId);
|
|
2092
|
+
if (typeof v === "string" && v.endsWith(suffix)) result.add(docId);
|
|
2093
|
+
}
|
|
2094
|
+
return result;
|
|
2095
|
+
}
|
|
2096
|
+
function applyContainsAll(values, docIds, getValue) {
|
|
2097
|
+
const result = /* @__PURE__ */ new Set();
|
|
2098
|
+
for (const docId of docIds) {
|
|
2099
|
+
const v = getValue(docId);
|
|
2100
|
+
if (!Array.isArray(v)) continue;
|
|
2101
|
+
const arr = v;
|
|
2102
|
+
let allFound = true;
|
|
2103
|
+
for (const val of values) {
|
|
2104
|
+
if (!arr.includes(val)) {
|
|
2105
|
+
allFound = false;
|
|
2106
|
+
break;
|
|
2107
|
+
}
|
|
2108
|
+
}
|
|
2109
|
+
if (allFound) result.add(docId);
|
|
2110
|
+
}
|
|
2111
|
+
return result;
|
|
2112
|
+
}
|
|
2113
|
+
function applyMatchesAny(values, docIds, getValue) {
|
|
2114
|
+
const valSet = new Set(values);
|
|
2115
|
+
const result = /* @__PURE__ */ new Set();
|
|
2116
|
+
for (const docId of docIds) {
|
|
2117
|
+
const v = getValue(docId);
|
|
2118
|
+
if (!Array.isArray(v)) continue;
|
|
2119
|
+
for (const item of v) {
|
|
2120
|
+
if (valSet.has(item)) {
|
|
2121
|
+
result.add(docId);
|
|
2122
|
+
break;
|
|
2123
|
+
}
|
|
2124
|
+
}
|
|
2125
|
+
}
|
|
2126
|
+
return result;
|
|
2127
|
+
}
|
|
2128
|
+
function applySize(sizeFilter, docIds, getValue) {
|
|
2129
|
+
const result = /* @__PURE__ */ new Set();
|
|
2130
|
+
for (const docId of docIds) {
|
|
2131
|
+
const v = getValue(docId);
|
|
2132
|
+
if (!Array.isArray(v)) continue;
|
|
2133
|
+
if (matchesNumericComparison(v.length, sizeFilter)) result.add(docId);
|
|
2134
|
+
}
|
|
2135
|
+
return result;
|
|
2136
|
+
}
|
|
2137
|
+
function applyExists(docIds, getValue) {
|
|
2138
|
+
const result = /* @__PURE__ */ new Set();
|
|
2139
|
+
for (const docId of docIds) {
|
|
2140
|
+
const v = getValue(docId);
|
|
2141
|
+
if (v !== void 0 && v !== null) result.add(docId);
|
|
2142
|
+
}
|
|
2143
|
+
return result;
|
|
2144
|
+
}
|
|
2145
|
+
function applyNotExists(docIds, getValue) {
|
|
2146
|
+
const result = /* @__PURE__ */ new Set();
|
|
2147
|
+
for (const docId of docIds) {
|
|
2148
|
+
const v = getValue(docId);
|
|
2149
|
+
if (v === void 0 || v === null) result.add(docId);
|
|
2150
|
+
}
|
|
2151
|
+
return result;
|
|
2152
|
+
}
|
|
2153
|
+
function applyIsEmpty(docIds, getValue) {
|
|
2154
|
+
const result = /* @__PURE__ */ new Set();
|
|
2155
|
+
for (const docId of docIds) {
|
|
2156
|
+
const v = getValue(docId);
|
|
2157
|
+
if (v === void 0 || v === null) {
|
|
2158
|
+
result.add(docId);
|
|
2159
|
+
continue;
|
|
2160
|
+
}
|
|
2161
|
+
if (typeof v === "string" && v === "") {
|
|
2162
|
+
result.add(docId);
|
|
2163
|
+
continue;
|
|
2164
|
+
}
|
|
2165
|
+
if (Array.isArray(v) && v.length === 0) {
|
|
2166
|
+
result.add(docId);
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
2169
|
+
return result;
|
|
2170
|
+
}
|
|
2171
|
+
function applyIsNotEmpty(docIds, getValue) {
|
|
2172
|
+
const result = /* @__PURE__ */ new Set();
|
|
2173
|
+
for (const docId of docIds) {
|
|
2174
|
+
const v = getValue(docId);
|
|
2175
|
+
if (v === void 0 || v === null) continue;
|
|
2176
|
+
if (typeof v === "string" && v === "") continue;
|
|
2177
|
+
if (Array.isArray(v) && v.length === 0) continue;
|
|
2178
|
+
result.add(docId);
|
|
2179
|
+
}
|
|
2180
|
+
return result;
|
|
2181
|
+
}
|
|
2182
|
+
function applyGeoRadius(filter, geoIndex) {
|
|
2183
|
+
const distanceMeters = convertToMeters(filter.distance, filter.unit);
|
|
2184
|
+
return geoIndex.radiusQuery(
|
|
2185
|
+
filter.lat,
|
|
2186
|
+
filter.lon,
|
|
2187
|
+
distanceMeters,
|
|
2188
|
+
filter.inside ?? true,
|
|
2189
|
+
filter.highPrecision ?? false
|
|
2190
|
+
);
|
|
2191
|
+
}
|
|
2192
|
+
function applyGeoPolygon(filter, geoIndex) {
|
|
2193
|
+
return geoIndex.polygonQuery(filter.points, filter.inside ?? true);
|
|
2194
|
+
}
|
|
2195
|
+
function matchesNumericComparison(value, filter) {
|
|
2196
|
+
if (filter.eq !== void 0 && value !== filter.eq) return false;
|
|
2197
|
+
if (filter.ne !== void 0 && value === filter.ne) return false;
|
|
2198
|
+
if (filter.gt !== void 0 && !(value > filter.gt)) return false;
|
|
2199
|
+
if (filter.lt !== void 0 && !(value < filter.lt)) return false;
|
|
2200
|
+
if (filter.gte !== void 0 && !(value >= filter.gte)) return false;
|
|
2201
|
+
if (filter.lte !== void 0 && !(value <= filter.lte)) return false;
|
|
2202
|
+
if (filter.between !== void 0 && !(value >= filter.between[0] && value <= filter.between[1])) return false;
|
|
2203
|
+
return true;
|
|
2204
|
+
}
|
|
2205
|
+
|
|
2206
|
+
// src/filters/evaluator.ts
|
|
2207
|
+
function evaluateFilters(expression, context) {
|
|
2208
|
+
const sets = [];
|
|
2209
|
+
if (expression.fields) {
|
|
2210
|
+
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
2211
|
+
sets.push(evaluateFieldFilter(fieldPath, filter, context));
|
|
2212
|
+
}
|
|
2213
|
+
}
|
|
2214
|
+
if (expression.and?.length) {
|
|
2215
|
+
const andSets = expression.and.map((expr) => evaluateFilters(expr, context));
|
|
2216
|
+
sets.push(applyAnd(andSets));
|
|
2217
|
+
}
|
|
2218
|
+
if (expression.or?.length) {
|
|
2219
|
+
const orSets = expression.or.map((expr) => evaluateFilters(expr, context));
|
|
2220
|
+
sets.push(applyOr(orSets));
|
|
2221
|
+
}
|
|
2222
|
+
if (expression.not) {
|
|
2223
|
+
const excluded = evaluateFilters(expression.not, context);
|
|
2224
|
+
sets.push(applyNot(context.allDocIds, excluded));
|
|
2225
|
+
}
|
|
2226
|
+
if (sets.length === 0) return new Set(context.allDocIds);
|
|
2227
|
+
if (sets.length === 1) return sets[0];
|
|
2228
|
+
return applyAnd(sets);
|
|
2229
|
+
}
|
|
2230
|
+
function evaluateFieldFilter(fieldPath, filter, context) {
|
|
2231
|
+
const fieldIndex = context.fieldIndexes[fieldPath];
|
|
2232
|
+
const getValue = (docId) => context.getFieldValue(docId, fieldPath);
|
|
2233
|
+
const f = filter;
|
|
2234
|
+
const sets = [];
|
|
2235
|
+
if ("radius" in f && f.radius) {
|
|
2236
|
+
const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
|
|
2237
|
+
sets.push(applyGeoRadius(filter.radius, geoIndex));
|
|
2238
|
+
}
|
|
2239
|
+
if ("polygon" in f && f.polygon) {
|
|
2240
|
+
const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
|
|
2241
|
+
sets.push(applyGeoPolygon(filter.polygon, geoIndex));
|
|
2242
|
+
}
|
|
2243
|
+
if ("eq" in f && f.eq !== void 0) {
|
|
2244
|
+
sets.push(applyEq(f.eq, context.allDocIds, getValue, fieldIndex));
|
|
2245
|
+
}
|
|
2246
|
+
if ("ne" in f && f.ne !== void 0) {
|
|
2247
|
+
sets.push(applyNe(f.ne, context.allDocIds, getValue, fieldIndex));
|
|
2248
|
+
}
|
|
2249
|
+
if ("gt" in f && f.gt !== void 0) {
|
|
2250
|
+
sets.push(applyGt(f.gt, context.allDocIds, getValue, fieldIndex));
|
|
2251
|
+
}
|
|
2252
|
+
if ("lt" in f && f.lt !== void 0) {
|
|
2253
|
+
sets.push(applyLt(f.lt, context.allDocIds, getValue, fieldIndex));
|
|
2254
|
+
}
|
|
2255
|
+
if ("gte" in f && f.gte !== void 0) {
|
|
2256
|
+
sets.push(applyGte(f.gte, context.allDocIds, getValue, fieldIndex));
|
|
2257
|
+
}
|
|
2258
|
+
if ("lte" in f && f.lte !== void 0) {
|
|
2259
|
+
sets.push(applyLte(f.lte, context.allDocIds, getValue, fieldIndex));
|
|
2260
|
+
}
|
|
2261
|
+
if ("between" in f && f.between !== void 0) {
|
|
2262
|
+
sets.push(applyBetween(f.between, context.allDocIds, getValue, fieldIndex));
|
|
2263
|
+
}
|
|
2264
|
+
if ("in" in f && f.in !== void 0) {
|
|
2265
|
+
sets.push(applyIn(f.in, context.allDocIds, getValue, fieldIndex));
|
|
2266
|
+
}
|
|
2267
|
+
if ("nin" in f && f.nin !== void 0) {
|
|
2268
|
+
sets.push(applyNin(f.nin, context.allDocIds, getValue, fieldIndex));
|
|
2269
|
+
}
|
|
2270
|
+
if ("startsWith" in f && f.startsWith !== void 0) {
|
|
2271
|
+
sets.push(applyStartsWith(f.startsWith, context.allDocIds, getValue));
|
|
2272
|
+
}
|
|
2273
|
+
if ("endsWith" in f && f.endsWith !== void 0) {
|
|
2274
|
+
sets.push(applyEndsWith(f.endsWith, context.allDocIds, getValue));
|
|
2275
|
+
}
|
|
2276
|
+
if ("containsAll" in f && f.containsAll !== void 0) {
|
|
2277
|
+
sets.push(applyContainsAll(f.containsAll, context.allDocIds, getValue));
|
|
2278
|
+
}
|
|
2279
|
+
if ("matchesAny" in f && f.matchesAny !== void 0) {
|
|
2280
|
+
sets.push(applyMatchesAny(f.matchesAny, context.allDocIds, getValue));
|
|
2281
|
+
}
|
|
2282
|
+
if ("size" in f && f.size !== void 0) {
|
|
2283
|
+
sets.push(applySize(f.size, context.allDocIds, getValue));
|
|
2284
|
+
}
|
|
2285
|
+
if ("exists" in f && f.exists !== void 0) {
|
|
2286
|
+
sets.push(f.exists ? applyExists(context.allDocIds, getValue) : applyNotExists(context.allDocIds, getValue));
|
|
2287
|
+
}
|
|
2288
|
+
if ("notExists" in f && f.notExists !== void 0) {
|
|
2289
|
+
sets.push(f.notExists ? applyNotExists(context.allDocIds, getValue) : applyExists(context.allDocIds, getValue));
|
|
2290
|
+
}
|
|
2291
|
+
if ("isEmpty" in f && f.isEmpty !== void 0) {
|
|
2292
|
+
sets.push(f.isEmpty ? applyIsEmpty(context.allDocIds, getValue) : applyIsNotEmpty(context.allDocIds, getValue));
|
|
2293
|
+
}
|
|
2294
|
+
if ("isNotEmpty" in f && f.isNotEmpty !== void 0) {
|
|
2295
|
+
sets.push(f.isNotEmpty ? applyIsNotEmpty(context.allDocIds, getValue) : applyIsEmpty(context.allDocIds, getValue));
|
|
2296
|
+
}
|
|
2297
|
+
if (sets.length === 0) return new Set(context.allDocIds);
|
|
2298
|
+
if (sets.length === 1) return sets[0];
|
|
2299
|
+
return applyAnd(sets);
|
|
2300
|
+
}
|
|
2301
|
+
function resolveGeoIndex(fieldPath, fieldIndex) {
|
|
2302
|
+
if (fieldIndex?.type !== "geopoint") {
|
|
2303
|
+
throw new NarsilError(
|
|
2304
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
2305
|
+
`Field "${fieldPath}" requires a geopoint index for geo filters`,
|
|
2306
|
+
{ fieldPath }
|
|
2307
|
+
);
|
|
2308
|
+
}
|
|
2309
|
+
return fieldIndex.index;
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
// src/core/scorer.ts
|
|
2313
|
+
var DEFAULT_K1 = 1.2;
|
|
2314
|
+
var DEFAULT_B = 0.75;
|
|
2315
|
+
function computeIDF(docFrequency, totalDocs) {
|
|
2316
|
+
if (totalDocs === 0) return 0;
|
|
2317
|
+
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
2318
|
+
}
|
|
2319
|
+
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
2320
|
+
if (totalDocs === 0) return 0;
|
|
2321
|
+
if (avgFieldLength === 0) return 0;
|
|
2322
|
+
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
2323
|
+
const b = params?.b ?? DEFAULT_B;
|
|
2324
|
+
const idf = computeIDF(docFrequency, totalDocs);
|
|
2325
|
+
const numerator = termFrequency * (k1 + 1);
|
|
2326
|
+
const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
|
|
2327
|
+
if (denominator === 0) return 0;
|
|
2328
|
+
return idf * (numerator / denominator);
|
|
2329
|
+
}
|
|
2330
|
+
function computeBM25WithGlobalStats(termFrequency, globalDocFrequency, globalTotalDocs, fieldLength, globalAvgFieldLength, params) {
|
|
2331
|
+
return computeBM25(termFrequency, globalDocFrequency, globalTotalDocs, fieldLength, globalAvgFieldLength, params);
|
|
2332
|
+
}
|
|
2333
|
+
|
|
2334
|
+
// src/core/partition/search.ts
|
|
2335
|
+
function searchFulltext(state, params) {
|
|
2336
|
+
const { queryTokens, fields, boost, tolerance = 0, prefixLength = 2, exact = false, bm25Params, globalStats } = params;
|
|
2337
|
+
if (queryTokens.length === 0) {
|
|
2338
|
+
return { scored: [], totalMatched: 0 };
|
|
2339
|
+
}
|
|
2340
|
+
const docScores = /* @__PURE__ */ new Map();
|
|
2341
|
+
const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
|
|
2342
|
+
const avgFieldLengths = globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths;
|
|
2343
|
+
const globalDocFreqs = globalStats?.docFrequencies ?? state.stats.docFrequencies;
|
|
2344
|
+
for (const qt of queryTokens) {
|
|
2345
|
+
const matchingPostings = exact ? (() => {
|
|
2346
|
+
const postingList = state.invertedIdx.lookup(qt.token);
|
|
2347
|
+
return postingList ? [{ token: qt.token, postingList }] : [];
|
|
2348
|
+
})() : state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength);
|
|
2349
|
+
for (const match of matchingPostings) {
|
|
2350
|
+
const docFreq = globalStats ? globalDocFreqs[match.token] ?? match.postingList.docFrequency : match.postingList.docFrequency;
|
|
2351
|
+
const idf = computeIDF(docFreq, totalDocs);
|
|
2352
|
+
for (const posting of match.postingList.postings) {
|
|
2353
|
+
if (fields && !fields.includes(posting.fieldName)) continue;
|
|
2354
|
+
const fieldBoost = boost?.[posting.fieldName] ?? 1;
|
|
2355
|
+
const avgLen = avgFieldLengths[posting.fieldName] ?? 1;
|
|
2356
|
+
const storedDoc = state.docStore.get(posting.docId);
|
|
2357
|
+
const actualFieldLength = storedDoc?.fieldLengths[posting.fieldName] ?? avgLen;
|
|
2358
|
+
const scoreFn = globalStats ? computeBM25WithGlobalStats : computeBM25;
|
|
2359
|
+
let termScore = scoreFn(posting.termFrequency, docFreq, totalDocs, actualFieldLength, avgLen, bm25Params);
|
|
2360
|
+
termScore *= fieldBoost;
|
|
2361
|
+
const existing = docScores.get(posting.docId);
|
|
2362
|
+
if (existing) {
|
|
2363
|
+
existing.score += termScore;
|
|
2364
|
+
existing.termFrequencies[`${posting.fieldName}:${match.token}`] = posting.termFrequency;
|
|
2365
|
+
existing.fieldLengths[posting.fieldName] = actualFieldLength;
|
|
2366
|
+
existing.idf[match.token] = idf;
|
|
2367
|
+
existing.matchedTermCount++;
|
|
2368
|
+
} else {
|
|
2369
|
+
docScores.set(posting.docId, {
|
|
2370
|
+
score: termScore,
|
|
2371
|
+
termFrequencies: { [`${posting.fieldName}:${match.token}`]: posting.termFrequency },
|
|
2372
|
+
fieldLengths: { [posting.fieldName]: actualFieldLength },
|
|
2373
|
+
idf: { [match.token]: idf },
|
|
2374
|
+
matchedTermCount: 1
|
|
2375
|
+
});
|
|
2376
|
+
}
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
const scored = [];
|
|
2381
|
+
for (const [docId, data] of docScores) {
|
|
2382
|
+
scored.push({
|
|
2383
|
+
docId,
|
|
2384
|
+
score: data.score,
|
|
2385
|
+
termFrequencies: data.termFrequencies,
|
|
2386
|
+
fieldLengths: data.fieldLengths,
|
|
2387
|
+
idf: data.idf
|
|
2388
|
+
});
|
|
2389
|
+
}
|
|
2390
|
+
scored.sort((a, b) => b.score - a.score);
|
|
2391
|
+
return { scored, totalMatched: scored.length };
|
|
2392
|
+
}
|
|
2393
|
+
function searchVector(state, params) {
|
|
2394
|
+
const { field, value, k, similarity = 0, metric = "cosine", filterDocIds } = params;
|
|
2395
|
+
const vecStore = state.vectorStores.get(field);
|
|
2396
|
+
if (!vecStore) {
|
|
2397
|
+
return { scored: [], totalMatched: 0 };
|
|
2398
|
+
}
|
|
2399
|
+
if (value.length !== vecStore.dimension) {
|
|
2400
|
+
throw new NarsilError(
|
|
2401
|
+
ErrorCodes.SEARCH_INVALID_VECTOR_SIZE,
|
|
2402
|
+
`Query vector dimension ${value.length} does not match field "${field}" dimension ${vecStore.dimension}`,
|
|
2403
|
+
{ field, expected: vecStore.dimension, received: value.length }
|
|
2404
|
+
);
|
|
2405
|
+
}
|
|
2406
|
+
const queryVec = new Float32Array(value);
|
|
2407
|
+
const results = vecStore.search(queryVec, k, metric, similarity, filterDocIds);
|
|
2408
|
+
return { scored: results, totalMatched: results.length };
|
|
2409
|
+
}
|
|
2410
|
+
function buildFilterContext(state, schema) {
|
|
2411
|
+
const flat = getFlatSchema(state, schema);
|
|
2412
|
+
const fieldIndexes = {};
|
|
2413
|
+
for (const [fieldPath, fieldType] of Object.entries(flat)) {
|
|
2414
|
+
if (fieldType === "number" || fieldType === "number[]") {
|
|
2415
|
+
const numIdx = state.numericIndexes.get(fieldPath);
|
|
2416
|
+
if (numIdx) {
|
|
2417
|
+
fieldIndexes[fieldPath] = {
|
|
2418
|
+
type: "numeric",
|
|
2419
|
+
index: {
|
|
2420
|
+
eq: (v) => numIdx.queryEq(v),
|
|
2421
|
+
gt: (v) => numIdx.queryGt(v),
|
|
2422
|
+
gte: (v) => numIdx.queryGte(v),
|
|
2423
|
+
lt: (v) => numIdx.queryLt(v),
|
|
2424
|
+
lte: (v) => numIdx.queryLte(v),
|
|
2425
|
+
between: (min, max) => numIdx.queryBetween(min, max),
|
|
2426
|
+
allDocIds: () => numIdx.getAllDocIds()
|
|
2427
|
+
}
|
|
2428
|
+
};
|
|
2429
|
+
}
|
|
2430
|
+
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
2431
|
+
const boolIdx = state.booleanIndexes.get(fieldPath);
|
|
2432
|
+
if (boolIdx) {
|
|
2433
|
+
fieldIndexes[fieldPath] = {
|
|
2434
|
+
type: "boolean",
|
|
2435
|
+
index: {
|
|
2436
|
+
getTrue: () => boolIdx.queryEq(true),
|
|
2437
|
+
getFalse: () => boolIdx.queryEq(false),
|
|
2438
|
+
allDocIds: () => boolIdx.getAllDocIds()
|
|
2439
|
+
}
|
|
2440
|
+
};
|
|
2441
|
+
}
|
|
2442
|
+
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
2443
|
+
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
2444
|
+
if (enumIdx) {
|
|
2445
|
+
fieldIndexes[fieldPath] = {
|
|
2446
|
+
type: "enum",
|
|
2447
|
+
index: {
|
|
2448
|
+
getDocIds: (v) => enumIdx.queryEq(v),
|
|
2449
|
+
allDocIds: () => enumIdx.getAllDocIds()
|
|
2450
|
+
}
|
|
2451
|
+
};
|
|
2452
|
+
}
|
|
2453
|
+
} else if (fieldType === "geopoint") {
|
|
2454
|
+
const geoIdx = state.geoIndexes.get(fieldPath);
|
|
2455
|
+
if (geoIdx) {
|
|
2456
|
+
fieldIndexes[fieldPath] = {
|
|
2457
|
+
type: "geopoint",
|
|
2458
|
+
index: geoIdx
|
|
2459
|
+
};
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
let cachedAllDocIds = null;
|
|
2464
|
+
return {
|
|
2465
|
+
fieldIndexes,
|
|
2466
|
+
getFieldValue: (docId, fieldPath) => getFieldValueForDoc(state.docStore, docId, fieldPath),
|
|
2467
|
+
get allDocIds() {
|
|
2468
|
+
if (!cachedAllDocIds) {
|
|
2469
|
+
cachedAllDocIds = getAllDocIds(state.docStore);
|
|
2470
|
+
}
|
|
2471
|
+
return cachedAllDocIds;
|
|
2472
|
+
}
|
|
2473
|
+
};
|
|
2474
|
+
}
|
|
2475
|
+
function applyPartitionFilters(state, filters, schema) {
|
|
2476
|
+
const context = buildFilterContext(state, schema);
|
|
2477
|
+
return evaluateFilters(filters, context);
|
|
2478
|
+
}
|
|
2479
|
+
function computeFacets(state, docIds, config, schema) {
|
|
2480
|
+
const result = {};
|
|
2481
|
+
const flatSchema = getFlatSchema(state, schema);
|
|
2482
|
+
for (const [fieldPath, facetOpts] of Object.entries(config)) {
|
|
2483
|
+
const fieldType = flatSchema[fieldPath];
|
|
2484
|
+
if (!fieldType) continue;
|
|
2485
|
+
const valueCounts = /* @__PURE__ */ new Map();
|
|
2486
|
+
if (facetOpts.ranges && (fieldType === "number" || fieldType === "number[]")) {
|
|
2487
|
+
for (const range of facetOpts.ranges) {
|
|
2488
|
+
const label = `${range.from}-${range.to}`;
|
|
2489
|
+
let count = 0;
|
|
2490
|
+
for (const docId of docIds) {
|
|
2491
|
+
const val = getFieldValueForDoc(state.docStore, docId, fieldPath);
|
|
2492
|
+
if (fieldType === "number[]" && Array.isArray(val)) {
|
|
2493
|
+
for (const v of val) {
|
|
2494
|
+
if (v >= range.from && v < range.to) {
|
|
2495
|
+
count++;
|
|
2496
|
+
break;
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
} else if (typeof val === "number" && val >= range.from && val < range.to) {
|
|
2500
|
+
count++;
|
|
2501
|
+
}
|
|
2502
|
+
}
|
|
2503
|
+
valueCounts.set(label, count);
|
|
2504
|
+
}
|
|
2505
|
+
} else {
|
|
2506
|
+
for (const docId of docIds) {
|
|
2507
|
+
const val = getFieldValueForDoc(state.docStore, docId, fieldPath);
|
|
2508
|
+
if (val === void 0 || val === null) continue;
|
|
2509
|
+
if (Array.isArray(val)) {
|
|
2510
|
+
for (const item of val) {
|
|
2511
|
+
const key = String(item);
|
|
2512
|
+
valueCounts.set(key, (valueCounts.get(key) ?? 0) + 1);
|
|
2513
|
+
}
|
|
2514
|
+
} else {
|
|
2515
|
+
const key = String(val);
|
|
2516
|
+
valueCounts.set(key, (valueCounts.get(key) ?? 0) + 1);
|
|
2517
|
+
}
|
|
2518
|
+
}
|
|
2519
|
+
}
|
|
2520
|
+
let entries = Array.from(valueCounts.entries());
|
|
2521
|
+
const sortDir = facetOpts.sort ?? "desc";
|
|
2522
|
+
entries.sort((a, b) => sortDir === "asc" ? a[1] - b[1] : b[1] - a[1]);
|
|
2523
|
+
if (facetOpts.limit && facetOpts.limit > 0) {
|
|
2524
|
+
entries = entries.slice(0, facetOpts.limit);
|
|
2525
|
+
}
|
|
2526
|
+
const values = {};
|
|
2527
|
+
for (const [key, count] of entries) {
|
|
2528
|
+
values[key] = count;
|
|
2529
|
+
}
|
|
2530
|
+
result[fieldPath] = { values, count: entries.length };
|
|
2531
|
+
}
|
|
2532
|
+
return result;
|
|
2533
|
+
}
|
|
2534
|
+
|
|
2535
|
+
// src/core/partition/serialization.ts
|
|
2536
|
+
function serializePartition(state, partitionId, indexName, totalPartitions, language, schema) {
|
|
2537
|
+
const flatSchema = getFlatSchema(state, schema);
|
|
2538
|
+
const flatSchemaStrings = {};
|
|
2539
|
+
for (const [k, v] of Object.entries(flatSchema)) {
|
|
2540
|
+
flatSchemaStrings[k] = v;
|
|
2541
|
+
}
|
|
2542
|
+
const serializedDocs = /* @__PURE__ */ Object.create(null);
|
|
2543
|
+
for (const [docId, stored] of state.docStore.all()) {
|
|
2544
|
+
serializedDocs[docId] = {
|
|
2545
|
+
fields: structuredClone(stored.fields),
|
|
2546
|
+
fieldLengths: { ...stored.fieldLengths }
|
|
2547
|
+
};
|
|
2548
|
+
}
|
|
2549
|
+
const serializedInverted = state.invertedIdx.serialize();
|
|
2550
|
+
const serializedInvertedIndex = /* @__PURE__ */ Object.create(null);
|
|
2551
|
+
for (const [token, list] of Object.entries(serializedInverted)) {
|
|
2552
|
+
serializedInvertedIndex[token] = {
|
|
2553
|
+
docFrequency: list.docFrequency,
|
|
2554
|
+
postings: list.postings.map((p) => ({
|
|
2555
|
+
docId: p.docId,
|
|
2556
|
+
termFrequency: p.termFrequency,
|
|
2557
|
+
field: p.fieldName,
|
|
2558
|
+
positions: [...p.positions]
|
|
2559
|
+
}))
|
|
2560
|
+
};
|
|
2561
|
+
}
|
|
2562
|
+
const serializedNumeric = {};
|
|
2563
|
+
for (const [path, idx] of state.numericIndexes) {
|
|
2564
|
+
serializedNumeric[path] = idx.serialize();
|
|
2565
|
+
}
|
|
2566
|
+
const serializedBoolean = {};
|
|
2567
|
+
for (const [path, idx] of state.booleanIndexes) {
|
|
2568
|
+
serializedBoolean[path] = idx.serialize();
|
|
2569
|
+
}
|
|
2570
|
+
const serializedEnum = {};
|
|
2571
|
+
for (const [path, idx] of state.enumIndexes) {
|
|
2572
|
+
serializedEnum[path] = idx.serialize();
|
|
2573
|
+
}
|
|
2574
|
+
const serializedGeo = {};
|
|
2575
|
+
for (const [path, idx] of state.geoIndexes) {
|
|
2576
|
+
serializedGeo[path] = idx.serialize();
|
|
2577
|
+
}
|
|
2578
|
+
const serializedVectors = {};
|
|
2579
|
+
for (const [path, store] of state.vectorStores) {
|
|
2580
|
+
const vectors = [];
|
|
2581
|
+
for (const [, entry] of store.entries()) {
|
|
2582
|
+
vectors.push({
|
|
2583
|
+
docId: entry.docId,
|
|
2584
|
+
vector: Array.from(entry.vector)
|
|
2585
|
+
});
|
|
2586
|
+
}
|
|
2587
|
+
serializedVectors[path] = {
|
|
2588
|
+
dimension: store.dimension,
|
|
2589
|
+
vectors,
|
|
2590
|
+
hnswGraph: null
|
|
2591
|
+
};
|
|
2592
|
+
}
|
|
2593
|
+
const serializedStats = state.stats.serialize();
|
|
2594
|
+
return {
|
|
2595
|
+
indexName,
|
|
2596
|
+
partitionId,
|
|
2597
|
+
totalPartitions,
|
|
2598
|
+
language,
|
|
2599
|
+
schema: flatSchemaStrings,
|
|
2600
|
+
docCount: state.docStore.count(),
|
|
2601
|
+
avgDocLength: Object.values(serializedStats.averageFieldLengths).reduce((sum, v) => sum + v, 0),
|
|
2602
|
+
documents: serializedDocs,
|
|
2603
|
+
invertedIndex: serializedInvertedIndex,
|
|
2604
|
+
fieldIndexes: {
|
|
2605
|
+
numeric: serializedNumeric,
|
|
2606
|
+
boolean: serializedBoolean,
|
|
2607
|
+
enum: serializedEnum,
|
|
2608
|
+
geopoint: serializedGeo
|
|
2609
|
+
},
|
|
2610
|
+
vectorData: serializedVectors,
|
|
2611
|
+
statistics: serializedStats
|
|
2612
|
+
};
|
|
2613
|
+
}
|
|
2614
|
+
function deserializePartition(state, data, clearFn, schema) {
|
|
2615
|
+
clearFn();
|
|
2616
|
+
const invertedData = /* @__PURE__ */ Object.create(null);
|
|
2617
|
+
for (const [token, list] of Object.entries(data.invertedIndex)) {
|
|
2618
|
+
invertedData[token] = {
|
|
2619
|
+
docFrequency: list.docFrequency,
|
|
2620
|
+
postings: list.postings.map((p) => ({
|
|
2621
|
+
docId: p.docId,
|
|
2622
|
+
termFrequency: p.termFrequency,
|
|
2623
|
+
fieldName: p.field,
|
|
2624
|
+
positions: [...p.positions]
|
|
2625
|
+
}))
|
|
2626
|
+
};
|
|
2627
|
+
}
|
|
2628
|
+
state.invertedIdx.deserialize(invertedData);
|
|
2629
|
+
const docsData = /* @__PURE__ */ Object.create(null);
|
|
2630
|
+
for (const [docId, doc] of Object.entries(data.documents)) {
|
|
2631
|
+
docsData[docId] = { fields: doc.fields, fieldLengths: doc.fieldLengths };
|
|
2632
|
+
}
|
|
2633
|
+
state.docStore.deserialize(docsData);
|
|
2634
|
+
for (const [path, entries] of Object.entries(data.fieldIndexes.numeric)) {
|
|
2635
|
+
const idx = createNumericIndex();
|
|
2636
|
+
idx.deserialize(entries);
|
|
2637
|
+
state.numericIndexes.set(path, idx);
|
|
2638
|
+
}
|
|
2639
|
+
for (const [path, serialized] of Object.entries(data.fieldIndexes.boolean)) {
|
|
2640
|
+
const idx = createBooleanIndex();
|
|
2641
|
+
idx.deserialize(serialized);
|
|
2642
|
+
state.booleanIndexes.set(path, idx);
|
|
2643
|
+
}
|
|
2644
|
+
for (const [path, serialized] of Object.entries(data.fieldIndexes.enum)) {
|
|
2645
|
+
const idx = createEnumIndex();
|
|
2646
|
+
idx.deserialize(serialized);
|
|
2647
|
+
state.enumIndexes.set(path, idx);
|
|
2648
|
+
}
|
|
2649
|
+
for (const [path, entries] of Object.entries(data.fieldIndexes.geopoint)) {
|
|
2650
|
+
const geoIdx = createGeoIndex();
|
|
2651
|
+
geoIdx.deserialize(entries);
|
|
2652
|
+
state.geoIndexes.set(path, geoIdx);
|
|
2653
|
+
}
|
|
2654
|
+
for (const [path, vecData] of Object.entries(data.vectorData)) {
|
|
2655
|
+
const store = createBruteForceVectorStore(vecData.dimension);
|
|
2656
|
+
for (const entry of vecData.vectors) {
|
|
2657
|
+
store.insert(entry.docId, new Float32Array(entry.vector));
|
|
2658
|
+
}
|
|
2659
|
+
state.vectorStores.set(path, store);
|
|
2660
|
+
}
|
|
2661
|
+
state.stats.deserialize(data.statistics);
|
|
2662
|
+
const flatSchema = getFlatSchema(state, schema);
|
|
2663
|
+
const serializedFields = data.schema;
|
|
2664
|
+
for (const [field, expectedType] of Object.entries(flatSchema)) {
|
|
2665
|
+
if (field in serializedFields && serializedFields[field] !== expectedType) {
|
|
2666
|
+
throw new Error(
|
|
2667
|
+
`Schema mismatch on field "${field}": expected "${expectedType}", found "${serializedFields[field]}"`
|
|
2668
|
+
);
|
|
2669
|
+
}
|
|
2670
|
+
}
|
|
2671
|
+
for (const field of Object.keys(serializedFields)) {
|
|
2672
|
+
if (!(field in flatSchema)) {
|
|
2673
|
+
throw new Error(`Schema mismatch: serialized data contains unknown field "${field}" not present in schema`);
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
}
|
|
2677
|
+
|
|
2678
|
+
// src/core/partition/index.ts
|
|
2679
|
+
function createPartitionIndex(partitionId) {
|
|
2680
|
+
const state = {
|
|
2681
|
+
invertedIdx: createInvertedIndex(),
|
|
2682
|
+
docStore: createDocumentStore(),
|
|
2683
|
+
stats: createPartitionStats(),
|
|
2684
|
+
numericIndexes: /* @__PURE__ */ new Map(),
|
|
2685
|
+
booleanIndexes: /* @__PURE__ */ new Map(),
|
|
2686
|
+
enumIndexes: /* @__PURE__ */ new Map(),
|
|
2687
|
+
geoIndexes: /* @__PURE__ */ new Map(),
|
|
2688
|
+
vectorStores: /* @__PURE__ */ new Map(),
|
|
2689
|
+
flatSchemaCache: null,
|
|
2690
|
+
lastSchemaRef: null
|
|
2691
|
+
};
|
|
2692
|
+
function clearAll() {
|
|
2693
|
+
state.invertedIdx.clear();
|
|
2694
|
+
state.docStore.clear();
|
|
2695
|
+
for (const idx of state.numericIndexes.values()) idx.clear();
|
|
2696
|
+
for (const idx of state.booleanIndexes.values()) idx.clear();
|
|
2697
|
+
for (const idx of state.enumIndexes.values()) idx.clear();
|
|
2698
|
+
for (const idx of state.geoIndexes.values()) idx.clear();
|
|
2699
|
+
for (const store of state.vectorStores.values()) store.clear();
|
|
2700
|
+
state.numericIndexes.clear();
|
|
2701
|
+
state.booleanIndexes.clear();
|
|
2702
|
+
state.enumIndexes.clear();
|
|
2703
|
+
state.geoIndexes.clear();
|
|
2704
|
+
state.vectorStores.clear();
|
|
2705
|
+
state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
|
|
2706
|
+
state.flatSchemaCache = null;
|
|
2707
|
+
state.lastSchemaRef = null;
|
|
2708
|
+
}
|
|
2709
|
+
const partition = {
|
|
2710
|
+
get partitionId() {
|
|
2711
|
+
return partitionId;
|
|
2712
|
+
},
|
|
2713
|
+
get stats() {
|
|
2714
|
+
return state.stats;
|
|
2715
|
+
},
|
|
2716
|
+
insert(docId, document, schema, language, options) {
|
|
2717
|
+
if (state.docStore.has(docId)) {
|
|
2718
|
+
throw new NarsilError(
|
|
2719
|
+
ErrorCodes.DOC_ALREADY_EXISTS,
|
|
2720
|
+
`Document "${docId}" already exists in partition ${partitionId}`,
|
|
2721
|
+
{ docId, partitionId }
|
|
2722
|
+
);
|
|
2723
|
+
}
|
|
2724
|
+
if (options?.validate !== false) {
|
|
2725
|
+
validateDocument(document, schema);
|
|
2726
|
+
}
|
|
2727
|
+
const flatSchema = getFlatSchema(state, schema);
|
|
2728
|
+
const { fieldLengths, tokensByField } = indexDocument(
|
|
2729
|
+
state,
|
|
2730
|
+
docId,
|
|
2731
|
+
document,
|
|
2732
|
+
flatSchema,
|
|
2733
|
+
language,
|
|
2734
|
+
options
|
|
2735
|
+
);
|
|
2736
|
+
state.docStore.store(docId, document, fieldLengths);
|
|
2737
|
+
state.stats.addDocument(fieldLengths, tokensByField);
|
|
2738
|
+
},
|
|
2739
|
+
remove(docId, schema, language, options) {
|
|
2740
|
+
const stored = state.docStore.get(docId);
|
|
2741
|
+
if (!stored) {
|
|
2742
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in partition ${partitionId}`, {
|
|
2743
|
+
docId,
|
|
2744
|
+
partitionId
|
|
2745
|
+
});
|
|
2746
|
+
}
|
|
2747
|
+
const flatSchema = getFlatSchema(state, schema);
|
|
2748
|
+
const { fieldLengths, tokensByField } = removeFromIndexes(state, docId, stored, flatSchema, language, options);
|
|
2749
|
+
state.docStore.remove(docId);
|
|
2750
|
+
state.stats.removeDocument(fieldLengths, tokensByField);
|
|
2751
|
+
},
|
|
2752
|
+
update(docId, document, schema, language, options) {
|
|
2753
|
+
const stored = state.docStore.get(docId);
|
|
2754
|
+
if (!stored) {
|
|
2755
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in partition ${partitionId}`, {
|
|
2756
|
+
docId,
|
|
2757
|
+
partitionId
|
|
2758
|
+
});
|
|
2759
|
+
}
|
|
2760
|
+
if (options?.validate !== false) {
|
|
2761
|
+
validateDocument(document, schema);
|
|
2762
|
+
}
|
|
2763
|
+
const flatSchema = getFlatSchema(state, schema);
|
|
2764
|
+
const needsTextReindex = textFieldsChanged(stored.fields, document, flatSchema);
|
|
2765
|
+
if (needsTextReindex) {
|
|
2766
|
+
const { fieldLengths: oldFieldLengths, tokensByField: oldTokens } = removeFromIndexes(
|
|
2767
|
+
state,
|
|
2768
|
+
docId,
|
|
2769
|
+
stored,
|
|
2770
|
+
flatSchema,
|
|
2771
|
+
language,
|
|
2772
|
+
options
|
|
2773
|
+
);
|
|
2774
|
+
state.stats.removeDocument(oldFieldLengths, oldTokens);
|
|
2775
|
+
state.docStore.remove(docId);
|
|
2776
|
+
const { fieldLengths: newFieldLengths, tokensByField: newTokens } = indexDocument(
|
|
2777
|
+
state,
|
|
2778
|
+
docId,
|
|
2779
|
+
document,
|
|
2780
|
+
flatSchema,
|
|
2781
|
+
language,
|
|
2782
|
+
options
|
|
2783
|
+
);
|
|
2784
|
+
state.docStore.store(docId, document, newFieldLengths);
|
|
2785
|
+
state.stats.addDocument(newFieldLengths, newTokens);
|
|
2786
|
+
} else {
|
|
2787
|
+
updateFieldIndexOnly(state, docId, stored.fields, document, flatSchema);
|
|
2788
|
+
state.docStore.store(docId, document, stored.fieldLengths);
|
|
2789
|
+
}
|
|
2790
|
+
},
|
|
2791
|
+
get(docId) {
|
|
2792
|
+
const stored = state.docStore.get(docId);
|
|
2793
|
+
if (!stored) return void 0;
|
|
2794
|
+
return structuredClone(stored.fields);
|
|
2795
|
+
},
|
|
2796
|
+
has(docId) {
|
|
2797
|
+
return state.docStore.has(docId);
|
|
2798
|
+
},
|
|
2799
|
+
count() {
|
|
2800
|
+
return state.docStore.count();
|
|
2801
|
+
},
|
|
2802
|
+
*docIds() {
|
|
2803
|
+
for (const [id] of state.docStore.all()) {
|
|
2804
|
+
yield id;
|
|
2805
|
+
}
|
|
2806
|
+
},
|
|
2807
|
+
clear: clearAll,
|
|
2808
|
+
searchFulltext(params) {
|
|
2809
|
+
return searchFulltext(state, params);
|
|
2810
|
+
},
|
|
2811
|
+
searchVector(params) {
|
|
2812
|
+
return searchVector(state, params);
|
|
2813
|
+
},
|
|
2814
|
+
applyFilters(filters, schema) {
|
|
2815
|
+
return applyPartitionFilters(state, filters, schema);
|
|
2816
|
+
},
|
|
2817
|
+
computeFacets(docIds, config, schema) {
|
|
2818
|
+
return computeFacets(state, docIds, config, schema);
|
|
2819
|
+
},
|
|
2820
|
+
serialize(indexName, totalPartitions, language, schema) {
|
|
2821
|
+
return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
|
|
2822
|
+
},
|
|
2823
|
+
deserialize(data, schema) {
|
|
2824
|
+
deserializePartition(state, data, clearAll, schema);
|
|
2825
|
+
}
|
|
2826
|
+
};
|
|
2827
|
+
return partition;
|
|
2828
|
+
}
|
|
2829
|
+
|
|
2830
|
+
// src/partitioning/manager.ts
|
|
2831
|
+
function createPartitionManager(indexName, config, language, router, initialPartitionCount) {
|
|
2832
|
+
const count = initialPartitionCount ?? 1;
|
|
2833
|
+
let partitions = [];
|
|
2834
|
+
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
2835
|
+
for (let i = 0; i < count; i++) {
|
|
2836
|
+
partitions.push(createPartitionIndex(i));
|
|
2837
|
+
}
|
|
2838
|
+
function validatePartitionId(partitionId) {
|
|
2839
|
+
if (partitionId < 0 || partitionId >= partitions.length) {
|
|
2840
|
+
throw new NarsilError(
|
|
2841
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
2842
|
+
`Partition ${partitionId} is out of bounds (0..${partitions.length - 1})`,
|
|
2843
|
+
{ partitionId, partitionCount: partitions.length }
|
|
2844
|
+
);
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
function rebuildDocPartitionMap() {
|
|
2848
|
+
docPartitionMap.clear();
|
|
2849
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
2850
|
+
for (const docId of partitions[i].docIds()) {
|
|
2851
|
+
docPartitionMap.set(docId, i);
|
|
2852
|
+
}
|
|
2853
|
+
}
|
|
2854
|
+
}
|
|
2855
|
+
function rebuildDocPartitionMapForPartition(partitionId) {
|
|
2856
|
+
for (const [docId, pid] of docPartitionMap) {
|
|
2857
|
+
if (pid === partitionId) {
|
|
2858
|
+
docPartitionMap.delete(docId);
|
|
2859
|
+
}
|
|
2860
|
+
}
|
|
2861
|
+
for (const docId of partitions[partitionId].docIds()) {
|
|
2862
|
+
docPartitionMap.set(docId, partitionId);
|
|
2863
|
+
}
|
|
2864
|
+
}
|
|
2865
|
+
const manager = {
|
|
2866
|
+
get partitionCount() {
|
|
2867
|
+
return partitions.length;
|
|
2868
|
+
},
|
|
2869
|
+
get indexName() {
|
|
2870
|
+
return indexName;
|
|
2871
|
+
},
|
|
2872
|
+
get schema() {
|
|
2873
|
+
return config.schema;
|
|
2874
|
+
},
|
|
2875
|
+
get language() {
|
|
2876
|
+
return language;
|
|
2877
|
+
},
|
|
2878
|
+
get config() {
|
|
2879
|
+
return config;
|
|
2880
|
+
},
|
|
2881
|
+
getPartition(partitionId) {
|
|
2882
|
+
validatePartitionId(partitionId);
|
|
2883
|
+
return partitions[partitionId];
|
|
2884
|
+
},
|
|
2885
|
+
getAllPartitions() {
|
|
2886
|
+
return [...partitions];
|
|
2887
|
+
},
|
|
2888
|
+
setPartitions(newPartitions) {
|
|
2889
|
+
partitions = [...newPartitions];
|
|
2890
|
+
rebuildDocPartitionMap();
|
|
2891
|
+
},
|
|
2892
|
+
addPartition() {
|
|
2893
|
+
const partition = createPartitionIndex(partitions.length);
|
|
2894
|
+
partitions.push(partition);
|
|
2895
|
+
return partition;
|
|
2896
|
+
},
|
|
2897
|
+
removePartition(partitionId) {
|
|
2898
|
+
validatePartitionId(partitionId);
|
|
2899
|
+
partitions.splice(partitionId, 1);
|
|
2900
|
+
rebuildDocPartitionMap();
|
|
2901
|
+
},
|
|
2902
|
+
insert(docId, document, options) {
|
|
2903
|
+
const pid = router.route(docId, partitions.length);
|
|
2904
|
+
partitions[pid].insert(docId, document, config.schema, language, options);
|
|
2905
|
+
docPartitionMap.set(docId, pid);
|
|
2906
|
+
},
|
|
2907
|
+
remove(docId) {
|
|
2908
|
+
const pid = docPartitionMap.get(docId);
|
|
2909
|
+
if (pid === void 0) {
|
|
2910
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
2911
|
+
}
|
|
2912
|
+
partitions[pid].remove(docId, config.schema, language);
|
|
2913
|
+
docPartitionMap.delete(docId);
|
|
2914
|
+
},
|
|
2915
|
+
update(docId, document, options) {
|
|
2916
|
+
const pid = docPartitionMap.get(docId);
|
|
2917
|
+
if (pid === void 0) {
|
|
2918
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
2919
|
+
}
|
|
2920
|
+
partitions[pid].update(docId, document, config.schema, language, options);
|
|
2921
|
+
},
|
|
2922
|
+
get(docId) {
|
|
2923
|
+
const pid = docPartitionMap.get(docId);
|
|
2924
|
+
if (pid === void 0) return void 0;
|
|
2925
|
+
return partitions[pid].get(docId);
|
|
2926
|
+
},
|
|
2927
|
+
has(docId) {
|
|
2928
|
+
return docPartitionMap.has(docId);
|
|
2929
|
+
},
|
|
2930
|
+
countDocuments() {
|
|
2931
|
+
return docPartitionMap.size;
|
|
2932
|
+
},
|
|
2933
|
+
serializePartition(partitionId) {
|
|
2934
|
+
validatePartitionId(partitionId);
|
|
2935
|
+
return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
|
|
2936
|
+
},
|
|
2937
|
+
deserializePartition(partitionId, data) {
|
|
2938
|
+
validatePartitionId(partitionId);
|
|
2939
|
+
partitions[partitionId].deserialize(data, config.schema);
|
|
2940
|
+
rebuildDocPartitionMapForPartition(partitionId);
|
|
2941
|
+
},
|
|
2942
|
+
getAggregateStats() {
|
|
2943
|
+
let totalDocuments = 0;
|
|
2944
|
+
const docFrequencies = {};
|
|
2945
|
+
const totalFieldLengths = {};
|
|
2946
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
2947
|
+
const stats = partitions[i].stats;
|
|
2948
|
+
totalDocuments += stats.totalDocuments;
|
|
2949
|
+
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
2950
|
+
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
2951
|
+
}
|
|
2952
|
+
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
2953
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
2954
|
+
}
|
|
2955
|
+
}
|
|
2956
|
+
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
2957
|
+
}
|
|
2958
|
+
};
|
|
2959
|
+
return manager;
|
|
2960
|
+
}
|
|
2961
|
+
|
|
2962
|
+
// src/core/hash.ts
|
|
2963
|
+
var FNV_OFFSET_BASIS = 2166136261;
|
|
2964
|
+
var FNV_PRIME = 16777619;
|
|
2965
|
+
var encoder = new TextEncoder();
|
|
2966
|
+
function fnv1a(input) {
|
|
2967
|
+
const bytes = encoder.encode(input);
|
|
2968
|
+
let hash = FNV_OFFSET_BASIS;
|
|
2969
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
2970
|
+
hash ^= bytes[i];
|
|
2971
|
+
hash = Math.imul(hash, FNV_PRIME);
|
|
2972
|
+
}
|
|
2973
|
+
return hash >>> 0;
|
|
2974
|
+
}
|
|
2975
|
+
|
|
2976
|
+
// src/partitioning/router.ts
|
|
2977
|
+
function createPartitionRouter() {
|
|
2978
|
+
function route(docId, partitionCount) {
|
|
2979
|
+
if (partitionCount <= 0) {
|
|
2980
|
+
throw new NarsilError(
|
|
2981
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
2982
|
+
`Partition count must be greater than 0, received ${partitionCount}`,
|
|
2983
|
+
{ partitionCount }
|
|
2984
|
+
);
|
|
2985
|
+
}
|
|
2986
|
+
return fnv1a(docId) % partitionCount;
|
|
2987
|
+
}
|
|
2988
|
+
function routeBatch(docIds, partitionCount) {
|
|
2989
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2990
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
2991
|
+
const pid = route(docIds[i], partitionCount);
|
|
2992
|
+
let group = groups.get(pid);
|
|
2993
|
+
if (!group) {
|
|
2994
|
+
group = [];
|
|
2995
|
+
groups.set(pid, group);
|
|
2996
|
+
}
|
|
2997
|
+
group.push(docIds[i]);
|
|
2998
|
+
}
|
|
2999
|
+
return groups;
|
|
3000
|
+
}
|
|
3001
|
+
return { route, routeBatch };
|
|
3002
|
+
}
|
|
3003
|
+
|
|
3004
|
+
// src/workers/direct-executor.ts
|
|
3005
|
+
function createDirectExecutor() {
|
|
3006
|
+
const indexes = /* @__PURE__ */ new Map();
|
|
3007
|
+
function requireIndex(indexName) {
|
|
3008
|
+
const entry = indexes.get(indexName);
|
|
3009
|
+
if (!entry) {
|
|
3010
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" does not exist`, {
|
|
3011
|
+
indexName
|
|
3012
|
+
});
|
|
3013
|
+
}
|
|
3014
|
+
return entry;
|
|
3015
|
+
}
|
|
3016
|
+
function createIndex(indexName, config, language) {
|
|
3017
|
+
if (indexes.has(indexName)) {
|
|
3018
|
+
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${indexName}" already exists`, {
|
|
3019
|
+
indexName
|
|
3020
|
+
});
|
|
3021
|
+
}
|
|
3022
|
+
const router = createPartitionRouter();
|
|
3023
|
+
const partitionCount = config.partitions?.maxPartitions ?? 1;
|
|
3024
|
+
const manager = createPartitionManager(indexName, config, language, router, partitionCount);
|
|
3025
|
+
indexes.set(indexName, { manager, config, language });
|
|
3026
|
+
}
|
|
3027
|
+
function dropIndex(indexName) {
|
|
3028
|
+
const entry = requireIndex(indexName);
|
|
3029
|
+
for (const partition of entry.manager.getAllPartitions()) {
|
|
3030
|
+
partition.clear();
|
|
3031
|
+
}
|
|
3032
|
+
indexes.delete(indexName);
|
|
3033
|
+
}
|
|
3034
|
+
function listIndexes() {
|
|
3035
|
+
return Array.from(indexes.keys());
|
|
3036
|
+
}
|
|
3037
|
+
function getManager(indexName) {
|
|
3038
|
+
return indexes.get(indexName)?.manager;
|
|
3039
|
+
}
|
|
3040
|
+
async function execute(action) {
|
|
3041
|
+
switch (action.type) {
|
|
3042
|
+
case "createIndex": {
|
|
3043
|
+
const language = getLanguage(action.config.language ?? "english");
|
|
3044
|
+
createIndex(action.indexName, action.config, language);
|
|
3045
|
+
return void 0;
|
|
3046
|
+
}
|
|
3047
|
+
case "dropIndex": {
|
|
3048
|
+
dropIndex(action.indexName);
|
|
3049
|
+
return void 0;
|
|
3050
|
+
}
|
|
3051
|
+
case "insert": {
|
|
3052
|
+
const entry = requireIndex(action.indexName);
|
|
3053
|
+
entry.manager.insert(action.docId, action.document);
|
|
3054
|
+
return void 0;
|
|
3055
|
+
}
|
|
3056
|
+
case "remove": {
|
|
3057
|
+
const entry = requireIndex(action.indexName);
|
|
3058
|
+
entry.manager.remove(action.docId);
|
|
3059
|
+
return void 0;
|
|
3060
|
+
}
|
|
3061
|
+
case "update": {
|
|
3062
|
+
const entry = requireIndex(action.indexName);
|
|
3063
|
+
entry.manager.update(action.docId, action.document);
|
|
3064
|
+
return void 0;
|
|
3065
|
+
}
|
|
3066
|
+
case "query": {
|
|
3067
|
+
const entry = requireIndex(action.indexName);
|
|
3068
|
+
const result = fanOutQuery(entry.manager, action.params, entry.language, entry.config.schema, {
|
|
3069
|
+
scoringMode: entry.config.defaultScoring ?? "local"
|
|
3070
|
+
});
|
|
3071
|
+
return result;
|
|
3072
|
+
}
|
|
3073
|
+
case "preflight": {
|
|
3074
|
+
const entry = requireIndex(action.indexName);
|
|
3075
|
+
const result = fanOutQuery(entry.manager, action.params, entry.language, entry.config.schema, {
|
|
3076
|
+
scoringMode: entry.config.defaultScoring ?? "local"
|
|
3077
|
+
});
|
|
3078
|
+
return { count: result.totalMatched };
|
|
3079
|
+
}
|
|
3080
|
+
case "get": {
|
|
3081
|
+
const entry = requireIndex(action.indexName);
|
|
3082
|
+
return entry.manager.get(action.docId);
|
|
3083
|
+
}
|
|
3084
|
+
case "has": {
|
|
3085
|
+
const entry = requireIndex(action.indexName);
|
|
3086
|
+
return entry.manager.has(action.docId);
|
|
3087
|
+
}
|
|
3088
|
+
case "count": {
|
|
3089
|
+
const entry = requireIndex(action.indexName);
|
|
3090
|
+
return entry.manager.countDocuments();
|
|
3091
|
+
}
|
|
3092
|
+
case "getStats": {
|
|
3093
|
+
const entry = requireIndex(action.indexName);
|
|
3094
|
+
return {
|
|
3095
|
+
documentCount: entry.manager.countDocuments(),
|
|
3096
|
+
partitionCount: entry.manager.partitionCount,
|
|
3097
|
+
language: entry.language.name,
|
|
3098
|
+
schema: entry.config.schema
|
|
3099
|
+
};
|
|
3100
|
+
}
|
|
3101
|
+
case "clear": {
|
|
3102
|
+
const entry = requireIndex(action.indexName);
|
|
3103
|
+
const partitions = entry.manager.getAllPartitions();
|
|
3104
|
+
for (const partition of partitions) {
|
|
3105
|
+
partition.clear();
|
|
3106
|
+
}
|
|
3107
|
+
entry.manager.setPartitions(partitions);
|
|
3108
|
+
return void 0;
|
|
3109
|
+
}
|
|
3110
|
+
case "serialize": {
|
|
3111
|
+
const entry = requireIndex(action.indexName);
|
|
3112
|
+
return entry.manager.serializePartition(action.partitionId);
|
|
3113
|
+
}
|
|
3114
|
+
case "deserialize": {
|
|
3115
|
+
const entry = requireIndex(action.indexName);
|
|
3116
|
+
entry.manager.deserializePartition(action.partitionId, action.data);
|
|
3117
|
+
return void 0;
|
|
3118
|
+
}
|
|
3119
|
+
case "memoryReport": {
|
|
3120
|
+
const report = typeof process !== "undefined" && process.memoryUsage ? process.memoryUsage() : {};
|
|
3121
|
+
return report;
|
|
3122
|
+
}
|
|
3123
|
+
case "shutdown": {
|
|
3124
|
+
for (const [, entry] of indexes) {
|
|
3125
|
+
for (const partition of entry.manager.getAllPartitions()) {
|
|
3126
|
+
partition.clear();
|
|
3127
|
+
}
|
|
3128
|
+
}
|
|
3129
|
+
indexes.clear();
|
|
3130
|
+
return void 0;
|
|
3131
|
+
}
|
|
3132
|
+
}
|
|
3133
|
+
}
|
|
3134
|
+
async function shutdown() {
|
|
3135
|
+
await execute({ type: "shutdown"});
|
|
3136
|
+
}
|
|
3137
|
+
return {
|
|
3138
|
+
execute,
|
|
3139
|
+
shutdown,
|
|
3140
|
+
getManager,
|
|
3141
|
+
createIndex,
|
|
3142
|
+
dropIndex,
|
|
3143
|
+
listIndexes
|
|
3144
|
+
};
|
|
3145
|
+
}
|
|
3146
|
+
|
|
3147
|
+
export { createDirectExecutor, fanOutQuery, getLanguage, registerLanguage, tokenize, validateSchema };
|