@delali/narsil 0.3.1-next.77 → 0.3.1-next.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-HH6ALOUZ.mjs → chunk-55WEJKWT.mjs} +117 -17
- package/dist/{chunk-DFFQOA7S.mjs → chunk-DFW3U5UW.mjs} +134 -159
- package/dist/{chunk-V3B24XR7.mjs → chunk-IOZPBNSG.mjs} +2 -1
- package/dist/{chunk-D4EF7J3I.mjs → chunk-P2BEMMOD.mjs} +193 -58
- package/dist/{chunk-PXDY5VRT.mjs → chunk-PJWG547Z.mjs} +2 -2
- package/dist/{chunk-356UOTK2.mjs → chunk-PXDQFYD5.mjs} +7 -1
- package/dist/{chunk-G7IP3UDZ.mjs → chunk-R6WBLBTC.mjs} +45 -26
- package/dist/{chunk-PJU6EF22.mjs → chunk-RTR44V2A.mjs} +267 -108
- package/dist/{chunk-NO3ABFLE.mjs → chunk-WFVOBZS4.mjs} +2 -49
- package/dist/{chunk-BT62MUM5.mjs → chunk-XSZTF4ZQ.mjs} +1 -1
- package/dist/{chunk-BT6BI4GV.mjs → chunk-Z3BXCZXS.mjs} +1 -1
- package/dist/{direct-executor-Q7PNCGZB.mjs → direct-executor-WB2HKU4N.mjs} +6 -6
- package/dist/distribution.mjs +10 -10
- package/dist/index.browser.mjs +488 -260
- package/dist/index.mjs +9 -9
- package/dist/partial-words.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +9 -9
- package/dist/server.mjs +6 -6
- package/dist/{thread-YOTNJEVY.mjs → thread-XBDVXKUE.mjs} +6 -6
- package/dist/workers/entry.mjs +3 -3
- package/package.json +7 -7
package/dist/index.browser.mjs
CHANGED
|
@@ -516,6 +516,7 @@ var MAX_VECTOR_DIMENSION = 4096;
|
|
|
516
516
|
var MAX_STORED_VECTOR_DIMENSION = 65535;
|
|
517
517
|
var MAX_TEXT_FIELDS = 65536;
|
|
518
518
|
var MAX_DOCUMENT_BYTES = 16777216;
|
|
519
|
+
var PARSED_FIELD_TYPE_CACHE_ENTRIES = 1024;
|
|
519
520
|
|
|
520
521
|
// src/schema/validator/shared.ts
|
|
521
522
|
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
@@ -606,10 +607,15 @@ function parseFieldType(type, path) {
|
|
|
606
607
|
dimension
|
|
607
608
|
};
|
|
608
609
|
}
|
|
610
|
+
var parsedTypesByName = /* @__PURE__ */ new Map();
|
|
609
611
|
function parsedTypeOf(type) {
|
|
610
612
|
if (type === void 0) return void 0;
|
|
613
|
+
const known = parsedTypesByName.get(type);
|
|
614
|
+
if (known !== void 0) return known;
|
|
611
615
|
try {
|
|
612
|
-
|
|
616
|
+
const parsed = Object.freeze(parseFieldType(type, ""));
|
|
617
|
+
if (parsedTypesByName.size < PARSED_FIELD_TYPE_CACHE_ENTRIES) parsedTypesByName.set(type, parsed);
|
|
618
|
+
return parsed;
|
|
613
619
|
} catch {
|
|
614
620
|
return void 0;
|
|
615
621
|
}
|
|
@@ -828,7 +834,9 @@ var EMPTY_FROZEN_SLOT = {
|
|
|
828
834
|
readerFor: () => null,
|
|
829
835
|
spellings: () => null,
|
|
830
836
|
bytes: () => 0,
|
|
831
|
-
afterRemoval: () => void 0
|
|
837
|
+
afterRemoval: () => void 0,
|
|
838
|
+
tableOfEveryDocument: () => null,
|
|
839
|
+
countKeptSpellings: () => void 0
|
|
832
840
|
};
|
|
833
841
|
function createGrowingPartialWords(view) {
|
|
834
842
|
return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
|
|
@@ -1289,53 +1297,6 @@ function tokenize(text, language, options) {
|
|
|
1289
1297
|
});
|
|
1290
1298
|
return { tokens, originalTokens, surfaces };
|
|
1291
1299
|
}
|
|
1292
|
-
function* tokenizeIterator(text, language, options) {
|
|
1293
|
-
const {
|
|
1294
|
-
stem: stem2 = true,
|
|
1295
|
-
removeStopWords = true,
|
|
1296
|
-
removeDiacritics = false,
|
|
1297
|
-
collectSurfaces = false,
|
|
1298
|
-
customTokenizer,
|
|
1299
|
-
stopWordOverride
|
|
1300
|
-
} = options ?? {};
|
|
1301
|
-
if (customTokenizer) {
|
|
1302
|
-
const customResult = customTokenizer.tokenize(text);
|
|
1303
|
-
for (const entry of customResult) {
|
|
1304
|
-
yield entry;
|
|
1305
|
-
}
|
|
1306
|
-
return;
|
|
1307
|
-
}
|
|
1308
|
-
const normalized = normalizeForSplitting(text);
|
|
1309
|
-
const rawParts = splitText(normalized, language);
|
|
1310
|
-
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
1311
|
-
const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
|
|
1312
|
-
const stemsTokens = stem2 && language.stemmer !== null;
|
|
1313
|
-
const wantSurfaces = collectSurfaces && stemsTokens;
|
|
1314
|
-
const stopWords2 = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
|
|
1315
|
-
const stripPossessives = language.tokenizer?.stripPossessive ?? false;
|
|
1316
|
-
let position = 0;
|
|
1317
|
-
for (const part of rawParts) {
|
|
1318
|
-
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
1319
|
-
if (candidate.length < minLength) {
|
|
1320
|
-
position++;
|
|
1321
|
-
continue;
|
|
1322
|
-
}
|
|
1323
|
-
if (stopWords2.has(candidate)) {
|
|
1324
|
-
position++;
|
|
1325
|
-
continue;
|
|
1326
|
-
}
|
|
1327
|
-
const processed = transformToken(candidate, language, stem2, effectiveDiacritics);
|
|
1328
|
-
if (processed.length > 0) {
|
|
1329
|
-
if (wantSurfaces) {
|
|
1330
|
-
const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
|
|
1331
|
-
yield { token: processed, position, surface: surface === processed ? void 0 : surface };
|
|
1332
|
-
} else {
|
|
1333
|
-
yield { token: processed, position };
|
|
1334
|
-
}
|
|
1335
|
-
}
|
|
1336
|
-
position++;
|
|
1337
|
-
}
|
|
1338
|
-
}
|
|
1339
1300
|
|
|
1340
1301
|
// src/languages/snowball/base-stemmer.ts
|
|
1341
1302
|
var BaseStemmer = class {
|
|
@@ -3297,6 +3258,9 @@ function createDocumentStore() {
|
|
|
3297
3258
|
},
|
|
3298
3259
|
toInternal(externalId) {
|
|
3299
3260
|
return forwardMap.get(externalId);
|
|
3261
|
+
},
|
|
3262
|
+
isLive(internalId) {
|
|
3263
|
+
return reverseMap[internalId] !== void 0;
|
|
3300
3264
|
}
|
|
3301
3265
|
};
|
|
3302
3266
|
return {
|
|
@@ -5274,8 +5238,25 @@ function createNumericIndex() {
|
|
|
5274
5238
|
};
|
|
5275
5239
|
}
|
|
5276
5240
|
|
|
5241
|
+
// src/core/partition/partial-words/constants.ts
|
|
5242
|
+
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
5243
|
+
|
|
5277
5244
|
// src/core/partition/partial-words/spellings.ts
|
|
5245
|
+
var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
|
|
5246
|
+
function codePointCountOf(text) {
|
|
5247
|
+
let count = 0;
|
|
5248
|
+
for (let index = 0; index < text.length; count++) {
|
|
5249
|
+
index += (text.codePointAt(index) ?? 0) > LAST_BASIC_PLANE_CODE_POINT2 ? 2 : 1;
|
|
5250
|
+
}
|
|
5251
|
+
return count;
|
|
5252
|
+
}
|
|
5253
|
+
function holdsRun(spelling) {
|
|
5254
|
+
if (spelling.length > 2 * PARTIAL_WORD_RUN_CODE_POINTS) return true;
|
|
5255
|
+
if (spelling.length <= PARTIAL_WORD_RUN_CODE_POINTS) return false;
|
|
5256
|
+
return codePointCountOf(spelling) > PARTIAL_WORD_RUN_CODE_POINTS;
|
|
5257
|
+
}
|
|
5278
5258
|
function countSpelling(counts, token, spelling, change) {
|
|
5259
|
+
if (!holdsRun(spelling)) return;
|
|
5279
5260
|
let byToken = counts.get(token);
|
|
5280
5261
|
if (byToken === void 0) {
|
|
5281
5262
|
byToken = /* @__PURE__ */ new Map();
|
|
@@ -5296,11 +5277,12 @@ function textsOf(value) {
|
|
|
5296
5277
|
}
|
|
5297
5278
|
function countDocumentSpellings(counts, document2, analysis, direction) {
|
|
5298
5279
|
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
5280
|
+
const countOccurrence2 = (token, _position, surface) => {
|
|
5281
|
+
countSpelling(counts, token, surface ?? token, direction);
|
|
5282
|
+
};
|
|
5299
5283
|
for (const field of analysis.fields) {
|
|
5300
5284
|
for (const text of textsOf(getNestedValue(document2, field))) {
|
|
5301
|
-
|
|
5302
|
-
countSpelling(counts, occurrence.token, occurrence.surface ?? occurrence.token, direction);
|
|
5303
|
-
}
|
|
5285
|
+
visitTokens(text, analysis.language, tokenizing, countOccurrence2);
|
|
5304
5286
|
}
|
|
5305
5287
|
}
|
|
5306
5288
|
}
|
|
@@ -5320,6 +5302,13 @@ function encodePartialSpellings(fields, counts) {
|
|
|
5320
5302
|
}
|
|
5321
5303
|
return encoded;
|
|
5322
5304
|
}
|
|
5305
|
+
function sameFields(left, right) {
|
|
5306
|
+
if (left.length !== right.length) return false;
|
|
5307
|
+
for (let at = 0; at < left.length; at++) {
|
|
5308
|
+
if (left[at] !== right[at]) return false;
|
|
5309
|
+
}
|
|
5310
|
+
return true;
|
|
5311
|
+
}
|
|
5323
5312
|
|
|
5324
5313
|
// src/core/partition/value-sizes.ts
|
|
5325
5314
|
function keepsValueSize(base) {
|
|
@@ -5438,27 +5427,27 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
5438
5427
|
const surfaceCounts = /* @__PURE__ */ new Map();
|
|
5439
5428
|
if (state.trackPositions) {
|
|
5440
5429
|
const tokenFreqs = /* @__PURE__ */ new Map();
|
|
5441
|
-
|
|
5442
|
-
const existing = tokenFreqs.get(
|
|
5430
|
+
visitTokens(text, language, opts, (token, position, surface) => {
|
|
5431
|
+
const existing = tokenFreqs.get(token);
|
|
5443
5432
|
if (existing) {
|
|
5444
5433
|
existing.count++;
|
|
5445
|
-
existing.positions.push(
|
|
5434
|
+
existing.positions.push(position);
|
|
5446
5435
|
} else {
|
|
5447
|
-
tokenFreqs.set(
|
|
5436
|
+
tokenFreqs.set(token, { count: 1, positions: [position] });
|
|
5448
5437
|
}
|
|
5449
|
-
fieldTokenList.push(
|
|
5450
|
-
countOccurrence(surfaceCounts, spellings,
|
|
5451
|
-
}
|
|
5438
|
+
fieldTokenList.push(token);
|
|
5439
|
+
countOccurrence(surfaceCounts, spellings, surface, token);
|
|
5440
|
+
});
|
|
5452
5441
|
for (const [token, freq] of tokenFreqs) {
|
|
5453
5442
|
state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex2, freq.positions);
|
|
5454
5443
|
}
|
|
5455
5444
|
} else {
|
|
5456
5445
|
const tokenCounts = /* @__PURE__ */ new Map();
|
|
5457
|
-
|
|
5458
|
-
tokenCounts.set(
|
|
5459
|
-
fieldTokenList.push(
|
|
5460
|
-
countOccurrence(surfaceCounts, spellings,
|
|
5461
|
-
}
|
|
5446
|
+
visitTokens(text, language, opts, (token, _position, surface) => {
|
|
5447
|
+
tokenCounts.set(token, (tokenCounts.get(token) ?? 0) + 1);
|
|
5448
|
+
fieldTokenList.push(token);
|
|
5449
|
+
countOccurrence(surfaceCounts, spellings, surface, token);
|
|
5450
|
+
});
|
|
5462
5451
|
for (const [token, count] of tokenCounts) {
|
|
5463
5452
|
state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
|
|
5464
5453
|
}
|
|
@@ -5475,20 +5464,22 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5475
5464
|
if (state.trackPositions) {
|
|
5476
5465
|
const tokenFreqs = /* @__PURE__ */ new Map();
|
|
5477
5466
|
let positionOffset = 0;
|
|
5478
|
-
|
|
5479
|
-
|
|
5480
|
-
|
|
5481
|
-
|
|
5482
|
-
|
|
5483
|
-
|
|
5484
|
-
|
|
5485
|
-
|
|
5486
|
-
tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
|
|
5487
|
-
}
|
|
5488
|
-
fieldTokenList.push(t.token);
|
|
5489
|
-
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
5490
|
-
itemTokenCount++;
|
|
5467
|
+
let itemTokenCount = 0;
|
|
5468
|
+
const addToken = (token, position, surface) => {
|
|
5469
|
+
const existing = tokenFreqs.get(token);
|
|
5470
|
+
if (existing) {
|
|
5471
|
+
existing.count++;
|
|
5472
|
+
existing.positions.push(positionOffset + position);
|
|
5473
|
+
} else {
|
|
5474
|
+
tokenFreqs.set(token, { count: 1, positions: [positionOffset + position] });
|
|
5491
5475
|
}
|
|
5476
|
+
fieldTokenList.push(token);
|
|
5477
|
+
countOccurrence(surfaceCounts, spellings, surface, token);
|
|
5478
|
+
itemTokenCount++;
|
|
5479
|
+
};
|
|
5480
|
+
for (const item of arr) {
|
|
5481
|
+
itemTokenCount = 0;
|
|
5482
|
+
visitTokens(item, language, opts, addToken);
|
|
5492
5483
|
positionOffset += itemTokenCount;
|
|
5493
5484
|
}
|
|
5494
5485
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
@@ -5497,13 +5488,12 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5497
5488
|
}
|
|
5498
5489
|
} else {
|
|
5499
5490
|
const tokenCounts = /* @__PURE__ */ new Map();
|
|
5500
|
-
|
|
5501
|
-
|
|
5502
|
-
|
|
5503
|
-
|
|
5504
|
-
|
|
5505
|
-
|
|
5506
|
-
}
|
|
5491
|
+
const addToken = (token, _position, surface) => {
|
|
5492
|
+
tokenCounts.set(token, (tokenCounts.get(token) ?? 0) + 1);
|
|
5493
|
+
fieldTokenList.push(token);
|
|
5494
|
+
countOccurrence(surfaceCounts, spellings, surface, token);
|
|
5495
|
+
};
|
|
5496
|
+
for (const item of arr) visitTokens(item, language, opts, addToken);
|
|
5507
5497
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
5508
5498
|
for (const [token, count] of tokenCounts) {
|
|
5509
5499
|
state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
|
|
@@ -5736,6 +5726,7 @@ function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
|
|
|
5736
5726
|
// src/filters/constants.ts
|
|
5737
5727
|
var MAX_FILTER_DEPTH = 30;
|
|
5738
5728
|
var MAX_FILTER_TESTS = 1024;
|
|
5729
|
+
var FILTER_OPERAND_COPY_DEPTH = 3;
|
|
5739
5730
|
var MAX_FILTER_ARRAY_SIZE = 65536;
|
|
5740
5731
|
var GEO_UNIT_METRES = { km: 1e3, mi: 1609.344, m: 1 };
|
|
5741
5732
|
|
|
@@ -6134,9 +6125,19 @@ function requireGeoOperands(fieldPath, filter) {
|
|
|
6134
6125
|
}
|
|
6135
6126
|
|
|
6136
6127
|
// src/filters/operands/lists.ts
|
|
6137
|
-
var WHOLE_VALUE_OPERATORS = [
|
|
6128
|
+
var WHOLE_VALUE_OPERATORS = [
|
|
6129
|
+
"eq",
|
|
6130
|
+
"ne",
|
|
6131
|
+
"in",
|
|
6132
|
+
"nin",
|
|
6133
|
+
"startsWith",
|
|
6134
|
+
"endsWith",
|
|
6135
|
+
"matchesAny",
|
|
6136
|
+
"containsAll"
|
|
6137
|
+
];
|
|
6138
6138
|
var ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
|
|
6139
6139
|
var PRESENCE_OPERATORS = ["exists", "notExists", "isEmpty", "isNotEmpty"];
|
|
6140
|
+
var SIZE_OPERAND_KEYS = ["size"];
|
|
6140
6141
|
var SIZE_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
|
|
6141
6142
|
var SIZE_OPERATOR_SET = new Set(SIZE_OPERATORS);
|
|
6142
6143
|
var ELEMENT_KIND_BY_BASE = {
|
|
@@ -6245,6 +6246,14 @@ function requirePatternText(fieldPath, operator, text, caseFold) {
|
|
|
6245
6246
|
}
|
|
6246
6247
|
}
|
|
6247
6248
|
var ELEMENT_OPERATORS2 = ["matchesAny", "containsAll"];
|
|
6249
|
+
var PATTERN_OPERAND_KEYS = [
|
|
6250
|
+
"contains",
|
|
6251
|
+
"caseFold",
|
|
6252
|
+
"tolerance",
|
|
6253
|
+
...PATTERN_EXPRESSION_OPERATORS,
|
|
6254
|
+
...PATTERN_TEST_OPERATORS,
|
|
6255
|
+
...ELEMENT_OPERATORS2
|
|
6256
|
+
];
|
|
6248
6257
|
function requireExpressions(fieldPath, filter, type) {
|
|
6249
6258
|
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
6250
6259
|
const pattern = filter[operator];
|
|
@@ -6320,18 +6329,20 @@ function requireTolerance(fieldPath, filter) {
|
|
|
6320
6329
|
{ fieldPath, operator: "tolerance", requiredRuns, distinctRuns }
|
|
6321
6330
|
);
|
|
6322
6331
|
}
|
|
6323
|
-
function
|
|
6324
|
-
|
|
6325
|
-
|
|
6326
|
-
|
|
6327
|
-
|
|
6328
|
-
|
|
6329
|
-
|
|
6330
|
-
|
|
6331
|
-
|
|
6332
|
-
|
|
6333
|
-
|
|
6332
|
+
function requireWellFormedOperand(fieldPath, operator, text) {
|
|
6333
|
+
if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) return;
|
|
6334
|
+
invalid(
|
|
6335
|
+
`The text of "${operator}" on field "${fieldPath}" holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so no stored value can hold it`,
|
|
6336
|
+
fieldPath,
|
|
6337
|
+
operator
|
|
6338
|
+
);
|
|
6339
|
+
}
|
|
6340
|
+
function requireWellFormedText(fieldPath, operator, operand) {
|
|
6341
|
+
if (!Array.isArray(operand)) {
|
|
6342
|
+
requireWellFormedOperand(fieldPath, operator, operand);
|
|
6343
|
+
return;
|
|
6334
6344
|
}
|
|
6345
|
+
for (const text of operand) requireWellFormedOperand(fieldPath, operator, text);
|
|
6335
6346
|
}
|
|
6336
6347
|
|
|
6337
6348
|
// src/filters/operands/index.ts
|
|
@@ -6341,9 +6352,16 @@ var EQUALITY_OPERATORS = ["eq", "ne"];
|
|
|
6341
6352
|
var LIST_OPERATORS = ["in", "nin"];
|
|
6342
6353
|
var GEO_OPERATORS = ["radius", "polygon"];
|
|
6343
6354
|
var TEST_MODIFIERS = /* @__PURE__ */ new Set(["caseFold", "tolerance"]);
|
|
6355
|
+
var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
|
|
6356
|
+
var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
|
|
6357
|
+
var PROTOTYPE_KEY = "__proto__";
|
|
6358
|
+
var validatedFilters = /* @__PURE__ */ new WeakMap();
|
|
6344
6359
|
function invalidShape(message) {
|
|
6345
6360
|
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message);
|
|
6346
6361
|
}
|
|
6362
|
+
function invalidKey(message, key) {
|
|
6363
|
+
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message, { key });
|
|
6364
|
+
}
|
|
6347
6365
|
function undeclaredFieldOnStrictIndex(field, option) {
|
|
6348
6366
|
return new NarsilError(
|
|
6349
6367
|
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
@@ -6420,24 +6438,47 @@ function requireValueOperands(fieldPath, filter, type) {
|
|
|
6420
6438
|
if (filter[operator] !== void 0) requireFieldKind(fieldPath, operator, type, ["geopoint"]);
|
|
6421
6439
|
}
|
|
6422
6440
|
}
|
|
6423
|
-
|
|
6424
|
-
|
|
6425
|
-
|
|
6426
|
-
|
|
6427
|
-
|
|
6428
|
-
|
|
6429
|
-
|
|
6430
|
-
|
|
6431
|
-
|
|
6432
|
-
|
|
6441
|
+
var OPERAND_CHECKS = [
|
|
6442
|
+
[[...RANGE_OPERATORS, "between"], requireRangeOperands],
|
|
6443
|
+
[[...TEXT_OPERATORS, ...EQUALITY_OPERATORS, ...LIST_OPERATORS, ...GEO_OPERATORS], requireValueOperands],
|
|
6444
|
+
[WHOLE_VALUE_OPERATORS, requireWholeValueIndex],
|
|
6445
|
+
[PRESENCE_OPERATORS, requirePresenceOperands],
|
|
6446
|
+
[ELEMENT_OPERATORS, requireElementOperands],
|
|
6447
|
+
[SIZE_OPERAND_KEYS, requireSizeOperands],
|
|
6448
|
+
[PATTERN_OPERAND_KEYS, requirePatternOperands],
|
|
6449
|
+
[GEO_OPERATORS, requireGeoOperands]
|
|
6450
|
+
];
|
|
6451
|
+
var CHECKS_BY_OPERATOR = checksByOperator();
|
|
6452
|
+
function checksByOperator() {
|
|
6453
|
+
const checks = /* @__PURE__ */ new Map();
|
|
6454
|
+
for (let at = 0; at < OPERAND_CHECKS.length; at++) {
|
|
6455
|
+
for (const operator of OPERAND_CHECKS[at][0]) checks.set(operator, (checks.get(operator) ?? 0) | 1 << at);
|
|
6456
|
+
}
|
|
6457
|
+
return checks;
|
|
6433
6458
|
}
|
|
6434
|
-
function
|
|
6459
|
+
function testsOfOperator(operator, operand) {
|
|
6460
|
+
if (TEST_MODIFIERS.has(operator)) return 0;
|
|
6461
|
+
return operator === "size" && isObjectRecord(operand) ? Math.max(1, Object.keys(operand).length) : 1;
|
|
6462
|
+
}
|
|
6463
|
+
function requireValidFieldTests(fieldPath, filter, type, walk) {
|
|
6464
|
+
const operators = Object.keys(filter);
|
|
6465
|
+
let wanted = 0;
|
|
6435
6466
|
let tests = 0;
|
|
6436
|
-
for (const
|
|
6437
|
-
if (
|
|
6438
|
-
|
|
6467
|
+
for (const operator of operators) {
|
|
6468
|
+
if (walk.unknownKey === null && !OPERATOR_KEYS.has(operator)) {
|
|
6469
|
+
walk.unknownKey = {
|
|
6470
|
+
message: `Field "${fieldPath}" is tested with "${operator}", which is no filter operator`,
|
|
6471
|
+
key: operator
|
|
6472
|
+
};
|
|
6473
|
+
}
|
|
6474
|
+
tests += testsOfOperator(operator, filter[operator]);
|
|
6475
|
+
wanted |= CHECKS_BY_OPERATOR.get(operator) ?? 0;
|
|
6476
|
+
}
|
|
6477
|
+
countTests(walk, Math.max(1, tests));
|
|
6478
|
+
for (const operator of operators) requireWellFormedText(fieldPath, operator, filter[operator]);
|
|
6479
|
+
for (let at = 0; wanted !== 0; at++, wanted >>>= 1) {
|
|
6480
|
+
if ((wanted & 1) !== 0) OPERAND_CHECKS[at][1](fieldPath, filter, type);
|
|
6439
6481
|
}
|
|
6440
|
-
return Math.max(1, tests);
|
|
6441
6482
|
}
|
|
6442
6483
|
function countTests(walk, tests) {
|
|
6443
6484
|
walk.tests += tests;
|
|
@@ -6445,51 +6486,92 @@ function countTests(walk, tests) {
|
|
|
6445
6486
|
invalidShape(`A filter holds at most ${MAX_FILTER_TESTS} tests across all of its fields and nested expressions`);
|
|
6446
6487
|
}
|
|
6447
6488
|
}
|
|
6448
|
-
function holdsNestedExpressions(
|
|
6449
|
-
const { and, or, not } = expression;
|
|
6489
|
+
function holdsNestedExpressions(and, or, not) {
|
|
6450
6490
|
return Array.isArray(and) && and.length > 0 || Array.isArray(or) && or.length > 0 || not !== void 0;
|
|
6451
6491
|
}
|
|
6452
|
-
function
|
|
6453
|
-
|
|
6454
|
-
|
|
6455
|
-
|
|
6456
|
-
|
|
6457
|
-
|
|
6492
|
+
function noteUnknownExpressionKey(expression, walk) {
|
|
6493
|
+
for (const key in expression) {
|
|
6494
|
+
if (!Object.hasOwn(expression, key) || EXPRESSION_KEYS.has(key)) continue;
|
|
6495
|
+
walk.unknownKey = {
|
|
6496
|
+
message: `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`,
|
|
6497
|
+
key
|
|
6498
|
+
};
|
|
6499
|
+
return;
|
|
6500
|
+
}
|
|
6501
|
+
}
|
|
6502
|
+
function setOwn(record, key, value) {
|
|
6503
|
+
if (key === PROTOTYPE_KEY) {
|
|
6504
|
+
Object.defineProperty(record, key, { value, enumerable: true, writable: true, configurable: true });
|
|
6505
|
+
} else {
|
|
6506
|
+
record[key] = value;
|
|
6507
|
+
}
|
|
6508
|
+
}
|
|
6509
|
+
function ownedOperandOf(value, fieldPath, operator, depth) {
|
|
6510
|
+
if (typeof value !== "object" || value === null || depth > FILTER_OPERAND_COPY_DEPTH) return value;
|
|
6511
|
+
if (Array.isArray(value)) {
|
|
6512
|
+
requireListLength(fieldPath, operator, value);
|
|
6513
|
+
const owned2 = [];
|
|
6514
|
+
for (let at = 0; at < value.length; at++) owned2.push(ownedOperandOf(value[at], fieldPath, operator, depth + 1));
|
|
6515
|
+
return owned2;
|
|
6458
6516
|
}
|
|
6459
|
-
const
|
|
6460
|
-
|
|
6461
|
-
for (const [fieldPath,
|
|
6517
|
+
const record = value;
|
|
6518
|
+
const owned = {};
|
|
6519
|
+
for (const key of Object.keys(record)) setOwn(owned, key, ownedOperandOf(record[key], fieldPath, operator, depth + 1));
|
|
6520
|
+
return owned;
|
|
6521
|
+
}
|
|
6522
|
+
function ownedFieldFilterOf(fieldPath, filter) {
|
|
6523
|
+
const owned = {};
|
|
6524
|
+
for (const operator of Object.keys(filter)) {
|
|
6525
|
+
setOwn(owned, operator, ownedOperandOf(filter[operator], fieldPath, operator, 1));
|
|
6526
|
+
}
|
|
6527
|
+
return owned;
|
|
6528
|
+
}
|
|
6529
|
+
function ownedFieldFiltersOf(fields, walk) {
|
|
6530
|
+
const owned = {};
|
|
6531
|
+
for (const fieldPath of Object.keys(fields)) {
|
|
6462
6532
|
const type = walk.fieldTypes[fieldPath];
|
|
6463
6533
|
if (!Object.hasOwn(walk.fieldTypes, fieldPath) || type === void 0) throw undeclaredFilterField(fieldPath);
|
|
6534
|
+
const filter = fields[fieldPath];
|
|
6464
6535
|
if (!isObjectRecord(filter)) invalidShape(`The filter on field "${fieldPath}" must be an object of tests`);
|
|
6465
|
-
|
|
6466
|
-
|
|
6536
|
+
const ownedFilter = ownedFieldFilterOf(fieldPath, filter);
|
|
6537
|
+
requireValidFieldTests(fieldPath, ownedFilter, type, walk);
|
|
6538
|
+
setOwn(owned, fieldPath, ownedFilter);
|
|
6467
6539
|
}
|
|
6468
|
-
|
|
6469
|
-
["and", and],
|
|
6470
|
-
["or", or]
|
|
6471
|
-
]) {
|
|
6472
|
-
if (list === void 0) continue;
|
|
6473
|
-
if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
|
|
6474
|
-
for (const nested of list) requireValidExpression(nested, walk, depth + 1);
|
|
6475
|
-
}
|
|
6476
|
-
if (not !== void 0) requireValidExpression(not, walk, depth + 1);
|
|
6540
|
+
return owned;
|
|
6477
6541
|
}
|
|
6478
|
-
function
|
|
6479
|
-
if (
|
|
6480
|
-
|
|
6542
|
+
function ownedClauseOf(clause, list, walk, depth) {
|
|
6543
|
+
if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
|
|
6544
|
+
const owned = [];
|
|
6545
|
+
for (let at = 0; at < list.length; at++) owned.push(ownedExpressionOf(list[at], walk, depth + 1));
|
|
6546
|
+
return owned;
|
|
6481
6547
|
}
|
|
6482
|
-
|
|
6483
|
-
|
|
6484
|
-
|
|
6485
|
-
|
|
6486
|
-
|
|
6487
|
-
|
|
6488
|
-
|
|
6489
|
-
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, describe(key), { key });
|
|
6490
|
-
}
|
|
6548
|
+
function ownedExpressionOf(expression, walk, depth) {
|
|
6549
|
+
if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
|
|
6550
|
+
if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
|
|
6551
|
+
if (walk.unknownKey === null) noteUnknownExpressionKey(expression, walk);
|
|
6552
|
+
const { fields, and, or, not } = expression;
|
|
6553
|
+
if (fields !== void 0 && !isObjectRecord(fields)) {
|
|
6554
|
+
invalidShape('"fields" in a filter expression must be an object keyed by field path');
|
|
6491
6555
|
}
|
|
6556
|
+
const owned = {};
|
|
6557
|
+
const fieldCount = fields === void 0 ? 0 : Object.keys(fields).length;
|
|
6558
|
+
if (fieldCount === 0 && !holdsNestedExpressions(and, or, not)) countTests(walk, 1);
|
|
6559
|
+
if (fields !== void 0) owned.fields = ownedFieldFiltersOf(fields, walk);
|
|
6560
|
+
if (and !== void 0) owned.and = ownedClauseOf("and", and, walk, depth);
|
|
6561
|
+
if (or !== void 0) owned.or = ownedClauseOf("or", or, walk, depth);
|
|
6562
|
+
if (not !== void 0) owned.not = ownedExpressionOf(not, walk, depth + 1);
|
|
6563
|
+
return owned;
|
|
6564
|
+
}
|
|
6565
|
+
function validatedFilterOf(expression, fieldTypes) {
|
|
6566
|
+
if (validatedFilters.get(expression) === fieldTypes) return expression;
|
|
6567
|
+
const walk = { fieldTypes, tests: 0, unknownKey: null };
|
|
6568
|
+
const validated = ownedExpressionOf(expression, walk, 0);
|
|
6569
|
+
if (walk.unknownKey !== null) invalidKey(walk.unknownKey.message, walk.unknownKey.key);
|
|
6570
|
+
validatedFilters.set(validated, fieldTypes);
|
|
6571
|
+
return validated;
|
|
6492
6572
|
}
|
|
6573
|
+
|
|
6574
|
+
// src/filters/evaluator.ts
|
|
6493
6575
|
var Narrowing = class {
|
|
6494
6576
|
bits = null;
|
|
6495
6577
|
narrow = (matched) => {
|
|
@@ -6501,24 +6583,11 @@ var Narrowing = class {
|
|
|
6501
6583
|
}
|
|
6502
6584
|
};
|
|
6503
6585
|
function narrowByField(fieldPath, filter, context, narrow) {
|
|
6504
|
-
|
|
6505
|
-
|
|
6506
|
-
|
|
6507
|
-
(key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
|
|
6508
|
-
);
|
|
6509
|
-
const type = context.fieldTypes[fieldPath];
|
|
6510
|
-
const parsed = parsedTypeOf(type);
|
|
6511
|
-
if (type === void 0 || parsed === void 0) throw undeclaredFilterField(fieldPath);
|
|
6512
|
-
const f = filter;
|
|
6513
|
-
requireValidOperands(fieldPath, f, type);
|
|
6514
|
-
narrowByFieldTests(fieldPath, f, parsed, context, narrow.narrow);
|
|
6586
|
+
const parsed = parsedTypeOf(context.fieldTypes[fieldPath]);
|
|
6587
|
+
if (parsed === void 0) throw undeclaredFilterField(fieldPath);
|
|
6588
|
+
narrowByFieldTests(fieldPath, filter, parsed, context, narrow.narrow);
|
|
6515
6589
|
}
|
|
6516
|
-
function
|
|
6517
|
-
requireKnownKeys2(
|
|
6518
|
-
expression,
|
|
6519
|
-
EXPRESSION_KEYS,
|
|
6520
|
-
(key) => `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`
|
|
6521
|
-
);
|
|
6590
|
+
function evaluateExpression(expression, context) {
|
|
6522
6591
|
const narrowing = new Narrowing();
|
|
6523
6592
|
if (expression.fields) {
|
|
6524
6593
|
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
@@ -6526,21 +6595,24 @@ function evaluateFilters(expression, context) {
|
|
|
6526
6595
|
}
|
|
6527
6596
|
}
|
|
6528
6597
|
if (expression.and?.length) {
|
|
6529
|
-
for (const nested of expression.and) narrowing.narrow(
|
|
6598
|
+
for (const nested of expression.and) narrowing.narrow(evaluateExpression(nested, context));
|
|
6530
6599
|
}
|
|
6531
6600
|
if (expression.or?.length) {
|
|
6532
6601
|
const union = createBitSet(context.capacity);
|
|
6533
|
-
for (const nested of expression.or) bitsetOrInto(union,
|
|
6602
|
+
for (const nested of expression.or) bitsetOrInto(union, evaluateExpression(nested, context));
|
|
6534
6603
|
narrowing.narrow(union);
|
|
6535
6604
|
}
|
|
6536
6605
|
if (expression.not) {
|
|
6537
|
-
const excluded =
|
|
6606
|
+
const excluded = evaluateExpression(expression.not, context);
|
|
6538
6607
|
const kept = context.allDocIdsBitset.slice();
|
|
6539
6608
|
bitsetAndNotInto(kept, excluded);
|
|
6540
6609
|
narrowing.narrow(kept);
|
|
6541
6610
|
}
|
|
6542
6611
|
return narrowing.value(() => context.allDocIdsBitset);
|
|
6543
6612
|
}
|
|
6613
|
+
function evaluateFilters(expression, context) {
|
|
6614
|
+
return evaluateExpression(validatedFilterOf(expression, context.fieldTypes), context);
|
|
6615
|
+
}
|
|
6544
6616
|
|
|
6545
6617
|
// src/core/partition/constants.ts
|
|
6546
6618
|
var TARGET_BLOCK_ENTRIES = 128;
|
|
@@ -6550,6 +6622,7 @@ var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
|
6550
6622
|
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
6551
6623
|
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
6552
6624
|
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
6625
|
+
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
6553
6626
|
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
6554
6627
|
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
6555
6628
|
|
|
@@ -7130,7 +7203,7 @@ function ensureColumns(state) {
|
|
|
7130
7203
|
function textRangeBitsetOf(state, fieldPath, fieldType, bounds, capacity) {
|
|
7131
7204
|
const column = ensureColumns(state).column(fieldPath, fieldType, "min");
|
|
7132
7205
|
const resolver = state.docStore.resolver();
|
|
7133
|
-
return selectTextRange(column, bounds, capacity, (internalId) => resolver.
|
|
7206
|
+
return selectTextRange(column, bounds, capacity, (internalId) => resolver.isLive(internalId));
|
|
7134
7207
|
}
|
|
7135
7208
|
function sortedPageOf(state, request) {
|
|
7136
7209
|
ensureColumns(state);
|
|
@@ -7267,7 +7340,7 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
7267
7340
|
const base = wordIndex << 5;
|
|
7268
7341
|
while (word !== 0) {
|
|
7269
7342
|
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
7270
|
-
if (resolver.
|
|
7343
|
+
if (resolver.isLive(base + trailingZeros)) count++;
|
|
7271
7344
|
word &= word - 1;
|
|
7272
7345
|
}
|
|
7273
7346
|
}
|
|
@@ -7536,7 +7609,7 @@ function searchFulltextMatches(state, params) {
|
|
|
7536
7609
|
const base = wordIndex << 5;
|
|
7537
7610
|
while (word !== 0) {
|
|
7538
7611
|
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
7539
|
-
if (resolver.
|
|
7612
|
+
if (resolver.isLive(base + trailingZeros)) count++;
|
|
7540
7613
|
word &= word - 1;
|
|
7541
7614
|
}
|
|
7542
7615
|
}
|
|
@@ -7990,7 +8063,7 @@ function kthBestScore(buffer, selection, resolver) {
|
|
|
7990
8063
|
const internalId = touched[index];
|
|
7991
8064
|
const score = scores[internalId];
|
|
7992
8065
|
if (size === wanted && score <= selection[0]) continue;
|
|
7993
|
-
if (resolver.
|
|
8066
|
+
if (!resolver.isLive(internalId)) continue;
|
|
7994
8067
|
let at = 0;
|
|
7995
8068
|
if (size < wanted) {
|
|
7996
8069
|
at = size++;
|
|
@@ -11053,7 +11126,7 @@ function compositeFilterMatches(subs, layout, filters, schema, meter) {
|
|
|
11053
11126
|
const base = wordIndex << 5;
|
|
11054
11127
|
while (word !== 0) {
|
|
11055
11128
|
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
11056
|
-
if (resolver.
|
|
11129
|
+
if (resolver.isLive(base + trailingZeros)) count++;
|
|
11057
11130
|
word &= word - 1;
|
|
11058
11131
|
}
|
|
11059
11132
|
}
|
|
@@ -11359,6 +11432,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11359
11432
|
}
|
|
11360
11433
|
return void 0;
|
|
11361
11434
|
}
|
|
11435
|
+
function frozenStored(docId) {
|
|
11436
|
+
for (const segment of frozen) {
|
|
11437
|
+
const stored = segment.docStore.get(docId);
|
|
11438
|
+
if (stored !== void 0) return stored;
|
|
11439
|
+
}
|
|
11440
|
+
return void 0;
|
|
11441
|
+
}
|
|
11362
11442
|
return {
|
|
11363
11443
|
get partitionId() {
|
|
11364
11444
|
return partitionId;
|
|
@@ -11531,18 +11611,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11531
11611
|
expandPartialWords(subs(), analysis, []);
|
|
11532
11612
|
},
|
|
11533
11613
|
get(docId, projection) {
|
|
11534
|
-
const
|
|
11535
|
-
if (
|
|
11536
|
-
const stored = owner.docStore.get(docId);
|
|
11537
|
-
return stored === void 0 ? void 0 : cloneProjected(stored.fields, projection);
|
|
11538
|
-
}
|
|
11614
|
+
const stored = frozenStored(docId);
|
|
11615
|
+
if (stored !== void 0) return cloneProjected(stored.fields, projection);
|
|
11539
11616
|
return live.get(docId, projection);
|
|
11540
11617
|
},
|
|
11541
11618
|
getRef(docId) {
|
|
11542
|
-
const
|
|
11543
|
-
if (
|
|
11544
|
-
return owner.docStore.get(docId)?.fields;
|
|
11545
|
-
}
|
|
11619
|
+
const stored = frozenStored(docId);
|
|
11620
|
+
if (stored !== void 0) return stored.fields;
|
|
11546
11621
|
return live.getRef(docId);
|
|
11547
11622
|
},
|
|
11548
11623
|
has(docId) {
|
|
@@ -18320,6 +18395,9 @@ function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
|
18320
18395
|
},
|
|
18321
18396
|
toInternal(externalId) {
|
|
18322
18397
|
return liveOrdinal(externalId);
|
|
18398
|
+
},
|
|
18399
|
+
isLive(internalId) {
|
|
18400
|
+
return internalId >= 0 && internalId < capacity && !tombstones.has(internalId);
|
|
18323
18401
|
}
|
|
18324
18402
|
};
|
|
18325
18403
|
return {
|
|
@@ -18481,7 +18559,14 @@ function wrapDocumentArray(documents) {
|
|
|
18481
18559
|
|
|
18482
18560
|
// src/core/partition/frozen/string-blob.ts
|
|
18483
18561
|
var SURROGATE = /[\ud800-\udfff]/;
|
|
18562
|
+
var UTF8_BYTES_PER_UTF16_UNIT = 3;
|
|
18484
18563
|
var encoder3 = new TextEncoder();
|
|
18564
|
+
var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
|
|
18565
|
+
function encodeLookupKey(text) {
|
|
18566
|
+
if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder3.encode(text);
|
|
18567
|
+
const { written } = encoder3.encodeInto(text, lookupKeyBytes);
|
|
18568
|
+
return lookupKeyBytes.subarray(0, written);
|
|
18569
|
+
}
|
|
18485
18570
|
function encodeOneByOne(strings) {
|
|
18486
18571
|
const encoded = new Array(strings.length);
|
|
18487
18572
|
let blobLength = 0;
|
|
@@ -18543,7 +18628,6 @@ function codePointOrder(strings) {
|
|
|
18543
18628
|
}
|
|
18544
18629
|
|
|
18545
18630
|
// src/core/partition/frozen/external-ids.ts
|
|
18546
|
-
var encoder4 = new TextEncoder();
|
|
18547
18631
|
var decoder2 = new TextDecoder();
|
|
18548
18632
|
function encodeExternalIdTableData(docIds) {
|
|
18549
18633
|
const { blob, offsets } = encodeStringBlob(docIds);
|
|
@@ -18562,11 +18646,13 @@ function wrapExternalIdTable(data) {
|
|
|
18562
18646
|
const { blob, offsets, sortedOrdinals } = data;
|
|
18563
18647
|
const count = sortedOrdinals.length;
|
|
18564
18648
|
const decoded = new Array(count);
|
|
18649
|
+
const decodedOrdinals = /* @__PURE__ */ new Map();
|
|
18565
18650
|
function idAt2(ordinal) {
|
|
18566
18651
|
let id = decoded[ordinal];
|
|
18567
18652
|
if (id === void 0) {
|
|
18568
18653
|
id = decoder2.decode(blob.subarray(offsets[ordinal], offsets[ordinal + 1]));
|
|
18569
18654
|
decoded[ordinal] = id;
|
|
18655
|
+
decodedOrdinals.set(id, ordinal);
|
|
18570
18656
|
}
|
|
18571
18657
|
return id;
|
|
18572
18658
|
}
|
|
@@ -18575,7 +18661,9 @@ function wrapExternalIdTable(data) {
|
|
|
18575
18661
|
bytes: blob.byteLength + offsets.byteLength + sortedOrdinals.byteLength,
|
|
18576
18662
|
idAt: idAt2,
|
|
18577
18663
|
ordinalOf(docId) {
|
|
18578
|
-
const
|
|
18664
|
+
const known = decodedOrdinals.get(docId);
|
|
18665
|
+
if (known !== void 0) return known;
|
|
18666
|
+
const query = encodeLookupKey(docId);
|
|
18579
18667
|
let lo = 0;
|
|
18580
18668
|
let hi = count;
|
|
18581
18669
|
while (lo < hi) {
|
|
@@ -19019,7 +19107,7 @@ function createLazySurfaceReader(load) {
|
|
|
19019
19107
|
}
|
|
19020
19108
|
|
|
19021
19109
|
// src/core/partition/frozen/token-table.ts
|
|
19022
|
-
var
|
|
19110
|
+
var encoder4 = new TextEncoder();
|
|
19023
19111
|
var decoder4 = new TextDecoder();
|
|
19024
19112
|
function compareBytes2(blob, start, end, query) {
|
|
19025
19113
|
const length = end - start;
|
|
@@ -19083,7 +19171,7 @@ function wrapFrozenTokenTable(data) {
|
|
|
19083
19171
|
size,
|
|
19084
19172
|
bytes: blob.byteLength + offsets.byteLength + payloadSlots.byteLength + documentFrequencies.byteLength,
|
|
19085
19173
|
find(token) {
|
|
19086
|
-
const query =
|
|
19174
|
+
const query = encodeLookupKey(token);
|
|
19087
19175
|
const at = lowerBound3(query);
|
|
19088
19176
|
if (at >= size) return -1;
|
|
19089
19177
|
return compareBytes2(blob, offsets[at], offsets[at + 1], query) === 0 ? at : -1;
|
|
@@ -19106,7 +19194,7 @@ function wrapFrozenTokenTable(data) {
|
|
|
19106
19194
|
return documentFrequencies[sortedIndex];
|
|
19107
19195
|
},
|
|
19108
19196
|
prefixRange(prefix) {
|
|
19109
|
-
const bytes =
|
|
19197
|
+
const bytes = encoder4.encode(prefix);
|
|
19110
19198
|
const start = lowerBound3(bytes);
|
|
19111
19199
|
return { start, end: prefixEnd(start, bytes) };
|
|
19112
19200
|
},
|
|
@@ -19124,8 +19212,8 @@ function wrapFrozenTokenTable(data) {
|
|
|
19124
19212
|
fromCodePoint = unit;
|
|
19125
19213
|
toCodePoint = nextCodePoint(unit);
|
|
19126
19214
|
}
|
|
19127
|
-
const from =
|
|
19128
|
-
const to =
|
|
19215
|
+
const from = encoder4.encode(String.fromCodePoint(fromCodePoint));
|
|
19216
|
+
const to = encoder4.encode(String.fromCodePoint(toCodePoint));
|
|
19129
19217
|
return { start: lowerBound3(from), end: lowerBound3(to) };
|
|
19130
19218
|
}
|
|
19131
19219
|
};
|
|
@@ -19146,7 +19234,7 @@ function createFrozenTombstones() {
|
|
|
19146
19234
|
return revision;
|
|
19147
19235
|
},
|
|
19148
19236
|
has(internalId) {
|
|
19149
|
-
return removed.has(internalId);
|
|
19237
|
+
return removed.size > 0 && removed.has(internalId);
|
|
19150
19238
|
},
|
|
19151
19239
|
add(internalId) {
|
|
19152
19240
|
if (removed.has(internalId)) return false;
|
|
@@ -19188,7 +19276,7 @@ function sharedSurfaceTable(payload) {
|
|
|
19188
19276
|
return { blob: sharedUint8(table.blob), offsets: sharedUint32(table.offsets), counts: sharedUint32(table.counts) };
|
|
19189
19277
|
}
|
|
19190
19278
|
function sharePartialWordTable(table) {
|
|
19191
|
-
if (typeof SharedArrayBuffer !== "function") return table;
|
|
19279
|
+
if (typeof SharedArrayBuffer !== "function" || table.runEntries.buffer instanceof SharedArrayBuffer) return table;
|
|
19192
19280
|
return {
|
|
19193
19281
|
fields: [...table.fields],
|
|
19194
19282
|
spellings: sharedUint8(table.spellings),
|
|
@@ -19209,17 +19297,19 @@ function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
|
|
|
19209
19297
|
const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
|
|
19210
19298
|
return table === null ? null : sharePartialWordTable(table);
|
|
19211
19299
|
}
|
|
19300
|
+
function partialWordsMatch(table, analysis) {
|
|
19301
|
+
if (table === null) return analysis === null || analysis.fields.length === 0;
|
|
19302
|
+
return analysis === null || sameFields(table.fields, analysis.fields);
|
|
19303
|
+
}
|
|
19212
19304
|
function shareSegmentArrays(arrays, analysis, segmentId) {
|
|
19213
|
-
|
|
19214
|
-
|
|
19215
|
-
segmentId: segmentId ?? generateId(),
|
|
19216
|
-
partialWords: sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis)
|
|
19217
|
-
};
|
|
19305
|
+
const partialWords = partialWordsMatch(arrays.partialWords, analysis) ? arrays.partialWords : sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis);
|
|
19306
|
+
return { ...arrays, segmentId: segmentId ?? generateId(), partialWords };
|
|
19218
19307
|
}
|
|
19219
|
-
function segmentArraysOf(payload, documentData) {
|
|
19308
|
+
function segmentArraysOf(payload, documentData, reusedPartialWords = null) {
|
|
19220
19309
|
const tokenTable = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
19221
19310
|
const idTable = encodeExternalIdTableData(payload.docIds);
|
|
19222
19311
|
const surfaceTable = payload.surfaceForms === null ? null : encodeSurfaceTable(payload.surfaceForms);
|
|
19312
|
+
const partialWords = reusedPartialWords ?? (payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, wrapFrozenTokenTable(tokenTable)));
|
|
19223
19313
|
return {
|
|
19224
19314
|
documentCount: payload.documentCount,
|
|
19225
19315
|
fieldNames: payload.fieldNames,
|
|
@@ -19241,14 +19331,15 @@ function segmentArraysOf(payload, documentData) {
|
|
|
19241
19331
|
enums: payload.enums,
|
|
19242
19332
|
geo: payload.geo,
|
|
19243
19333
|
pattern: payload.pattern,
|
|
19244
|
-
surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable
|
|
19334
|
+
surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable,
|
|
19335
|
+
partialWords
|
|
19245
19336
|
};
|
|
19246
19337
|
}
|
|
19247
19338
|
function freezeSegmentShared(payload, documents, segmentId, analysis = null) {
|
|
19248
19339
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
19249
19340
|
return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId, analysis);
|
|
19250
19341
|
}
|
|
19251
|
-
function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null) {
|
|
19342
|
+
function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null, reusedPartialWords = null) {
|
|
19252
19343
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
19253
19344
|
const tokenData = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
19254
19345
|
const idData = encodeExternalIdTableData(payload.docIds);
|
|
@@ -19316,7 +19407,7 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis =
|
|
|
19316
19407
|
postings: sharedUint32(entry.postings)
|
|
19317
19408
|
})),
|
|
19318
19409
|
surfaceTable: sharedSurfaceTable(payload),
|
|
19319
|
-
partialWords: sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis)
|
|
19410
|
+
partialWords: reusedPartialWords === null ? sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis) : sharePartialWordTable(reusedPartialWords)
|
|
19320
19411
|
};
|
|
19321
19412
|
}
|
|
19322
19413
|
|
|
@@ -19352,11 +19443,6 @@ function buildGeoReaders(entries) {
|
|
|
19352
19443
|
}
|
|
19353
19444
|
return readers;
|
|
19354
19445
|
}
|
|
19355
|
-
function* survivingDocuments(documentSource, tombstones) {
|
|
19356
|
-
for (let ordinal = 0; ordinal < documentSource.count; ordinal++) {
|
|
19357
|
-
if (!tombstones.has(ordinal)) yield documentSource.readAt(ordinal);
|
|
19358
|
-
}
|
|
19359
|
-
}
|
|
19360
19446
|
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
|
|
19361
19447
|
const tombstones = createFrozenTombstones();
|
|
19362
19448
|
const postingViews = createFrozenPostingViews(source, tombstones);
|
|
@@ -19366,8 +19452,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
19366
19452
|
}
|
|
19367
19453
|
const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
|
|
19368
19454
|
const partialWords = createFrozenPartialWords(partialWordTable, source.tokenTable, {
|
|
19455
|
+
count: documentSource.count,
|
|
19369
19456
|
documentAt: (ordinal) => documentSource.readAt(ordinal),
|
|
19370
|
-
|
|
19457
|
+
isTombstoned: (ordinal) => tombstones.has(ordinal)
|
|
19371
19458
|
});
|
|
19372
19459
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
19373
19460
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
@@ -19819,9 +19906,9 @@ function uniqueSurvivors(range) {
|
|
|
19819
19906
|
}
|
|
19820
19907
|
return seen.size;
|
|
19821
19908
|
}
|
|
19822
|
-
function countRange(range, counts,
|
|
19909
|
+
function countRange(range, counts, keepsEveryDocument2) {
|
|
19823
19910
|
const { arrays } = range.input.segment;
|
|
19824
|
-
if (
|
|
19911
|
+
if (keepsEveryDocument2) {
|
|
19825
19912
|
counts.postings += range.end - range.start;
|
|
19826
19913
|
if (arrays.positionOffsets !== null) {
|
|
19827
19914
|
counts.positions += arrays.positionOffsets[range.end] - arrays.positionOffsets[range.start];
|
|
@@ -19846,15 +19933,15 @@ function mergePostings2(inputs) {
|
|
|
19846
19933
|
for (const input of inputs) {
|
|
19847
19934
|
if (input.segment.arrays.positionOffsets !== null) hasPositions = true;
|
|
19848
19935
|
}
|
|
19849
|
-
const
|
|
19936
|
+
const keepsEveryDocument2 = /* @__PURE__ */ new Map();
|
|
19850
19937
|
for (const input of inputs)
|
|
19851
|
-
|
|
19938
|
+
keepsEveryDocument2.set(
|
|
19852
19939
|
input,
|
|
19853
19940
|
input.remap.every((target) => target >= 0)
|
|
19854
19941
|
);
|
|
19855
19942
|
walkTokens(inputs, (_token, ranges) => {
|
|
19856
19943
|
const counts = { postings: 0, positions: 0 };
|
|
19857
|
-
for (const range of ranges) countRange(range, counts,
|
|
19944
|
+
for (const range of ranges) countRange(range, counts, keepsEveryDocument2.get(range.input) === true);
|
|
19858
19945
|
if (counts.postings === 0) return;
|
|
19859
19946
|
tokenCount += 1;
|
|
19860
19947
|
totalPostings += counts.postings;
|
|
@@ -19878,7 +19965,7 @@ function mergePostings2(inputs) {
|
|
|
19878
19965
|
const remap = range.input.remap;
|
|
19879
19966
|
const fieldIndexRemap = range.input.fieldIndexRemap;
|
|
19880
19967
|
docs += uniqueSurvivors(range);
|
|
19881
|
-
if (
|
|
19968
|
+
if (keepsEveryDocument2.get(range.input) === true) {
|
|
19882
19969
|
const count = range.end - range.start;
|
|
19883
19970
|
for (let k = 0; k < count; k++) {
|
|
19884
19971
|
postingDocIds[postingCursor + k] = remap[arrays.postingDocIds[range.start + k]];
|
|
@@ -20055,12 +20142,33 @@ function mergeDocuments2(inputs, documentCount) {
|
|
|
20055
20142
|
}
|
|
20056
20143
|
return { blob, offsets };
|
|
20057
20144
|
}
|
|
20145
|
+
function keepsEveryDocument(remap) {
|
|
20146
|
+
for (let ordinal = 0; ordinal < remap.length; ordinal++) {
|
|
20147
|
+
if (remap[ordinal] < 0) return false;
|
|
20148
|
+
}
|
|
20149
|
+
return true;
|
|
20150
|
+
}
|
|
20151
|
+
function mergePartialWords(inputs, mergedTokenCount, analysis) {
|
|
20152
|
+
if (analysis === null || analysis.fields.length === 0) return { spellings: null, reused: null };
|
|
20153
|
+
if (inputs.length === 1) {
|
|
20154
|
+
const [only] = inputs;
|
|
20155
|
+
const table = only.segment.partialWords.tableOfEveryDocument(analysis);
|
|
20156
|
+
const keepsTokens = mergedTokenCount === only.segment.arrays.tokenTable.size;
|
|
20157
|
+
if (table !== null && keepsTokens && keepsEveryDocument(only.remap)) return { spellings: null, reused: table };
|
|
20158
|
+
}
|
|
20159
|
+
const counts = /* @__PURE__ */ new Map();
|
|
20160
|
+
for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
|
|
20161
|
+
return { spellings: encodePartialSpellings(analysis.fields, counts), reused: null };
|
|
20162
|
+
}
|
|
20058
20163
|
function mergeFrozenSegments(segments, analysis) {
|
|
20059
20164
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
20060
|
-
const merged = mergeSegmentContents(
|
|
20061
|
-
|
|
20165
|
+
const merged = mergeSegmentContents(
|
|
20166
|
+
segments.map((segment) => ({ segment })),
|
|
20167
|
+
analysis
|
|
20168
|
+
);
|
|
20169
|
+
return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis, merged.reusedPartialWords);
|
|
20062
20170
|
}
|
|
20063
|
-
function mergeSegmentContents(selections) {
|
|
20171
|
+
function mergeSegmentContents(selections, analysis = null) {
|
|
20064
20172
|
const docIds = [];
|
|
20065
20173
|
const fieldNames = [];
|
|
20066
20174
|
const inputs = selections.map((selection) => ({
|
|
@@ -20071,6 +20179,7 @@ function mergeSegmentContents(selections) {
|
|
|
20071
20179
|
const documentCount = docIds.length;
|
|
20072
20180
|
const postings = mergePostings2(inputs);
|
|
20073
20181
|
const lengths = mergeFieldLengths(inputs, documentCount);
|
|
20182
|
+
const partialWords = mergePartialWords(inputs, postings.tokens.length, analysis);
|
|
20074
20183
|
const payload = {
|
|
20075
20184
|
documentCount,
|
|
20076
20185
|
docIds,
|
|
@@ -20087,10 +20196,10 @@ function mergeSegmentContents(selections) {
|
|
|
20087
20196
|
totalFieldLengths: lengths.totalFieldLengths,
|
|
20088
20197
|
docFrequencies: postings.docFrequencies,
|
|
20089
20198
|
surfaceForms: mergeSurfaceForms2(inputs),
|
|
20090
|
-
partialSpellings:
|
|
20199
|
+
partialSpellings: partialWords.spellings,
|
|
20091
20200
|
...mergeFieldIndexes2(inputs)
|
|
20092
20201
|
};
|
|
20093
|
-
return { payload, documents: mergeDocuments2(inputs, documentCount) };
|
|
20202
|
+
return { payload, documents: mergeDocuments2(inputs, documentCount), reusedPartialWords: partialWords.reused };
|
|
20094
20203
|
}
|
|
20095
20204
|
|
|
20096
20205
|
// src/core/partition/growable-uint32.ts
|
|
@@ -23889,7 +23998,7 @@ function selectionOfMask(segment, keep) {
|
|
|
23889
23998
|
}
|
|
23890
23999
|
function frozenLiveTail(live, shared2) {
|
|
23891
24000
|
if (live.count() === 0) return null;
|
|
23892
|
-
const payload =
|
|
24001
|
+
const payload = live.encodeSegment();
|
|
23893
24002
|
const documents = survivorDocumentsOf(partitionStateOf(live), payload);
|
|
23894
24003
|
if (shared2) {
|
|
23895
24004
|
const snapshot = freezeSegmentShared(payload, documents);
|
|
@@ -24709,6 +24818,23 @@ function requireEntriesAscend(blob, offsets, order, name) {
|
|
|
24709
24818
|
}
|
|
24710
24819
|
}
|
|
24711
24820
|
}
|
|
24821
|
+
function requireSpellingEntriesAscend(blob, offsets, tokens, name) {
|
|
24822
|
+
for (let at = 1; at < tokens.length; at++) {
|
|
24823
|
+
const order = compareEntries2(blob, offsets, at - 1, at) || tokens[at - 1] - tokens[at];
|
|
24824
|
+
if (order >= 0) corrupt2(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
|
|
24825
|
+
}
|
|
24826
|
+
}
|
|
24827
|
+
function requireBelow(values, limit, name) {
|
|
24828
|
+
for (let at = 0; at < values.length; at++) {
|
|
24829
|
+
if (values[at] >= limit)
|
|
24830
|
+
corrupt2(`${name} holds ${values[at]} at entry ${at}, at or above ${limit}`, { field: name, at });
|
|
24831
|
+
}
|
|
24832
|
+
}
|
|
24833
|
+
function requireNonZero(values, name) {
|
|
24834
|
+
for (let at = 0; at < values.length; at++) {
|
|
24835
|
+
if (values[at] === 0) corrupt2(`${name} holds 0 at entry ${at}`, { field: name, at });
|
|
24836
|
+
}
|
|
24837
|
+
}
|
|
24712
24838
|
function requirePermutation(order, name) {
|
|
24713
24839
|
const seen = new Uint8Array(order.length);
|
|
24714
24840
|
for (let at = 0; at < order.length; at++) {
|
|
@@ -24848,6 +24974,34 @@ function decodeSurfaceForms(raw, context) {
|
|
|
24848
24974
|
requireUtf8Entries(blob, offsets, "surfaceForms.text");
|
|
24849
24975
|
return counts.length === 0 ? null : { blob, offsets, counts };
|
|
24850
24976
|
}
|
|
24977
|
+
function decodePartialWords(raw, tokenCount, context) {
|
|
24978
|
+
if (raw === null || raw === void 0) return null;
|
|
24979
|
+
const column = requireRecord(raw, "partialWords");
|
|
24980
|
+
const fields = requireStrings(column.fields, "partialWords.fields");
|
|
24981
|
+
const spellings = bytesFrom(requireBytes(column.text, "partialWords.text", 1), context.shared);
|
|
24982
|
+
const spellingOffsets = uint32Column(column, "offsets", "partialWords", context);
|
|
24983
|
+
const spellingTokens = uint32Column(column, "tokens", "partialWords", context);
|
|
24984
|
+
const spellingCounts = uint32Column(column, "counts", "partialWords", context);
|
|
24985
|
+
const entries = spellingTokens.length;
|
|
24986
|
+
requireCount(spellingOffsets.length, entries + 1, "partialWords.offsets");
|
|
24987
|
+
requireCount(spellingCounts.length, entries, "partialWords.counts");
|
|
24988
|
+
requireOffsets(spellingOffsets, spellings.length, "partialWords.offsets");
|
|
24989
|
+
requireUtf8Entries(spellings, spellingOffsets, "partialWords.text");
|
|
24990
|
+
requireBelow(spellingTokens, tokenCount, "partialWords.tokens");
|
|
24991
|
+
requireNonZero(spellingCounts, "partialWords.counts");
|
|
24992
|
+
requireSpellingEntriesAscend(spellings, spellingOffsets, spellingTokens, "partialWords");
|
|
24993
|
+
const runs = uint32sFrom(requireBytes(column.runs, "partialWords.runs", 4 * CODE_POINTS_PER_RUN), context.shared);
|
|
24994
|
+
requireRunsAscend(runs, "partialWords.runs");
|
|
24995
|
+
const runOffsets = uint32Column(column, "runOffsets", "partialWords", context);
|
|
24996
|
+
const runEntries = uint32Column(column, "runEntries", "partialWords", context);
|
|
24997
|
+
requireCount(runOffsets.length, runs.length / CODE_POINTS_PER_RUN + 1, "partialWords.runOffsets");
|
|
24998
|
+
requireOffsets(runOffsets, runEntries.length, "partialWords.runOffsets");
|
|
24999
|
+
requireBelow(runEntries, entries, "partialWords.runEntries");
|
|
25000
|
+
for (let run = 0; run + 1 < runOffsets.length; run++) {
|
|
25001
|
+
requireStrictlyAscending(runEntries, runOffsets[run], runOffsets[run + 1], "partialWords.runEntries");
|
|
25002
|
+
}
|
|
25003
|
+
return { fields, spellings, spellingOffsets, spellingTokens, spellingCounts, runs, runOffsets, runEntries };
|
|
25004
|
+
}
|
|
24851
25005
|
|
|
24852
25006
|
// src/serialization/keyword-segment/types.ts
|
|
24853
25007
|
var KEYWORD_SEGMENT_PAYLOAD_VERSION = 1;
|
|
@@ -24968,12 +25122,13 @@ function decodeKeywordSegment(payload, shared2) {
|
|
|
24968
25122
|
corrupt2(`fieldNames holds ${fieldNames.length} names, above the limit of ${MAX_SEGMENT_FIELD_NAMES}`);
|
|
24969
25123
|
}
|
|
24970
25124
|
const context = { documentCount, shared: shared2 };
|
|
25125
|
+
const postings = decodePostings(raw, fieldNames, context);
|
|
24971
25126
|
const arrays = {
|
|
24972
25127
|
documentCount,
|
|
24973
25128
|
fieldNames,
|
|
24974
25129
|
idTable: decodeIds(raw, context),
|
|
24975
25130
|
documentTable: decodeDocuments(raw, schema, context),
|
|
24976
|
-
...
|
|
25131
|
+
...postings,
|
|
24977
25132
|
...decodeFieldLengths(raw.fieldLengths, context),
|
|
24978
25133
|
numeric: decodeValueColumns(raw.numeric, "numeric", context),
|
|
24979
25134
|
sizes: decodeValueColumns(raw.sizes, "sizes", context),
|
|
@@ -24981,7 +25136,8 @@ function decodeKeywordSegment(payload, shared2) {
|
|
|
24981
25136
|
enums: decodeEnumColumns(raw.enum, context),
|
|
24982
25137
|
geo: decodeGeopointColumns(raw.geopoint, context),
|
|
24983
25138
|
pattern: decodePatternColumns(raw.pattern, context),
|
|
24984
|
-
surfaceTable: decodeSurfaceForms(raw.surfaceForms, context)
|
|
25139
|
+
surfaceTable: decodeSurfaceForms(raw.surfaceForms, context),
|
|
25140
|
+
partialWords: decodePartialWords(raw.partialWords, postings.tokenTable.offsets.length - 1, context)
|
|
24985
25141
|
};
|
|
24986
25142
|
return { schema, tombstones, arrays };
|
|
24987
25143
|
}
|
|
@@ -25105,7 +25261,7 @@ function postingsInTokenOrder(arrays) {
|
|
|
25105
25261
|
}
|
|
25106
25262
|
|
|
25107
25263
|
// src/serialization/keyword-segment/encode.ts
|
|
25108
|
-
var TOP_LEVEL_ENTRIES =
|
|
25264
|
+
var TOP_LEVEL_ENTRIES = 28;
|
|
25109
25265
|
function writeFieldLengths(out, arrays) {
|
|
25110
25266
|
out.array(arrays.fieldLengthNames.length);
|
|
25111
25267
|
for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
|
|
@@ -25167,6 +25323,21 @@ function writeSurfaceForms(out, arrays) {
|
|
|
25167
25323
|
out.key("offsets").binary(littleEndianBytes(table.offsets));
|
|
25168
25324
|
out.key("counts").binary(littleEndianBytes(table.counts));
|
|
25169
25325
|
}
|
|
25326
|
+
function writePartialWords(out, arrays) {
|
|
25327
|
+
const table = arrays.partialWords;
|
|
25328
|
+
if (table === null) {
|
|
25329
|
+
out.value(null);
|
|
25330
|
+
return;
|
|
25331
|
+
}
|
|
25332
|
+
out.map(8).key("fields").value(table.fields);
|
|
25333
|
+
out.key("text").binary(table.spellings);
|
|
25334
|
+
out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
|
|
25335
|
+
out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
|
|
25336
|
+
out.key("counts").binary(littleEndianBytes(table.spellingCounts));
|
|
25337
|
+
out.key("runs").binary(littleEndianBytes(table.runs));
|
|
25338
|
+
out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
|
|
25339
|
+
out.key("runEntries").binary(littleEndianBytes(table.runEntries));
|
|
25340
|
+
}
|
|
25170
25341
|
function encodeKeywordSegment(contents) {
|
|
25171
25342
|
const { arrays } = contents;
|
|
25172
25343
|
const postings = postingsInTokenOrder(arrays);
|
|
@@ -25207,6 +25378,8 @@ function encodeKeywordSegment(contents) {
|
|
|
25207
25378
|
writePatternColumns(out, arrays.pattern);
|
|
25208
25379
|
out.key("surfaceForms");
|
|
25209
25380
|
writeSurfaceForms(out, arrays);
|
|
25381
|
+
out.key("partialWords");
|
|
25382
|
+
writePartialWords(out, arrays);
|
|
25210
25383
|
return out;
|
|
25211
25384
|
}
|
|
25212
25385
|
|
|
@@ -25230,7 +25403,7 @@ var VALUE_ENTRY_BYTES = 12;
|
|
|
25230
25403
|
var GEOPOINT_ENTRY_BYTES = 20;
|
|
25231
25404
|
var TOKEN_ENTRY_BYTES = 12;
|
|
25232
25405
|
var STRING_HEADER_BYTES3 = 5;
|
|
25233
|
-
var
|
|
25406
|
+
var encoder5 = new TextEncoder();
|
|
25234
25407
|
function documentBytes(segment, ordinal) {
|
|
25235
25408
|
const table = segment.arrays.documentTable;
|
|
25236
25409
|
if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
|
|
@@ -25273,7 +25446,7 @@ function chargeColumns(segment, costs) {
|
|
|
25273
25446
|
}
|
|
25274
25447
|
function stringListBytes(values) {
|
|
25275
25448
|
let bytes = STRING_HEADER_BYTES3;
|
|
25276
|
-
for (const value of values) bytes += STRING_HEADER_BYTES3 +
|
|
25449
|
+
for (const value of values) bytes += STRING_HEADER_BYTES3 + encoder5.encode(value).byteLength;
|
|
25277
25450
|
return bytes;
|
|
25278
25451
|
}
|
|
25279
25452
|
function fixedBytesOf(segment, chargesTokens) {
|
|
@@ -25285,7 +25458,7 @@ function fixedBytesOf(segment, chargesTokens) {
|
|
|
25285
25458
|
const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
|
|
25286
25459
|
if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
|
|
25287
25460
|
else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
|
|
25288
|
-
return bytes;
|
|
25461
|
+
return bytes + segment.partialWords.bytes();
|
|
25289
25462
|
}
|
|
25290
25463
|
function keptDocumentsOf(selection, chargesTokens) {
|
|
25291
25464
|
const { segment } = selection;
|
|
@@ -25293,7 +25466,7 @@ function keptDocumentsOf(selection, chargesTokens) {
|
|
|
25293
25466
|
const costs = new Float64Array(count);
|
|
25294
25467
|
const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
|
|
25295
25468
|
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
25296
|
-
costs[ordinal] = perDocument +
|
|
25469
|
+
costs[ordinal] = perDocument + encoder5.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
|
|
25297
25470
|
}
|
|
25298
25471
|
chargePostings(segment, costs, chargesTokens);
|
|
25299
25472
|
chargeColumns(segment, costs);
|
|
@@ -25414,16 +25587,26 @@ function emptyPayload() {
|
|
|
25414
25587
|
sizes: []
|
|
25415
25588
|
};
|
|
25416
25589
|
}
|
|
25417
|
-
function
|
|
25590
|
+
function noSpellingsOf(analysis) {
|
|
25591
|
+
if (analysis === null || analysis.fields.length === 0) return null;
|
|
25592
|
+
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
25593
|
+
}
|
|
25594
|
+
function contentsOfGroup(group, request, tombstones) {
|
|
25595
|
+
const { schema, partialWordAnalysis } = request;
|
|
25418
25596
|
if (group === null) {
|
|
25597
|
+
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
25419
25598
|
return {
|
|
25420
25599
|
schema,
|
|
25421
25600
|
tombstones,
|
|
25422
|
-
arrays: segmentArraysOf(
|
|
25601
|
+
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
25423
25602
|
};
|
|
25424
25603
|
}
|
|
25425
|
-
const merged = mergeSegmentContents(group.selections);
|
|
25426
|
-
return {
|
|
25604
|
+
const merged = mergeSegmentContents(group.selections, partialWordAnalysis);
|
|
25605
|
+
return {
|
|
25606
|
+
schema,
|
|
25607
|
+
tombstones,
|
|
25608
|
+
arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
|
|
25609
|
+
};
|
|
25427
25610
|
}
|
|
25428
25611
|
function requireRoomForAnotherSegment(state) {
|
|
25429
25612
|
if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
|
|
@@ -25436,7 +25619,7 @@ function requireRoomForAnotherSegment(state) {
|
|
|
25436
25619
|
async function writeGroup(state, group) {
|
|
25437
25620
|
const { request } = state;
|
|
25438
25621
|
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
25439
|
-
const contents = contentsOfGroup(group, request
|
|
25622
|
+
const contents = contentsOfGroup(group, request, tombstones);
|
|
25440
25623
|
const encoded = encodeKeywordSegment(contents);
|
|
25441
25624
|
if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
|
|
25442
25625
|
const [first, second] = halvesOf(group);
|
|
@@ -25552,6 +25735,7 @@ async function compactPartitionSegments(input) {
|
|
|
25552
25735
|
partitionId: input.partitionId,
|
|
25553
25736
|
nextSegmentId: input.nextSegmentId,
|
|
25554
25737
|
schema: input.schema,
|
|
25738
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
25555
25739
|
selections: frozen.map((segment) => ({ segment })),
|
|
25556
25740
|
tombstones: retainedTombstones(frozen, contents),
|
|
25557
25741
|
mergesIntoOneFile: true
|
|
@@ -25567,6 +25751,27 @@ async function compactPartitionSegments(input) {
|
|
|
25567
25751
|
}
|
|
25568
25752
|
|
|
25569
25753
|
// src/persistence/durability/segment/write.ts
|
|
25754
|
+
function partialWordAnalysisOf2(config) {
|
|
25755
|
+
const fields = partialFieldsOf(config.schema);
|
|
25756
|
+
if (fields.length === 0) return null;
|
|
25757
|
+
let resolved = null;
|
|
25758
|
+
const resolve = () => {
|
|
25759
|
+
resolved ??= {
|
|
25760
|
+
language: getLanguage(config.language ?? "english"),
|
|
25761
|
+
options: resolvePartitionInsertOptions(config, resolveIndexAnalysis(config))
|
|
25762
|
+
};
|
|
25763
|
+
return resolved;
|
|
25764
|
+
};
|
|
25765
|
+
return {
|
|
25766
|
+
fields,
|
|
25767
|
+
get language() {
|
|
25768
|
+
return resolve().language;
|
|
25769
|
+
},
|
|
25770
|
+
get options() {
|
|
25771
|
+
return resolve().options;
|
|
25772
|
+
}
|
|
25773
|
+
};
|
|
25774
|
+
}
|
|
25570
25775
|
function resolveCompactionThreshold(configured) {
|
|
25571
25776
|
if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
|
|
25572
25777
|
return DEFAULT_COMPACTION_THRESHOLD;
|
|
@@ -25586,6 +25791,7 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
|
|
|
25586
25791
|
partitionId,
|
|
25587
25792
|
nextSegmentId: priorPartition?.nextSegmentId ?? 0,
|
|
25588
25793
|
schema: context.input.metadata.schema,
|
|
25794
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
25589
25795
|
selections: whole.selections,
|
|
25590
25796
|
tombstones: []
|
|
25591
25797
|
});
|
|
@@ -25608,6 +25814,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
25608
25814
|
partitionId,
|
|
25609
25815
|
nextSegmentId,
|
|
25610
25816
|
schema: context.input.metadata.schema,
|
|
25817
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
25611
25818
|
selections: built.selections,
|
|
25612
25819
|
tombstones: built.tombstones
|
|
25613
25820
|
});
|
|
@@ -25626,6 +25833,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
25626
25833
|
indexName: context.indexName,
|
|
25627
25834
|
partitionId,
|
|
25628
25835
|
schema: context.input.metadata.schema,
|
|
25836
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
25629
25837
|
segments,
|
|
25630
25838
|
nextSegmentId,
|
|
25631
25839
|
compactionThreshold: context.compactionThreshold
|
|
@@ -25652,6 +25860,7 @@ async function writeCheckpointSegments(input) {
|
|
|
25652
25860
|
input,
|
|
25653
25861
|
indexName: input.metadata.indexName,
|
|
25654
25862
|
config,
|
|
25863
|
+
partialWordAnalysis: partialWordAnalysisOf2(config),
|
|
25655
25864
|
vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
|
|
25656
25865
|
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
|
|
25657
25866
|
};
|
|
@@ -34313,7 +34522,7 @@ async function* exportLines2(header2, checkpoint) {
|
|
|
34313
34522
|
}
|
|
34314
34523
|
function documentExportStream(header2, checkpoint) {
|
|
34315
34524
|
const lines = exportLines2(header2, checkpoint);
|
|
34316
|
-
const
|
|
34525
|
+
const encoder8 = new TextEncoder();
|
|
34317
34526
|
let released = false;
|
|
34318
34527
|
const release = () => {
|
|
34319
34528
|
if (released) return;
|
|
@@ -34327,14 +34536,14 @@ function documentExportStream(header2, checkpoint) {
|
|
|
34327
34536
|
while (text.length < EXPORT_CHUNK_BYTES) {
|
|
34328
34537
|
const step = await lines.next();
|
|
34329
34538
|
if (step.done) {
|
|
34330
|
-
if (text.length > 0) controller.enqueue(
|
|
34539
|
+
if (text.length > 0) controller.enqueue(encoder8.encode(text));
|
|
34331
34540
|
release();
|
|
34332
34541
|
controller.close();
|
|
34333
34542
|
return;
|
|
34334
34543
|
}
|
|
34335
34544
|
text += step.value;
|
|
34336
34545
|
}
|
|
34337
|
-
controller.enqueue(
|
|
34546
|
+
controller.enqueue(encoder8.encode(text));
|
|
34338
34547
|
} catch (error) {
|
|
34339
34548
|
release();
|
|
34340
34549
|
controller.error(error);
|
|
@@ -34664,10 +34873,10 @@ var REGULAR_FILE = 48;
|
|
|
34664
34873
|
var OLD_REGULAR_FILE = 0;
|
|
34665
34874
|
var PAX_EXTENDED_HEADER = 120;
|
|
34666
34875
|
var PAX_HEADER_NAME = "PaxHeader";
|
|
34667
|
-
var
|
|
34876
|
+
var encoder6 = new TextEncoder();
|
|
34668
34877
|
var decoder7 = new TextDecoder("utf-8", { fatal: true });
|
|
34669
34878
|
function writeString(block, offset, length, value) {
|
|
34670
|
-
const bytes =
|
|
34879
|
+
const bytes = encoder6.encode(value);
|
|
34671
34880
|
block.set(bytes.subarray(0, length), offset);
|
|
34672
34881
|
}
|
|
34673
34882
|
function writeOctal(block, offset, digits, value) {
|
|
@@ -34700,11 +34909,11 @@ function ustarBlock(name, prefix, size, typeflag) {
|
|
|
34700
34909
|
return block;
|
|
34701
34910
|
}
|
|
34702
34911
|
function splitForUstar(name) {
|
|
34703
|
-
if (
|
|
34912
|
+
if (encoder6.encode(name).byteLength <= NAME_BYTES) return { name, prefix: "" };
|
|
34704
34913
|
for (let slash = name.indexOf("/"); slash >= 0; slash = name.indexOf("/", slash + 1)) {
|
|
34705
34914
|
const prefix = name.slice(0, slash);
|
|
34706
34915
|
const rest = name.slice(slash + 1);
|
|
34707
|
-
if (
|
|
34916
|
+
if (encoder6.encode(prefix).byteLength <= PREFIX_BYTES && encoder6.encode(rest).byteLength <= NAME_BYTES) {
|
|
34708
34917
|
return { name: rest, prefix };
|
|
34709
34918
|
}
|
|
34710
34919
|
}
|
|
@@ -34713,7 +34922,7 @@ function splitForUstar(name) {
|
|
|
34713
34922
|
function paxRecord(key, value) {
|
|
34714
34923
|
const body = ` ${key}=${value}
|
|
34715
34924
|
`;
|
|
34716
|
-
const bodyBytes =
|
|
34925
|
+
const bodyBytes = encoder6.encode(body).byteLength;
|
|
34717
34926
|
let digits = 1;
|
|
34718
34927
|
while (String(bodyBytes + digits).length !== digits) digits += 1;
|
|
34719
34928
|
return `${bodyBytes + digits}${body}`;
|
|
@@ -34725,7 +34934,7 @@ function tarPadding(size) {
|
|
|
34725
34934
|
function tarEntryHeaders(name, size) {
|
|
34726
34935
|
const split = splitForUstar(name);
|
|
34727
34936
|
if (split !== null) return [ustarBlock(split.name, split.prefix, size, REGULAR_FILE)];
|
|
34728
|
-
const record =
|
|
34937
|
+
const record = encoder6.encode(paxRecord("path", name));
|
|
34729
34938
|
return [
|
|
34730
34939
|
ustarBlock(PAX_HEADER_NAME, "", record.byteLength, PAX_EXTENDED_HEADER),
|
|
34731
34940
|
record,
|
|
@@ -35436,7 +35645,9 @@ var NUMBER_TAG2 = 4;
|
|
|
35436
35645
|
var STRING_TAG = 5;
|
|
35437
35646
|
var LIST_TAG = 6;
|
|
35438
35647
|
var MAP_TAG = 7;
|
|
35439
|
-
var
|
|
35648
|
+
var LENGTH_BYTES = 4;
|
|
35649
|
+
var LAST_ASCII_UNIT = 127;
|
|
35650
|
+
var encoder7 = new TextEncoder();
|
|
35440
35651
|
function ensureCapacity2(stream, extra) {
|
|
35441
35652
|
if (stream.length + extra <= stream.bytes.length) return;
|
|
35442
35653
|
const grown = new Uint8Array(Math.max(stream.bytes.length * 2, stream.length + extra));
|
|
@@ -35465,13 +35676,28 @@ function writeNumber(stream, value) {
|
|
|
35465
35676
|
}
|
|
35466
35677
|
stream.length += 8;
|
|
35467
35678
|
}
|
|
35468
|
-
function
|
|
35469
|
-
const encoded =
|
|
35679
|
+
function writeEncodedString(stream, value) {
|
|
35680
|
+
const encoded = encoder7.encode(value);
|
|
35470
35681
|
writeUint32(stream, encoded.length);
|
|
35471
35682
|
ensureCapacity2(stream, encoded.length);
|
|
35472
35683
|
stream.bytes.set(encoded, stream.length);
|
|
35473
35684
|
stream.length += encoded.length;
|
|
35474
35685
|
}
|
|
35686
|
+
function writeString2(stream, value) {
|
|
35687
|
+
ensureCapacity2(stream, LENGTH_BYTES + value.length);
|
|
35688
|
+
const start = stream.length + LENGTH_BYTES;
|
|
35689
|
+
const bytes = stream.bytes;
|
|
35690
|
+
for (let at = 0; at < value.length; at++) {
|
|
35691
|
+
const unit = value.charCodeAt(at);
|
|
35692
|
+
if (unit > LAST_ASCII_UNIT) {
|
|
35693
|
+
writeEncodedString(stream, value);
|
|
35694
|
+
return;
|
|
35695
|
+
}
|
|
35696
|
+
bytes[start + at] = unit;
|
|
35697
|
+
}
|
|
35698
|
+
writeUint32(stream, value.length);
|
|
35699
|
+
stream.length += value.length;
|
|
35700
|
+
}
|
|
35475
35701
|
var NumberList = class {
|
|
35476
35702
|
constructor(members) {
|
|
35477
35703
|
this.members = members;
|
|
@@ -35748,16 +35974,18 @@ function requireBooleanPartialWords(params) {
|
|
|
35748
35974
|
received: setting === null ? "null" : typeof setting
|
|
35749
35975
|
});
|
|
35750
35976
|
}
|
|
35751
|
-
function
|
|
35977
|
+
function validQueryParams(params, config) {
|
|
35752
35978
|
const strict = config.strict === true;
|
|
35753
35979
|
requireValidTypoTolerance(params);
|
|
35754
35980
|
requireBooleanPartialWords(params);
|
|
35755
35981
|
requireSortableFields(params.sort, config.schema, strict);
|
|
35756
|
-
|
|
35982
|
+
const filters = validQueryFilter(params.filters, config.schema);
|
|
35757
35983
|
requireUsableQueryFields(params, config.schema, strict);
|
|
35984
|
+
return filters === params.filters ? params : { ...params, filters };
|
|
35758
35985
|
}
|
|
35759
|
-
function
|
|
35760
|
-
if (filters
|
|
35986
|
+
function validQueryFilter(filters, schema) {
|
|
35987
|
+
if (filters === void 0 || filters === null) return filters;
|
|
35988
|
+
return validatedFilterOf(filters, flattenSchema(schema));
|
|
35761
35989
|
}
|
|
35762
35990
|
function requireVectorSearchable(params, context, needsVector) {
|
|
35763
35991
|
if (!needsVector) return;
|
|
@@ -35873,7 +36101,7 @@ function executeListDocuments(params, context) {
|
|
|
35873
36101
|
const limit = clampListLimit(params.limit);
|
|
35874
36102
|
const signature = sortSignatureOf(params.sort);
|
|
35875
36103
|
requireSortableFields(params.sort, schema, context.strict === true);
|
|
35876
|
-
|
|
36104
|
+
const filters = validQueryFilter(params.filters, schema);
|
|
35877
36105
|
const binding = listBindingOf(params.filters);
|
|
35878
36106
|
let cursor = null;
|
|
35879
36107
|
if (params.cursor !== void 0) {
|
|
@@ -35881,7 +36109,7 @@ function executeListDocuments(params, context) {
|
|
|
35881
36109
|
requireMatchingCursor(cursor, params.cursor, signature, false, binding);
|
|
35882
36110
|
}
|
|
35883
36111
|
const partitions = context.partitionIds === void 0 ? manager.getAllPartitions() : context.partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
|
|
35884
|
-
const filtered =
|
|
36112
|
+
const filtered = filters === void 0 ? null : collectFilterMatches(partitions, filters, schema, context.patternWorkCap);
|
|
35885
36113
|
const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
|
|
35886
36114
|
const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
|
|
35887
36115
|
const projection = resolveProjection(params.document);
|
|
@@ -36618,28 +36846,28 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
36618
36846
|
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
36619
36847
|
return decoded.depth ?? 0;
|
|
36620
36848
|
}
|
|
36621
|
-
async function executeQuery(
|
|
36849
|
+
async function executeQuery(request, callerContext) {
|
|
36622
36850
|
const context = withPatternWork(callerContext);
|
|
36623
36851
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
36624
36852
|
const startTime = now();
|
|
36625
|
-
const limit = clampLimit(
|
|
36626
|
-
const offset = clampOffset(
|
|
36853
|
+
const limit = clampLimit(request.limit);
|
|
36854
|
+
const offset = clampOffset(request.offset);
|
|
36627
36855
|
requireWithinResultWindow(limit, offset);
|
|
36628
|
-
const sortFields = normalizeSort(
|
|
36629
|
-
requireKnownMode(
|
|
36630
|
-
const sortSignature = sortSignatureOf(
|
|
36631
|
-
const hasTerm =
|
|
36632
|
-
const hasVector =
|
|
36633
|
-
const isHybridMode =
|
|
36634
|
-
const isVectorOnly = (
|
|
36635
|
-
if (sortSignature !== null && (isHybridMode ||
|
|
36856
|
+
const sortFields = normalizeSort(request.sort);
|
|
36857
|
+
requireKnownMode(request);
|
|
36858
|
+
const sortSignature = sortSignatureOf(request.sort);
|
|
36859
|
+
const hasTerm = request.term !== void 0 && request.term.trim().length > 0;
|
|
36860
|
+
const hasVector = request.vector !== void 0 && request.vector.value !== void 0;
|
|
36861
|
+
const isHybridMode = request.mode === "hybrid" || hasTerm && hasVector;
|
|
36862
|
+
const isVectorOnly = (request.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
|
|
36863
|
+
if (sortSignature !== null && (isHybridMode || request.hybrid !== void 0)) {
|
|
36636
36864
|
throw new NarsilError(
|
|
36637
36865
|
ErrorCodes.SEARCH_INVALID_MODE,
|
|
36638
36866
|
"A hybrid query cannot carry a sort, because fusion defines the order of hybrid results",
|
|
36639
36867
|
{ sort: sortFields.map((entry) => entry.field) }
|
|
36640
36868
|
);
|
|
36641
36869
|
}
|
|
36642
|
-
|
|
36870
|
+
const params = validQueryParams(request, config);
|
|
36643
36871
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
36644
36872
|
let paginated;
|
|
36645
36873
|
let nextCursor;
|
|
@@ -36802,12 +37030,12 @@ async function executeQuery(params, callerContext) {
|
|
|
36802
37030
|
coverage: coverageFor(manager, context.partitionIds)
|
|
36803
37031
|
};
|
|
36804
37032
|
}
|
|
36805
|
-
async function executePreflight(
|
|
37033
|
+
async function executePreflight(request, callerContext) {
|
|
36806
37034
|
const context = withPatternWork(callerContext);
|
|
36807
37035
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
36808
37036
|
const startTime = now();
|
|
36809
|
-
requireKnownMode(
|
|
36810
|
-
|
|
37037
|
+
requireKnownMode(request);
|
|
37038
|
+
const params = validQueryParams(request, config);
|
|
36811
37039
|
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
36812
37040
|
const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
|
|
36813
37041
|
const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
|