@delali/narsil 0.3.1-next.79 → 0.3.1-next.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-1lEYL3WK.d.ts → capabilities-CAdkp2P-.d.ts} +2 -2
- package/dist/{chunk-XSZTF4ZQ.mjs → chunk-2453RJYK.mjs} +2 -2
- package/dist/{chunk-DFW3U5UW.mjs → chunk-4CWHWOBI.mjs} +24 -195
- package/dist/{chunk-R6WBLBTC.mjs → chunk-4D6OH4EF.mjs} +10 -46
- package/dist/{chunk-RTR44V2A.mjs → chunk-4VBG6PEH.mjs} +586 -4
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-5XNIRLFC.mjs} +2 -2
- package/dist/{chunk-2E35UM3G.mjs → chunk-B3UOQJD3.mjs} +1 -1
- package/dist/{chunk-PJWG547Z.mjs → chunk-GNYXCLJS.mjs} +5 -5
- package/dist/{chunk-55WEJKWT.mjs → chunk-IH3GKNDQ.mjs} +197 -93
- package/dist/{chunk-AMGDG4ZJ.mjs → chunk-IN5CH4P2.mjs} +3 -3
- package/dist/{chunk-PXDQFYD5.mjs → chunk-KJ7LB3TR.mjs} +24 -140
- package/dist/chunk-LS5IRM23.mjs +140 -0
- package/dist/{chunk-4MQ5F4IL.mjs → chunk-OLXM65YH.mjs} +1 -1
- package/dist/{chunk-IBR6634K.mjs → chunk-R6A773WK.mjs} +2 -69
- package/dist/{chunk-4344I4MR.mjs → chunk-SREPGC6O.mjs} +336 -13
- package/dist/{chunk-I64LXKO3.mjs → chunk-TPJU7NDJ.mjs} +1 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-P2BEMMOD.mjs → chunk-XYDP7CCJ.mjs} +151 -204
- package/dist/{chunk-Z3BXCZXS.mjs → chunk-XZF4LZ3X.mjs} +5 -3
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-ZCKPRCYI.mjs +30 -0
- package/dist/{client-ZXcbSViH.d.ts → client-CKUtkLz3.d.ts} +2 -2
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +1 -0
- package/dist/{config-DHL2KsfD.d.ts → config-BuIkeM5C.d.ts} +39 -3
- package/dist/{config-DTe5EbWW.d.ts → config-CSDEPraV.d.ts} +1 -1
- package/dist/direct-executor-JVXDR5Y5.mjs +19 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +3 -2
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +2 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +759 -65
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-BjPBzWkA.d.ts → engine-DMNZJAUq.d.ts} +1 -1
- package/dist/{types-ChrVy4lH.d.ts → index-DzONZUKE.d.ts} +79 -37
- package/dist/index.browser.mjs +138 -539
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +17 -15
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/{memory-DW3OJ0im.d.ts → memory-BPKNx9Tl.d.ts} +1288 -1287
- package/dist/partial-words.mjs +6 -5
- package/dist/pattern.mjs +5 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -7
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +21 -13
- package/dist/{tasks-BTbwBqar.d.ts → tasks-DGyz586z.d.ts} +1 -1
- package/dist/{thread-XBDVXKUE.mjs → thread-33M5AOEI.mjs} +13 -10
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/workers/entry.mjs +5 -5
- package/package.json +13 -7
- package/dist/chunk-IOZPBNSG.mjs +0 -55
- package/dist/chunk-WFVOBZS4.mjs +0 -485
- package/dist/direct-executor-WB2HKU4N.mjs +0 -17
package/dist/index.browser.mjs
CHANGED
|
@@ -49,6 +49,7 @@ var ErrorCodes = {
|
|
|
49
49
|
QUERY_NODE_TIMEOUT: "QUERY_NODE_TIMEOUT",
|
|
50
50
|
QUERY_NO_ACTIVE_REPLICA: "QUERY_NO_ACTIVE_REPLICA",
|
|
51
51
|
CLUSTER_OPERATION_UNSUPPORTED: "CLUSTER_OPERATION_UNSUPPORTED",
|
|
52
|
+
CLUSTER_WRITE_BLOCKED: "CLUSTER_WRITE_BLOCKED",
|
|
52
53
|
ALLOCATION_NO_DATA_NODES: "ALLOCATION_NO_DATA_NODES",
|
|
53
54
|
ALLOCATION_INVALID_CONFIG: "ALLOCATION_INVALID_CONFIG",
|
|
54
55
|
ALLOCATION_FAILED: "ALLOCATION_FAILED",
|
|
@@ -371,20 +372,6 @@ function appendFoldedCodePoint(codePoint, out) {
|
|
|
371
372
|
}
|
|
372
373
|
for (const foldedCodePoint of folded) out.push(foldedCodePoint);
|
|
373
374
|
}
|
|
374
|
-
function appendFoldedCodePoints(text, out) {
|
|
375
|
-
const single = singleFoldTable();
|
|
376
|
-
const multi = multiFoldTable();
|
|
377
|
-
const cursor = { index: 0, pending: null, pendingIndex: 0 };
|
|
378
|
-
for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
|
|
379
|
-
out.push(codePoint);
|
|
380
|
-
codePoint = nextFoldedCodePoint(text, cursor, single, multi);
|
|
381
|
-
}
|
|
382
|
-
}
|
|
383
|
-
function foldedCodePointCount(text) {
|
|
384
|
-
const folded = [];
|
|
385
|
-
appendFoldedCodePoints(text, folded);
|
|
386
|
-
return folded.length;
|
|
387
|
-
}
|
|
388
375
|
|
|
389
376
|
// src/core/ordering/constants.ts
|
|
390
377
|
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
@@ -797,8 +784,21 @@ function extractVectorFieldsFromSchema(schema) {
|
|
|
797
784
|
|
|
798
785
|
// src/core/partition/partial-words/registry.ts
|
|
799
786
|
var registered = null;
|
|
787
|
+
var MODULE_FUNCTIONS = [
|
|
788
|
+
"createGrowingSlot",
|
|
789
|
+
"createFrozenSlot",
|
|
790
|
+
"buildTable",
|
|
791
|
+
"expand",
|
|
792
|
+
"countSpelling",
|
|
793
|
+
"encodeSpellings",
|
|
794
|
+
"spellingCountsOf",
|
|
795
|
+
"highlightMatcher",
|
|
796
|
+
"addTermMembers",
|
|
797
|
+
"highlightWords"
|
|
798
|
+
];
|
|
800
799
|
function isPartialWordSearchModule(value) {
|
|
801
|
-
|
|
800
|
+
if (value === null || typeof value !== "object" || value.name !== "partialWords") return false;
|
|
801
|
+
return MODULE_FUNCTIONS.every((name) => typeof Reflect.get(value, name) === "function");
|
|
802
802
|
}
|
|
803
803
|
function registerPartialWordSearch(module) {
|
|
804
804
|
if (!isPartialWordSearchModule(module)) {
|
|
@@ -812,12 +812,21 @@ function registerPartialWordSearch(module) {
|
|
|
812
812
|
function availablePartialWordSearch() {
|
|
813
813
|
return registered ?? platformPartialWordSearch();
|
|
814
814
|
}
|
|
815
|
+
var PARTIAL_WORD_IMPORT_ADVICE = `Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load an index with a partial field`;
|
|
816
|
+
function partialWordModule() {
|
|
817
|
+
const module = availablePartialWordSearch();
|
|
818
|
+
if (module !== null) return module;
|
|
819
|
+
throw new NarsilError(
|
|
820
|
+
ErrorCodes.CONFIG_INVALID,
|
|
821
|
+
`This bundle contains no partial-word search code. ${PARTIAL_WORD_IMPORT_ADVICE}`
|
|
822
|
+
);
|
|
823
|
+
}
|
|
815
824
|
function requirePartialWordSearchFor(schema) {
|
|
816
825
|
const fields = partialFieldsOf(schema);
|
|
817
826
|
if (fields.length === 0 || availablePartialWordSearch() !== null) return;
|
|
818
827
|
throw new NarsilError(
|
|
819
828
|
ErrorCodes.CONFIG_INVALID,
|
|
820
|
-
`The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code.
|
|
829
|
+
`The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. ${PARTIAL_WORD_IMPORT_ADVICE}`,
|
|
821
830
|
{ field: fields[0] }
|
|
822
831
|
);
|
|
823
832
|
}
|
|
@@ -851,6 +860,25 @@ function expandPartialWords(subs, analysis, requests) {
|
|
|
851
860
|
const module = availablePartialWordSearch();
|
|
852
861
|
return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
|
|
853
862
|
}
|
|
863
|
+
function countPartialSpelling(counts, token, spelling, change) {
|
|
864
|
+
partialWordModule().countSpelling(counts, token, spelling, change);
|
|
865
|
+
}
|
|
866
|
+
function encodePartialSpellingsOf(fields, counts) {
|
|
867
|
+
return partialWordModule().encodeSpellings(fields, counts);
|
|
868
|
+
}
|
|
869
|
+
function partialSpellingsOfDocuments(documents, analysis) {
|
|
870
|
+
const module = partialWordModule();
|
|
871
|
+
return module.encodeSpellings(analysis.fields, module.spellingCountsOf(documents, analysis));
|
|
872
|
+
}
|
|
873
|
+
function partialWordHighlighter(text, language, partial) {
|
|
874
|
+
return partialWordModule().highlightMatcher(text, language, partial);
|
|
875
|
+
}
|
|
876
|
+
function addPartialWordMembers(query, expand, groupOf) {
|
|
877
|
+
partialWordModule().addTermMembers(query, expand, groupOf);
|
|
878
|
+
}
|
|
879
|
+
function partialWordsToHighlight(queryTokens, written) {
|
|
880
|
+
return partialWordModule().highlightWords(queryTokens, written);
|
|
881
|
+
}
|
|
854
882
|
|
|
855
883
|
// src/core/pattern-index/platform.browser.ts
|
|
856
884
|
function platformPatternSearch() {
|
|
@@ -859,8 +887,22 @@ function platformPatternSearch() {
|
|
|
859
887
|
|
|
860
888
|
// src/core/pattern-index/registry.ts
|
|
861
889
|
var registered2 = null;
|
|
890
|
+
var MODULE_FUNCTIONS2 = [
|
|
891
|
+
"createIndex",
|
|
892
|
+
"readArrays",
|
|
893
|
+
"mergeArrays",
|
|
894
|
+
"matchInto",
|
|
895
|
+
"checkExpression",
|
|
896
|
+
"expressionFootprint",
|
|
897
|
+
"largestFootprint",
|
|
898
|
+
"checkOperands",
|
|
899
|
+
"evaluateTests",
|
|
900
|
+
"serializeIndex",
|
|
901
|
+
"decodeIndex"
|
|
902
|
+
];
|
|
862
903
|
function isPatternSearchModule(value) {
|
|
863
|
-
|
|
904
|
+
if (value === null || typeof value !== "object" || value.name !== "pattern") return false;
|
|
905
|
+
return MODULE_FUNCTIONS2.every((name) => typeof Reflect.get(value, name) === "function");
|
|
864
906
|
}
|
|
865
907
|
function registerPatternSearch(module) {
|
|
866
908
|
if (!isPatternSearchModule(module)) {
|
|
@@ -2584,17 +2626,11 @@ function generateId() {
|
|
|
2584
2626
|
|
|
2585
2627
|
// src/core/pattern-index/constants.ts
|
|
2586
2628
|
var DEFAULT_PATTERN_WORK_CAP = 25e6;
|
|
2587
|
-
var MAX_PATTERN_TEXT_CODE_POINTS = 1024;
|
|
2588
|
-
var MAX_PATTERN_POSITIONS = 2048;
|
|
2589
2629
|
var MATCHER_STATE_BUDGET_BYTES = 1e6;
|
|
2590
|
-
var MATCHER_CODE_POINT_BYTES = 8;
|
|
2591
|
-
var MATCHER_SIMULATION_BYTES_PER_POSITION = 16;
|
|
2592
|
-
var MATCHER_STATE_OVERHEAD_BYTES = 96;
|
|
2593
2630
|
var PATTERN_WORK_FLUSH_UNITS = 65536;
|
|
2594
|
-
var PATTERN_RUN_CODE_POINTS = 3;
|
|
2595
|
-
var MAX_PATTERN_TOLERANCE = 2;
|
|
2596
2631
|
var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "contains"];
|
|
2597
2632
|
var PATTERN_EXPRESSION_OPERATORS = ["wildcard", "regex"];
|
|
2633
|
+
var PATTERN_ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
|
|
2598
2634
|
|
|
2599
2635
|
// src/geo/constants.ts
|
|
2600
2636
|
var DEG_TO_RAD = Math.PI / 180;
|
|
@@ -4124,14 +4160,6 @@ function bitsetHas(bs, index) {
|
|
|
4124
4160
|
if (word >= bs.length) return false;
|
|
4125
4161
|
return (bs[word] & 1 << (index & 31)) !== 0;
|
|
4126
4162
|
}
|
|
4127
|
-
function bitsetAnd(a, b) {
|
|
4128
|
-
const len = Math.min(a.length, b.length);
|
|
4129
|
-
const result = new Uint32Array(len);
|
|
4130
|
-
for (let i = 0; i < len; i++) {
|
|
4131
|
-
result[i] = a[i] & b[i];
|
|
4132
|
-
}
|
|
4133
|
-
return result;
|
|
4134
|
-
}
|
|
4135
4163
|
function bitsetAndInto(target, other) {
|
|
4136
4164
|
const shared2 = Math.min(target.length, other.length);
|
|
4137
4165
|
for (let i = 0; i < shared2; i++) target[i] &= other[i];
|
|
@@ -4145,18 +4173,6 @@ function bitsetAndNotInto(target, removed) {
|
|
|
4145
4173
|
const shared2 = Math.min(target.length, removed.length);
|
|
4146
4174
|
for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
|
|
4147
4175
|
}
|
|
4148
|
-
function bitsetNot(bs, capacity) {
|
|
4149
|
-
const result = new Uint32Array(bs.length);
|
|
4150
|
-
const fullWords = capacity >>> 5;
|
|
4151
|
-
for (let i = 0; i < fullWords; i++) {
|
|
4152
|
-
result[i] = ~bs[i];
|
|
4153
|
-
}
|
|
4154
|
-
const remainder = capacity & 31;
|
|
4155
|
-
if (remainder > 0 && fullWords < bs.length) {
|
|
4156
|
-
result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
|
|
4157
|
-
}
|
|
4158
|
-
return result;
|
|
4159
|
-
}
|
|
4160
4176
|
function bitsetIsEmpty(bs) {
|
|
4161
4177
|
for (let i = 0; i < bs.length; i++) {
|
|
4162
4178
|
if (bs[i] !== 0) return false;
|
|
@@ -4378,24 +4394,7 @@ function computeFacets(state, matched, config, schema) {
|
|
|
4378
4394
|
}
|
|
4379
4395
|
|
|
4380
4396
|
// src/core/pattern-index/host.ts
|
|
4381
|
-
var
|
|
4382
|
-
var foldSources = null;
|
|
4383
|
-
function sourcesByFold() {
|
|
4384
|
-
if (foldSources !== null) return foldSources;
|
|
4385
|
-
const sources = /* @__PURE__ */ new Map();
|
|
4386
|
-
const add = (source, folded) => {
|
|
4387
|
-
const known = sources.get(folded);
|
|
4388
|
-
if (known === void 0) sources.set(folded, [source]);
|
|
4389
|
-
else known.push(source);
|
|
4390
|
-
};
|
|
4391
|
-
for (const [source, folded] of singleFoldTable()) add(source, folded);
|
|
4392
|
-
foldSources = sources;
|
|
4393
|
-
return sources;
|
|
4394
|
-
}
|
|
4395
|
-
function singleFoldSources(folded) {
|
|
4396
|
-
return sourcesByFold().get(folded) ?? NO_SOURCES;
|
|
4397
|
-
}
|
|
4398
|
-
var PATTERN_HOST = { appendFoldedCodePoint, singleFoldSources };
|
|
4397
|
+
var PATTERN_HOST = { appendFoldedCodePoint, singleFolds: singleFoldTable };
|
|
4399
4398
|
function patternIndexOf(indexes, fieldPath) {
|
|
4400
4399
|
let index = indexes.get(fieldPath);
|
|
4401
4400
|
if (index === void 0) {
|
|
@@ -5238,78 +5237,6 @@ function createNumericIndex() {
|
|
|
5238
5237
|
};
|
|
5239
5238
|
}
|
|
5240
5239
|
|
|
5241
|
-
// src/core/partition/partial-words/constants.ts
|
|
5242
|
-
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
5243
|
-
|
|
5244
|
-
// src/core/partition/partial-words/spellings.ts
|
|
5245
|
-
var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
|
|
5246
|
-
function codePointCountOf(text) {
|
|
5247
|
-
let count = 0;
|
|
5248
|
-
for (let index = 0; index < text.length; count++) {
|
|
5249
|
-
index += (text.codePointAt(index) ?? 0) > LAST_BASIC_PLANE_CODE_POINT2 ? 2 : 1;
|
|
5250
|
-
}
|
|
5251
|
-
return count;
|
|
5252
|
-
}
|
|
5253
|
-
function holdsRun(spelling) {
|
|
5254
|
-
if (spelling.length > 2 * PARTIAL_WORD_RUN_CODE_POINTS) return true;
|
|
5255
|
-
if (spelling.length <= PARTIAL_WORD_RUN_CODE_POINTS) return false;
|
|
5256
|
-
return codePointCountOf(spelling) > PARTIAL_WORD_RUN_CODE_POINTS;
|
|
5257
|
-
}
|
|
5258
|
-
function countSpelling(counts, token, spelling, change) {
|
|
5259
|
-
if (!holdsRun(spelling)) return;
|
|
5260
|
-
let byToken = counts.get(token);
|
|
5261
|
-
if (byToken === void 0) {
|
|
5262
|
-
byToken = /* @__PURE__ */ new Map();
|
|
5263
|
-
counts.set(token, byToken);
|
|
5264
|
-
}
|
|
5265
|
-
const next = (byToken.get(spelling) ?? 0) + change;
|
|
5266
|
-
if (next > 0) {
|
|
5267
|
-
byToken.set(spelling, next);
|
|
5268
|
-
return;
|
|
5269
|
-
}
|
|
5270
|
-
byToken.delete(spelling);
|
|
5271
|
-
if (byToken.size === 0) counts.delete(token);
|
|
5272
|
-
}
|
|
5273
|
-
function textsOf(value) {
|
|
5274
|
-
if (typeof value === "string") return [value];
|
|
5275
|
-
if (!Array.isArray(value)) return [];
|
|
5276
|
-
return value.filter((element) => typeof element === "string");
|
|
5277
|
-
}
|
|
5278
|
-
function countDocumentSpellings(counts, document2, analysis, direction) {
|
|
5279
|
-
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
5280
|
-
const countOccurrence2 = (token, _position, surface) => {
|
|
5281
|
-
countSpelling(counts, token, surface ?? token, direction);
|
|
5282
|
-
};
|
|
5283
|
-
for (const field of analysis.fields) {
|
|
5284
|
-
for (const text of textsOf(getNestedValue(document2, field))) {
|
|
5285
|
-
visitTokens(text, analysis.language, tokenizing, countOccurrence2);
|
|
5286
|
-
}
|
|
5287
|
-
}
|
|
5288
|
-
}
|
|
5289
|
-
function spellingCountsOf(documents, analysis) {
|
|
5290
|
-
const counts = /* @__PURE__ */ new Map();
|
|
5291
|
-
for (const document2 of documents) countDocumentSpellings(counts, document2, analysis, 1);
|
|
5292
|
-
return counts;
|
|
5293
|
-
}
|
|
5294
|
-
function encodePartialSpellings(fields, counts) {
|
|
5295
|
-
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
5296
|
-
for (const [token, byToken] of counts) {
|
|
5297
|
-
for (const [spelling, count] of byToken) {
|
|
5298
|
-
encoded.tokens.push(token);
|
|
5299
|
-
encoded.spellings.push(spelling);
|
|
5300
|
-
encoded.counts.push(count);
|
|
5301
|
-
}
|
|
5302
|
-
}
|
|
5303
|
-
return encoded;
|
|
5304
|
-
}
|
|
5305
|
-
function sameFields(left, right) {
|
|
5306
|
-
if (left.length !== right.length) return false;
|
|
5307
|
-
for (let at = 0; at < left.length; at++) {
|
|
5308
|
-
if (left[at] !== right[at]) return false;
|
|
5309
|
-
}
|
|
5310
|
-
return true;
|
|
5311
|
-
}
|
|
5312
|
-
|
|
5313
5240
|
// src/core/partition/value-sizes.ts
|
|
5314
5241
|
function keepsValueSize(base) {
|
|
5315
5242
|
return base === "string" || base === "verbatim" || base.endsWith("[]");
|
|
@@ -5401,7 +5328,7 @@ function countSurface(counts, surface, token) {
|
|
|
5401
5328
|
}
|
|
5402
5329
|
function countOccurrence(counts, spellings, surface, token) {
|
|
5403
5330
|
countSurface(counts, surface, token);
|
|
5404
|
-
if (spellings !== null)
|
|
5331
|
+
if (spellings !== null) countPartialSpelling(spellings, token, surface ?? token, 1);
|
|
5405
5332
|
}
|
|
5406
5333
|
function addFieldOccurrences(occurrencesByToken, fieldTokens) {
|
|
5407
5334
|
const fieldCounts = /* @__PURE__ */ new Map();
|
|
@@ -5730,73 +5657,6 @@ var FILTER_OPERAND_COPY_DEPTH = 3;
|
|
|
5730
5657
|
var MAX_FILTER_ARRAY_SIZE = 65536;
|
|
5731
5658
|
var GEO_UNIT_METRES = { km: 1e3, mi: 1609.344, m: 1 };
|
|
5732
5659
|
|
|
5733
|
-
// src/core/pattern-index/typo-runs.ts
|
|
5734
|
-
var BASIC_PLANE_LAST = 65535;
|
|
5735
|
-
function isPatternTolerance(value) {
|
|
5736
|
-
if (value === "auto") return true;
|
|
5737
|
-
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_PATTERN_TOLERANCE;
|
|
5738
|
-
}
|
|
5739
|
-
function foldTypoText(text, host) {
|
|
5740
|
-
const folded = [];
|
|
5741
|
-
let longestFold = 0;
|
|
5742
|
-
for (let at = 0; at < text.length; at++) {
|
|
5743
|
-
const codePoint = text.codePointAt(at) ?? 0;
|
|
5744
|
-
if (codePoint > BASIC_PLANE_LAST) at++;
|
|
5745
|
-
const before = folded.length;
|
|
5746
|
-
host.appendFoldedCodePoint(codePoint, folded);
|
|
5747
|
-
longestFold = Math.max(longestFold, folded.length - before);
|
|
5748
|
-
}
|
|
5749
|
-
return { folded, longestFold };
|
|
5750
|
-
}
|
|
5751
|
-
function distinctTypoRuns(text, limit) {
|
|
5752
|
-
const { folded } = text;
|
|
5753
|
-
const seen = /* @__PURE__ */ new Set();
|
|
5754
|
-
const runs = [];
|
|
5755
|
-
for (let at = 0; at + PATTERN_RUN_CODE_POINTS <= folded.length && seen.size < limit; at++) {
|
|
5756
|
-
const key = String.fromCodePoint(folded[at], folded[at + 1], folded[at + 2]);
|
|
5757
|
-
if (seen.has(key)) continue;
|
|
5758
|
-
seen.add(key);
|
|
5759
|
-
runs.push(folded[at], folded[at + 1], folded[at + 2]);
|
|
5760
|
-
}
|
|
5761
|
-
return runs;
|
|
5762
|
-
}
|
|
5763
|
-
function typoRunsNeeded(tolerance, longestFold) {
|
|
5764
|
-
return tolerance * (2 + longestFold) + 1;
|
|
5765
|
-
}
|
|
5766
|
-
function distinctTypoRunCount(text, limit) {
|
|
5767
|
-
return distinctTypoRuns(text, limit).length / PATTERN_RUN_CODE_POINTS;
|
|
5768
|
-
}
|
|
5769
|
-
|
|
5770
|
-
// src/filters/pattern-tests.ts
|
|
5771
|
-
function excluding(context, fieldPath, excluded) {
|
|
5772
|
-
const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
|
|
5773
|
-
return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
|
|
5774
|
-
}
|
|
5775
|
-
function textsOf2(operand) {
|
|
5776
|
-
const texts = /* @__PURE__ */ new Set();
|
|
5777
|
-
if (!Array.isArray(operand)) return texts;
|
|
5778
|
-
for (const text of operand) if (typeof text === "string") texts.add(text);
|
|
5779
|
-
return texts;
|
|
5780
|
-
}
|
|
5781
|
-
function evaluatePatternTests(fieldPath, filter, context, bitsets) {
|
|
5782
|
-
const caseFold = filter.caseFold === true;
|
|
5783
|
-
const matching = (operator, texts, tolerance = 0) => {
|
|
5784
|
-
const into = createBitSet(context.capacity);
|
|
5785
|
-
for (const text of texts) context.matchPattern(fieldPath, operator, text, caseFold, tolerance, into);
|
|
5786
|
-
return into;
|
|
5787
|
-
};
|
|
5788
|
-
const eqTolerance = isPatternTolerance(filter.tolerance) ? filter.tolerance : 0;
|
|
5789
|
-
if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq], eqTolerance));
|
|
5790
|
-
if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
|
|
5791
|
-
if (filter.in !== void 0) bitsets.push(matching("eq", textsOf2(filter.in)));
|
|
5792
|
-
if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf2(filter.nin))));
|
|
5793
|
-
if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
|
|
5794
|
-
if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
|
|
5795
|
-
if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
|
|
5796
|
-
if (typeof filter.wildcard === "string") bitsets.push(matching("wildcard", [filter.wildcard]));
|
|
5797
|
-
if (typeof filter.regex === "string") bitsets.push(matching("regex", [filter.regex]));
|
|
5798
|
-
}
|
|
5799
|
-
|
|
5800
5660
|
// src/filters/field-tests.ts
|
|
5801
5661
|
var COMPARISON_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
|
|
5802
5662
|
var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
|
|
@@ -5976,7 +5836,7 @@ function narrowByFieldTests(fieldPath, f, parsed, context, narrow) {
|
|
|
5976
5836
|
narrowByTextRange(fieldPath, f, context, narrow);
|
|
5977
5837
|
if (parsed.pattern) {
|
|
5978
5838
|
const patternBits = [];
|
|
5979
|
-
|
|
5839
|
+
requirePatternSearch(fieldPath).evaluateTests(fieldPath, f, context, patternBits);
|
|
5980
5840
|
for (const matched of patternBits) narrow(matched);
|
|
5981
5841
|
} else {
|
|
5982
5842
|
narrowByValue(fieldPath, f, parsed, context, bits, narrow);
|
|
@@ -6227,50 +6087,14 @@ function requirePatternField(fieldPath, operator, type) {
|
|
|
6227
6087
|
operator
|
|
6228
6088
|
);
|
|
6229
6089
|
}
|
|
6230
|
-
function requireTextLength(fieldPath, operator, text) {
|
|
6231
|
-
if (!exceedsCodePoints(text, MAX_PATTERN_TEXT_CODE_POINTS)) return;
|
|
6232
|
-
invalid(
|
|
6233
|
-
`The text of "${operator}" on pattern field "${fieldPath}" holds more than ${MAX_PATTERN_TEXT_CODE_POINTS} code points`,
|
|
6234
|
-
fieldPath,
|
|
6235
|
-
operator
|
|
6236
|
-
);
|
|
6237
|
-
}
|
|
6238
|
-
function requirePatternText(fieldPath, operator, text, caseFold) {
|
|
6239
|
-
requireTextLength(fieldPath, operator, text);
|
|
6240
|
-
if (caseFold && foldedCodePointCount(text) > MAX_PATTERN_POSITIONS) {
|
|
6241
|
-
invalid(
|
|
6242
|
-
`The text of "${operator}" on pattern field "${fieldPath}" has a case fold of more than ${MAX_PATTERN_POSITIONS} code points, while a pattern holds at most ${MAX_PATTERN_POSITIONS} positions`,
|
|
6243
|
-
fieldPath,
|
|
6244
|
-
operator
|
|
6245
|
-
);
|
|
6246
|
-
}
|
|
6247
|
-
}
|
|
6248
|
-
var ELEMENT_OPERATORS2 = ["matchesAny", "containsAll"];
|
|
6249
6090
|
var PATTERN_OPERAND_KEYS = [
|
|
6250
6091
|
"contains",
|
|
6251
6092
|
"caseFold",
|
|
6252
6093
|
"tolerance",
|
|
6253
6094
|
...PATTERN_EXPRESSION_OPERATORS,
|
|
6254
6095
|
...PATTERN_TEST_OPERATORS,
|
|
6255
|
-
...
|
|
6096
|
+
...PATTERN_ELEMENT_OPERATORS
|
|
6256
6097
|
];
|
|
6257
|
-
function requireExpressions(fieldPath, filter, type) {
|
|
6258
|
-
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
6259
|
-
const pattern = filter[operator];
|
|
6260
|
-
if (pattern === void 0) continue;
|
|
6261
|
-
if (typeof pattern !== "string")
|
|
6262
|
-
invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
|
|
6263
|
-
requirePatternField(fieldPath, operator, type);
|
|
6264
|
-
requireTextLength(fieldPath, operator, pattern);
|
|
6265
|
-
requirePatternSearch(fieldPath).checkExpression(
|
|
6266
|
-
operator,
|
|
6267
|
-
pattern,
|
|
6268
|
-
filter.caseFold === true,
|
|
6269
|
-
fieldPath,
|
|
6270
|
-
PATTERN_HOST
|
|
6271
|
-
);
|
|
6272
|
-
}
|
|
6273
|
-
}
|
|
6274
6098
|
function requirePatternOperands(fieldPath, filter, type) {
|
|
6275
6099
|
if (filter.contains !== void 0) {
|
|
6276
6100
|
if (typeof filter.contains !== "string") {
|
|
@@ -6284,50 +6108,16 @@ function requirePatternOperands(fieldPath, filter, type) {
|
|
|
6284
6108
|
}
|
|
6285
6109
|
requirePatternField(fieldPath, "caseFold", type);
|
|
6286
6110
|
}
|
|
6287
|
-
|
|
6111
|
+
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
6112
|
+
const pattern = filter[operator];
|
|
6113
|
+
if (pattern === void 0) continue;
|
|
6114
|
+
if (typeof pattern !== "string")
|
|
6115
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
|
|
6116
|
+
requirePatternField(fieldPath, operator, type);
|
|
6117
|
+
}
|
|
6288
6118
|
if (filter.tolerance !== void 0) requirePatternField(fieldPath, "tolerance", type);
|
|
6289
6119
|
if (parsedTypeOf(type)?.pattern !== true) return;
|
|
6290
|
-
|
|
6291
|
-
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
6292
|
-
const operand = filter[operator];
|
|
6293
|
-
if (typeof operand === "string") requirePatternText(fieldPath, operator, operand, caseFold);
|
|
6294
|
-
if (!Array.isArray(operand)) continue;
|
|
6295
|
-
for (const text of operand) {
|
|
6296
|
-
if (typeof text === "string") requirePatternText(fieldPath, operator, text, caseFold);
|
|
6297
|
-
}
|
|
6298
|
-
}
|
|
6299
|
-
for (const operator of ELEMENT_OPERATORS2) {
|
|
6300
|
-
const operand = filter[operator];
|
|
6301
|
-
if (!Array.isArray(operand)) continue;
|
|
6302
|
-
for (const text of operand) {
|
|
6303
|
-
if (typeof text === "string") requirePatternText(fieldPath, operator, text, false);
|
|
6304
|
-
}
|
|
6305
|
-
}
|
|
6306
|
-
requireTolerance(fieldPath, filter);
|
|
6307
|
-
}
|
|
6308
|
-
function requireTolerance(fieldPath, filter) {
|
|
6309
|
-
const { tolerance, eq } = filter;
|
|
6310
|
-
if (tolerance === void 0) return;
|
|
6311
|
-
if (!isPatternTolerance(tolerance)) {
|
|
6312
|
-
invalid(`"tolerance" on field "${fieldPath}" must be 0, 1, 2, or "auto"`, fieldPath, "tolerance");
|
|
6313
|
-
}
|
|
6314
|
-
if (typeof eq !== "string") {
|
|
6315
|
-
invalid(
|
|
6316
|
-
`"tolerance" applies only to an "eq" test, while the filter on field "${fieldPath}" sets no "eq" with text`,
|
|
6317
|
-
fieldPath,
|
|
6318
|
-
"tolerance"
|
|
6319
|
-
);
|
|
6320
|
-
}
|
|
6321
|
-
if (tolerance === 0 || tolerance === "auto") return;
|
|
6322
|
-
const text = foldTypoText(eq, PATTERN_HOST);
|
|
6323
|
-
const requiredRuns = typoRunsNeeded(tolerance, text.longestFold);
|
|
6324
|
-
const distinctRuns = distinctTypoRunCount(text, requiredRuns);
|
|
6325
|
-
if (distinctRuns >= requiredRuns) return;
|
|
6326
|
-
throw new NarsilError(
|
|
6327
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
6328
|
-
`A tolerance of ${tolerance} on "eq" of pattern field "${fieldPath}" needs ${requiredRuns} distinct runs of three code points in the case fold of the text, while the text holds ${distinctRuns}. Lengthen the text, lower the tolerance, or set it to "auto"`,
|
|
6329
|
-
{ fieldPath, operator: "tolerance", requiredRuns, distinctRuns }
|
|
6330
|
-
);
|
|
6120
|
+
requirePatternSearch(fieldPath).checkOperands(fieldPath, filter, PATTERN_HOST);
|
|
6331
6121
|
}
|
|
6332
6122
|
function requireWellFormedOperand(fieldPath, operator, text) {
|
|
6333
6123
|
if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) return;
|
|
@@ -8923,99 +8713,18 @@ function mergeSegmentPayload(target, payload, documents) {
|
|
|
8923
8713
|
}
|
|
8924
8714
|
|
|
8925
8715
|
// src/core/partition/pattern-snapshot.ts
|
|
8926
|
-
function loadFailed(fieldPath, problem) {
|
|
8927
|
-
return new NarsilError(
|
|
8928
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
8929
|
-
`The pattern index of field "${fieldPath}" in this partition is corrupt: ${problem}`,
|
|
8930
|
-
{ field: fieldPath }
|
|
8931
|
-
);
|
|
8932
|
-
}
|
|
8933
|
-
function serializePatternIndex(state, fieldPath, index) {
|
|
8934
|
-
const resolver = state.docStore.resolver();
|
|
8935
|
-
const arrays = index.toArrays(fieldPath, null);
|
|
8936
|
-
const positionOf = /* @__PURE__ */ new Map();
|
|
8937
|
-
const docIds = [];
|
|
8938
|
-
for (const ordinal of arrays.docIds) {
|
|
8939
|
-
const externalId = resolver.toExternal(ordinal);
|
|
8940
|
-
if (externalId === void 0) continue;
|
|
8941
|
-
positionOf.set(ordinal, docIds.length);
|
|
8942
|
-
docIds.push(externalId);
|
|
8943
|
-
}
|
|
8944
|
-
const runs = /* @__PURE__ */ Object.create(null);
|
|
8945
|
-
for (let run = 0; run + 1 < arrays.offsets.length; run++) {
|
|
8946
|
-
const positions = [];
|
|
8947
|
-
for (let at = arrays.offsets[run]; at < arrays.offsets[run + 1]; at++) {
|
|
8948
|
-
const position = positionOf.get(arrays.postings[at]);
|
|
8949
|
-
if (position !== void 0) positions.push(position);
|
|
8950
|
-
}
|
|
8951
|
-
if (positions.length === 0) continue;
|
|
8952
|
-
const base = run * PATTERN_RUN_CODE_POINTS;
|
|
8953
|
-
runs[String.fromCodePoint(arrays.runs[base], arrays.runs[base + 1], arrays.runs[base + 2])] = positions;
|
|
8954
|
-
}
|
|
8955
|
-
return { docIds, runs };
|
|
8956
|
-
}
|
|
8957
8716
|
function serializePatternIndexes(state, schema) {
|
|
8958
8717
|
const patternFields = schemaFieldsOf(schema).filter((field) => field.pattern);
|
|
8959
8718
|
if (patternFields.length === 0) return void 0;
|
|
8719
|
+
const resolver = state.docStore.resolver();
|
|
8720
|
+
const toExternal = (ordinal) => resolver.toExternal(ordinal);
|
|
8960
8721
|
const serialized = /* @__PURE__ */ Object.create(null);
|
|
8961
8722
|
for (const { path: fieldPath } of patternFields) {
|
|
8962
8723
|
const index = state.patternIndexes.get(fieldPath);
|
|
8963
|
-
serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } :
|
|
8724
|
+
serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : requirePatternSearch(fieldPath).serializeIndex(fieldPath, index, toExternal);
|
|
8964
8725
|
}
|
|
8965
8726
|
return serialized;
|
|
8966
8727
|
}
|
|
8967
|
-
function appendRunCodePoints(fieldPath, key, out) {
|
|
8968
|
-
const start = out.length;
|
|
8969
|
-
for (const character of key) out.push(character.codePointAt(0) ?? 0);
|
|
8970
|
-
const count = out.length - start;
|
|
8971
|
-
if (count !== PATTERN_RUN_CODE_POINTS) {
|
|
8972
|
-
throw loadFailed(
|
|
8973
|
-
fieldPath,
|
|
8974
|
-
`the key "${key}" holds ${count} code points, while each key must hold ${PATTERN_RUN_CODE_POINTS}`
|
|
8975
|
-
);
|
|
8976
|
-
}
|
|
8977
|
-
}
|
|
8978
|
-
function requirePositions(fieldPath, key, list, documentCount) {
|
|
8979
|
-
if (!Array.isArray(list)) throw loadFailed(fieldPath, `the run "${key}" must map to a list of positions`);
|
|
8980
|
-
let previous = -1;
|
|
8981
|
-
for (const position of list) {
|
|
8982
|
-
if (!Number.isInteger(position) || position < 0 || position >= documentCount) {
|
|
8983
|
-
throw loadFailed(fieldPath, `the run "${key}" names position ${String(position)} outside its document list`);
|
|
8984
|
-
}
|
|
8985
|
-
if (position <= previous) {
|
|
8986
|
-
throw loadFailed(fieldPath, `the positions of the run "${key}" are out of ascending order`);
|
|
8987
|
-
}
|
|
8988
|
-
previous = position;
|
|
8989
|
-
}
|
|
8990
|
-
return list;
|
|
8991
|
-
}
|
|
8992
|
-
function requireDistinctDocuments(fieldPath, docIds) {
|
|
8993
|
-
const seen = /* @__PURE__ */ new Set();
|
|
8994
|
-
for (const docId of docIds) {
|
|
8995
|
-
if (seen.has(docId)) throw loadFailed(fieldPath, `the document list names "${docId}" twice`);
|
|
8996
|
-
seen.add(docId);
|
|
8997
|
-
}
|
|
8998
|
-
}
|
|
8999
|
-
function decodePatternEntry(fieldPath, entry) {
|
|
9000
|
-
requireDistinctDocuments(fieldPath, entry.docIds);
|
|
9001
|
-
const runs = [];
|
|
9002
|
-
const offsets = [0];
|
|
9003
|
-
const postings = [];
|
|
9004
|
-
for (const [key, list] of Object.entries(entry.runs)) {
|
|
9005
|
-
appendRunCodePoints(fieldPath, key, runs);
|
|
9006
|
-
for (const position of requirePositions(fieldPath, key, list, entry.docIds.length)) postings.push(position);
|
|
9007
|
-
offsets.push(postings.length);
|
|
9008
|
-
}
|
|
9009
|
-
const positions = new Uint32Array(entry.docIds.length);
|
|
9010
|
-
for (let position = 0; position < positions.length; position++) positions[position] = position;
|
|
9011
|
-
return {
|
|
9012
|
-
fieldPath,
|
|
9013
|
-
docIds: positions,
|
|
9014
|
-
runs: Uint32Array.from(runs),
|
|
9015
|
-
offsets: Uint32Array.from(offsets),
|
|
9016
|
-
postings: Uint32Array.from(postings)
|
|
9017
|
-
};
|
|
9018
|
-
}
|
|
9019
8728
|
function rebuildPatternIndex(state, fieldPath) {
|
|
9020
8729
|
for (const [docId, stored] of state.docStore.all()) {
|
|
9021
8730
|
const value = getNestedValue(stored.fields, fieldPath);
|
|
@@ -9033,7 +8742,7 @@ function loadPatternIndexes(state, serialized, schema) {
|
|
|
9033
8742
|
rebuildPatternIndex(state, fieldPath);
|
|
9034
8743
|
continue;
|
|
9035
8744
|
}
|
|
9036
|
-
const decoded =
|
|
8745
|
+
const decoded = requirePatternSearch(fieldPath).decodeIndex(fieldPath, entry);
|
|
9037
8746
|
const ordinals = entry.docIds.map((docId) => resolver.toInternal(docId) ?? -1);
|
|
9038
8747
|
patternIndexOf(state.patternIndexes, fieldPath).addArrays(decoded, (position) => ordinals[position]);
|
|
9039
8748
|
}
|
|
@@ -10273,8 +9982,6 @@ var MAX_SORT_FIELDS = 8;
|
|
|
10273
9982
|
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
10274
9983
|
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
10275
9984
|
var PREFIX_MAX_EXPANSIONS = 50;
|
|
10276
|
-
var PARTIAL_WORDS_ADDED_PER_QUERY = 50;
|
|
10277
|
-
var PARTIAL_WORD_MIN_CODE_POINTS = 3;
|
|
10278
9985
|
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
10279
9986
|
var MAX_TOLERANCE = 10;
|
|
10280
9987
|
var MAX_PREFIX_LENGTH = 1024;
|
|
@@ -10624,18 +10331,6 @@ function addCompletions(groups, partition, lastToken, typedPrefix) {
|
|
|
10624
10331
|
group.members.push({ token: completion, factor, partialFactor: factor });
|
|
10625
10332
|
}
|
|
10626
10333
|
}
|
|
10627
|
-
function addPartialWords(group, added) {
|
|
10628
|
-
const membersByToken = /* @__PURE__ */ new Map();
|
|
10629
|
-
for (const member of group.members) membersByToken.set(member.token, member);
|
|
10630
|
-
for (const word of added) {
|
|
10631
|
-
const member = membersByToken.get(word.token);
|
|
10632
|
-
if (member === void 0) {
|
|
10633
|
-
group.members.push({ token: word.token, factor: 0, partialFactor: word.factor });
|
|
10634
|
-
} else if (word.factor > member.partialFactor) {
|
|
10635
|
-
member.partialFactor = word.factor;
|
|
10636
|
-
}
|
|
10637
|
-
}
|
|
10638
|
-
}
|
|
10639
10334
|
function partialWordAnalysisOf(fields, language, analysis) {
|
|
10640
10335
|
return {
|
|
10641
10336
|
fields,
|
|
@@ -10654,29 +10349,6 @@ function searchesAPartialField(params, partialFields) {
|
|
|
10654
10349
|
}
|
|
10655
10350
|
return false;
|
|
10656
10351
|
}
|
|
10657
|
-
function partialWordRequests(params, queryTokens, written, prefixToken) {
|
|
10658
|
-
const eligible = [];
|
|
10659
|
-
for (const queryToken of queryTokens) {
|
|
10660
|
-
const word = written.get(queryToken.position);
|
|
10661
|
-
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
10662
|
-
eligible.push({ token: queryToken.token, written: word });
|
|
10663
|
-
if (eligible.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
10664
|
-
}
|
|
10665
|
-
const count = eligible.length;
|
|
10666
|
-
if (count === 0) return [];
|
|
10667
|
-
const evenShare = Math.floor(PARTIAL_WORDS_ADDED_PER_QUERY / count);
|
|
10668
|
-
const remainder = PARTIAL_WORDS_ADDED_PER_QUERY % count;
|
|
10669
|
-
return eligible.map((entry, order) => ({
|
|
10670
|
-
token: entry.token,
|
|
10671
|
-
request: {
|
|
10672
|
-
written: entry.written,
|
|
10673
|
-
ownToken: entry.token,
|
|
10674
|
-
share: evenShare + (order < remainder ? 1 : 0),
|
|
10675
|
-
tolerance: entry.token === prefixToken ? 0 : params.tolerance ?? 0,
|
|
10676
|
-
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
10677
|
-
}
|
|
10678
|
-
}));
|
|
10679
|
-
}
|
|
10680
10352
|
function resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis) {
|
|
10681
10353
|
const term = params.term;
|
|
10682
10354
|
if (term === void 0 || params.exact === true) return {};
|
|
@@ -10689,16 +10361,18 @@ function resolveTermGroups(partition, params, language, schema, queryTokens, las
|
|
|
10689
10361
|
if (usesPrefix) addCompletions(groups, partition, lastToken, written.get(lastToken.position));
|
|
10690
10362
|
const prefixToken = groups.has(lastToken.token) ? lastToken.token : void 0;
|
|
10691
10363
|
if (usesPartialWords) {
|
|
10692
|
-
|
|
10693
|
-
|
|
10694
|
-
|
|
10695
|
-
|
|
10364
|
+
addPartialWordMembers(
|
|
10365
|
+
{
|
|
10366
|
+
queryTokens,
|
|
10367
|
+
written,
|
|
10368
|
+
prefixToken,
|
|
10369
|
+
tolerance: params.tolerance ?? 0,
|
|
10370
|
+
prefixLength: params.prefixLength,
|
|
10371
|
+
analysis: partialWordAnalysisOf(partialFields, language, analysis)
|
|
10372
|
+
},
|
|
10373
|
+
(partialAnalysis, requests) => partition.expandPartialWords(partialAnalysis, requests),
|
|
10374
|
+
(token, typoMatches) => groupFor(groups, token, typoMatches)
|
|
10696
10375
|
);
|
|
10697
|
-
for (let at = 0; at < requests.length; at++) {
|
|
10698
|
-
if (added[at].length === 0) continue;
|
|
10699
|
-
const token = requests[at].token;
|
|
10700
|
-
addPartialWords(groupFor(groups, token, token !== prefixToken), added[at]);
|
|
10701
|
-
}
|
|
10702
10376
|
}
|
|
10703
10377
|
const termGroups = [...groups.values()];
|
|
10704
10378
|
if (termGroups.length === 0) return {};
|
|
@@ -19248,6 +18922,15 @@ function createFrozenTombstones() {
|
|
|
19248
18922
|
};
|
|
19249
18923
|
}
|
|
19250
18924
|
|
|
18925
|
+
// src/core/partition/partial-words/spellings.ts
|
|
18926
|
+
function sameFields(left, right) {
|
|
18927
|
+
if (left.length !== right.length) return false;
|
|
18928
|
+
for (let at = 0; at < left.length; at++) {
|
|
18929
|
+
if (left[at] !== right[at]) return false;
|
|
18930
|
+
}
|
|
18931
|
+
return true;
|
|
18932
|
+
}
|
|
18933
|
+
|
|
19251
18934
|
// src/core/partition/frozen/shared-snapshot.ts
|
|
19252
18935
|
function sharedUint32(source) {
|
|
19253
18936
|
const copy = new Uint32Array(new SharedArrayBuffer(source.length * 4));
|
|
@@ -19291,7 +18974,7 @@ function sharePartialWordTable(table) {
|
|
|
19291
18974
|
function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
|
|
19292
18975
|
let source = spellings;
|
|
19293
18976
|
if (source === null && analysis !== null && analysis.fields.length > 0) {
|
|
19294
|
-
source =
|
|
18977
|
+
source = partialSpellingsOfDocuments(storedDocumentsOf(documentData), analysis);
|
|
19295
18978
|
}
|
|
19296
18979
|
if (source === null || source.fields.length === 0) return null;
|
|
19297
18980
|
const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
|
|
@@ -20158,7 +19841,7 @@ function mergePartialWords(inputs, mergedTokenCount, analysis) {
|
|
|
20158
19841
|
}
|
|
20159
19842
|
const counts = /* @__PURE__ */ new Map();
|
|
20160
19843
|
for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
|
|
20161
|
-
return { spellings:
|
|
19844
|
+
return { spellings: encodePartialSpellingsOf(analysis.fields, counts), reused: null };
|
|
20162
19845
|
}
|
|
20163
19846
|
function mergeFrozenSegments(segments, analysis) {
|
|
20164
19847
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
@@ -20504,7 +20187,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
20504
20187
|
const addToken = (token, position, surface) => {
|
|
20505
20188
|
postings.addToken(token, positionOffset + position);
|
|
20506
20189
|
surfaces.count(surface, token);
|
|
20507
|
-
if (fieldSpellings !== null)
|
|
20190
|
+
if (fieldSpellings !== null) countPartialSpelling(fieldSpellings, token, surface ?? token, 1);
|
|
20508
20191
|
};
|
|
20509
20192
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
20510
20193
|
const document2 = documents[ordinal].document;
|
|
@@ -20538,7 +20221,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
20538
20221
|
fieldLengthColumns: lengths.columns,
|
|
20539
20222
|
totalFieldLengths: lengths.totalsByName(),
|
|
20540
20223
|
surfaceForms: surfaces.encode(),
|
|
20541
|
-
partialSpellings: partialFields.length === 0 ? null :
|
|
20224
|
+
partialSpellings: partialFields.length === 0 ? null : encodePartialSpellingsOf(partialFields, spellings),
|
|
20542
20225
|
...encodeFieldIndexes(filters, identity)
|
|
20543
20226
|
};
|
|
20544
20227
|
}
|
|
@@ -21406,7 +21089,6 @@ var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
|
|
|
21406
21089
|
var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
|
|
21407
21090
|
var DIRECTORY_LOCK_ATTEMPTS = 3;
|
|
21408
21091
|
var DIRECTORY_LOCK_SETTLE_MS = 50;
|
|
21409
|
-
var DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS = 1e3;
|
|
21410
21092
|
var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
|
|
21411
21093
|
var DEFAULT_COMPACTION_THRESHOLD = 12;
|
|
21412
21094
|
var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
|
|
@@ -21434,6 +21116,16 @@ async function getPath() {
|
|
|
21434
21116
|
return filesystemUnavailable2();
|
|
21435
21117
|
}
|
|
21436
21118
|
|
|
21119
|
+
// src/persistence/durability/lock-holder.browser.ts
|
|
21120
|
+
async function holderIsLive(_holder, _self, _lockPath) {
|
|
21121
|
+
return true;
|
|
21122
|
+
}
|
|
21123
|
+
function holdDirectoryOpen(_directory) {
|
|
21124
|
+
return () => void 0;
|
|
21125
|
+
}
|
|
21126
|
+
async function removeStaleLock(_lockPath, _asidePath, _isStale) {
|
|
21127
|
+
}
|
|
21128
|
+
|
|
21437
21129
|
// src/persistence/durability/directory-lock.ts
|
|
21438
21130
|
var LOCK_FILE_NAME = ".narsil.lock";
|
|
21439
21131
|
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
@@ -21441,14 +21133,6 @@ var lockedDirectories = /* @__PURE__ */ new Set();
|
|
|
21441
21133
|
function errnoOf(err) {
|
|
21442
21134
|
return err?.code;
|
|
21443
21135
|
}
|
|
21444
|
-
function processIsRunning(pid) {
|
|
21445
|
-
try {
|
|
21446
|
-
process.kill(pid, 0);
|
|
21447
|
-
return true;
|
|
21448
|
-
} catch (err) {
|
|
21449
|
-
return errnoOf(err) === "EPERM";
|
|
21450
|
-
}
|
|
21451
|
-
}
|
|
21452
21136
|
function integerOrNull(text) {
|
|
21453
21137
|
const value = Number.parseInt(text ?? "", 10);
|
|
21454
21138
|
return Number.isSafeInteger(value) ? value : null;
|
|
@@ -21486,11 +21170,6 @@ async function settledHolder(fs, lockPath) {
|
|
|
21486
21170
|
await new Promise((resolve) => setTimeout(resolve, DIRECTORY_LOCK_SETTLE_MS));
|
|
21487
21171
|
return readHolder(fs, lockPath);
|
|
21488
21172
|
}
|
|
21489
|
-
function holderIsLive(holder, self) {
|
|
21490
|
-
if (holder.pid !== self.pid) return processIsRunning(holder.pid);
|
|
21491
|
-
if (holder.processStartMs === null || self.processStartMs === null) return false;
|
|
21492
|
-
return Math.abs(holder.processStartMs - self.processStartMs) <= DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS;
|
|
21493
|
-
}
|
|
21494
21173
|
function heldElsewhere(root, holder, lockPath) {
|
|
21495
21174
|
return new NarsilError(
|
|
21496
21175
|
ErrorCodes.CONFIG_INVALID,
|
|
@@ -21533,12 +21212,10 @@ async function acquire(fs, root, lockPath, self) {
|
|
|
21533
21212
|
for (let attempt = 0; attempt < DIRECTORY_LOCK_ATTEMPTS; attempt++) {
|
|
21534
21213
|
if (await createLockFile(fs, lockPath, draftPath, content)) return;
|
|
21535
21214
|
const holder = await settledHolder(fs, lockPath);
|
|
21536
|
-
if (typeof holder === "object" && holderIsLive(
|
|
21215
|
+
if (typeof holder === "object" && await holderIsLive()) {
|
|
21537
21216
|
throw heldElsewhere(root, holderLabel(holder, self), lockPath);
|
|
21538
21217
|
}
|
|
21539
|
-
await
|
|
21540
|
-
if (errnoOf(err) !== "ENOENT") throw err;
|
|
21541
|
-
});
|
|
21218
|
+
await removeStaleLock();
|
|
21542
21219
|
}
|
|
21543
21220
|
throw heldElsewhere(root, "another process", lockPath);
|
|
21544
21221
|
}
|
|
@@ -21556,7 +21233,15 @@ async function lockDirectory(root) {
|
|
|
21556
21233
|
const resolvedRoot = pathMod.resolve(root);
|
|
21557
21234
|
const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
|
|
21558
21235
|
const self = await describeSelf();
|
|
21236
|
+
async function holds() {
|
|
21237
|
+
const holder = await readHolder(fs, lockPath);
|
|
21238
|
+
return typeof holder === "object" && holder.nonce === self.nonce;
|
|
21239
|
+
}
|
|
21240
|
+
async function removeOwnLockFile() {
|
|
21241
|
+
if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
|
|
21242
|
+
}
|
|
21559
21243
|
let directoryKey = null;
|
|
21244
|
+
let closeDirectory = () => void 0;
|
|
21560
21245
|
try {
|
|
21561
21246
|
await fs.mkdir(resolvedRoot, { recursive: true });
|
|
21562
21247
|
const stat = await fs.stat(resolvedRoot, { bigint: true });
|
|
@@ -21565,8 +21250,10 @@ async function lockDirectory(root) {
|
|
|
21565
21250
|
lockedDirectories.add(key);
|
|
21566
21251
|
directoryKey = key;
|
|
21567
21252
|
await acquire(fs, root, lockPath, self);
|
|
21253
|
+
closeDirectory = holdDirectoryOpen(resolvedRoot);
|
|
21568
21254
|
} catch (err) {
|
|
21569
21255
|
if (directoryKey !== null) lockedDirectories.delete(directoryKey);
|
|
21256
|
+
await removeOwnLockFile();
|
|
21570
21257
|
if (err instanceof NarsilError) throw err;
|
|
21571
21258
|
throw new NarsilError(
|
|
21572
21259
|
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
@@ -21576,17 +21263,14 @@ async function lockDirectory(root) {
|
|
|
21576
21263
|
}
|
|
21577
21264
|
const heldKey = directoryKey;
|
|
21578
21265
|
let released = false;
|
|
21579
|
-
async function holds() {
|
|
21580
|
-
const holder = await readHolder(fs, lockPath);
|
|
21581
|
-
return typeof holder === "object" && holder.nonce === self.nonce;
|
|
21582
|
-
}
|
|
21583
21266
|
return {
|
|
21584
21267
|
holds,
|
|
21585
21268
|
async release() {
|
|
21586
21269
|
if (released) return;
|
|
21587
21270
|
released = true;
|
|
21588
21271
|
lockedDirectories.delete(heldKey);
|
|
21589
|
-
|
|
21272
|
+
await removeOwnLockFile();
|
|
21273
|
+
closeDirectory();
|
|
21590
21274
|
}
|
|
21591
21275
|
};
|
|
21592
21276
|
}
|
|
@@ -22112,7 +21796,7 @@ async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, wha
|
|
|
22112
21796
|
checksum});
|
|
22113
21797
|
return { chunks: [header2, ...payloadChunks], byteLength: HEADER_SIZE + payloadLength, checksum };
|
|
22114
21798
|
}
|
|
22115
|
-
function
|
|
21799
|
+
function loadFailed(message, details) {
|
|
22116
21800
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, message, details);
|
|
22117
21801
|
}
|
|
22118
21802
|
async function gunzipWithinLimit(compressed, what) {
|
|
@@ -22125,7 +21809,7 @@ async function gunzipWithinLimit(compressed, what) {
|
|
|
22125
21809
|
try {
|
|
22126
21810
|
step = await reader.read();
|
|
22127
21811
|
} catch (error) {
|
|
22128
|
-
|
|
21812
|
+
loadFailed(`The ${what} holds a gzip payload that fails to inflate`, {
|
|
22129
21813
|
cause: error instanceof Error ? error.message : String(error)
|
|
22130
21814
|
});
|
|
22131
21815
|
}
|
|
@@ -22133,7 +21817,7 @@ async function gunzipWithinLimit(compressed, what) {
|
|
|
22133
21817
|
total += step.value.byteLength;
|
|
22134
21818
|
if (total > MAX_FILE_BYTES) {
|
|
22135
21819
|
await reader.cancel();
|
|
22136
|
-
|
|
21820
|
+
loadFailed(`The ${what} inflates past the file size limit of ${MAX_FILE_BYTES} bytes`);
|
|
22137
21821
|
}
|
|
22138
21822
|
chunks.push(step.value);
|
|
22139
21823
|
}
|
|
@@ -22162,7 +21846,7 @@ async function openChecksummedEnvelope(data, expected) {
|
|
|
22162
21846
|
);
|
|
22163
21847
|
}
|
|
22164
21848
|
if (header2.flags.compressionEnabled && !expected.allowsCompression) {
|
|
22165
|
-
|
|
21849
|
+
loadFailed(`The ${what} is compressed, while its format stores it raw`);
|
|
22166
21850
|
}
|
|
22167
21851
|
if (header2.flags.compressionEnabled && header2.flags.compressionAlgorithm !== "gzip") {
|
|
22168
21852
|
throw new NarsilError(
|
|
@@ -22170,9 +21854,9 @@ async function openChecksummedEnvelope(data, expected) {
|
|
|
22170
21854
|
`The ${what} uses ${header2.flags.compressionAlgorithm} compression, which this build cannot read`
|
|
22171
21855
|
);
|
|
22172
21856
|
}
|
|
22173
|
-
if (!header2.flags.checksumPresent)
|
|
21857
|
+
if (!header2.flags.checksumPresent) loadFailed(`The ${what} carries no checksum, while its format requires one`);
|
|
22174
21858
|
if (data.byteLength !== HEADER_SIZE + header2.payloadLength) {
|
|
22175
|
-
|
|
21859
|
+
loadFailed(
|
|
22176
21860
|
`The ${what} holds ${data.byteLength} bytes where its header gives ${HEADER_SIZE + header2.payloadLength}`,
|
|
22177
21861
|
{ bytes: data.byteLength, expected: HEADER_SIZE + header2.payloadLength }
|
|
22178
21862
|
);
|
|
@@ -30974,65 +30658,18 @@ function stopIdleSweep(state) {
|
|
|
30974
30658
|
}
|
|
30975
30659
|
|
|
30976
30660
|
// src/core/pattern-index/matcher-budget.ts
|
|
30977
|
-
var POSITIONS_PER_WORD = 32;
|
|
30978
|
-
function positionWords(positions) {
|
|
30979
|
-
return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
|
|
30980
|
-
}
|
|
30981
|
-
function smallestMatcherBytes(positions) {
|
|
30982
|
-
return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
|
|
30983
|
-
}
|
|
30984
30661
|
function widestPatternSplit(footprintBytes) {
|
|
30985
30662
|
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / Math.max(1, footprintBytes)));
|
|
30986
30663
|
}
|
|
30987
30664
|
|
|
30988
30665
|
// src/core/pattern-index/query-positions.ts
|
|
30989
|
-
function isRecord6(value) {
|
|
30990
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
30991
|
-
}
|
|
30992
|
-
function literalFootprint(text, caseFold) {
|
|
30993
|
-
if (typeof text !== "string") return null;
|
|
30994
|
-
return smallestMatcherBytes(caseFold ? foldedCodePointCount(text) : codePointCount(text));
|
|
30995
|
-
}
|
|
30996
|
-
function larger(left, right) {
|
|
30997
|
-
if (left === null) return right;
|
|
30998
|
-
if (right === null) return left;
|
|
30999
|
-
return Math.max(left, right);
|
|
31000
|
-
}
|
|
31001
|
-
function largestTestOf(filter, module) {
|
|
31002
|
-
const caseFold = filter.caseFold === true;
|
|
31003
|
-
let largest = null;
|
|
31004
|
-
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
31005
|
-
const operand = filter[operator];
|
|
31006
|
-
if (operand === void 0) continue;
|
|
31007
|
-
const texts = Array.isArray(operand) ? operand : [operand];
|
|
31008
|
-
for (const text of texts) largest = larger(largest, literalFootprint(text, caseFold));
|
|
31009
|
-
}
|
|
31010
|
-
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
31011
|
-
const pattern = filter[operator];
|
|
31012
|
-
if (typeof pattern !== "string") continue;
|
|
31013
|
-
largest = larger(largest, module.expressionFootprint(operator, pattern, caseFold, PATTERN_HOST));
|
|
31014
|
-
}
|
|
31015
|
-
return largest;
|
|
31016
|
-
}
|
|
31017
|
-
function largestIn(expression, patternFields, module) {
|
|
31018
|
-
let largest = null;
|
|
31019
|
-
if (isRecord6(expression.fields)) {
|
|
31020
|
-
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
31021
|
-
if (patternFields.has(fieldPath) && isRecord6(filter)) largest = larger(largest, largestTestOf(filter, module));
|
|
31022
|
-
}
|
|
31023
|
-
}
|
|
31024
|
-
const nested = [...expression.and ?? [], ...expression.or ?? []];
|
|
31025
|
-
if (expression.not !== void 0) nested.push(expression.not);
|
|
31026
|
-
for (const child of nested) largest = larger(largest, largestIn(child, patternFields, module));
|
|
31027
|
-
return largest;
|
|
31028
|
-
}
|
|
31029
30666
|
function largestPatternFootprint(expression, schema) {
|
|
31030
30667
|
if (expression === void 0) return null;
|
|
31031
30668
|
const module = availablePatternSearch();
|
|
31032
30669
|
if (module === null) return null;
|
|
31033
30670
|
const patternFields = /* @__PURE__ */ new Set();
|
|
31034
30671
|
for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
|
|
31035
|
-
return patternFields.size === 0 ? null :
|
|
30672
|
+
return patternFields.size === 0 ? null : module.largestFootprint(expression, patternFields, PATTERN_HOST);
|
|
31036
30673
|
}
|
|
31037
30674
|
|
|
31038
30675
|
// src/engine/orchestration/search.ts
|
|
@@ -34270,7 +33907,7 @@ function refuse(path, reason) {
|
|
|
34270
33907
|
field: path
|
|
34271
33908
|
});
|
|
34272
33909
|
}
|
|
34273
|
-
function
|
|
33910
|
+
function isRecord6(value) {
|
|
34274
33911
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
34275
33912
|
}
|
|
34276
33913
|
function startsWithTagPrefix(key) {
|
|
@@ -34337,7 +33974,7 @@ function untagTag(tag, value, path, depth) {
|
|
|
34337
33974
|
case NUMBER_TAG:
|
|
34338
33975
|
return numberOf(value, path);
|
|
34339
33976
|
case OBJECT_TAG:
|
|
34340
|
-
if (!
|
|
33977
|
+
if (!isRecord6(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
34341
33978
|
return untagMap(value, path, depth);
|
|
34342
33979
|
default:
|
|
34343
33980
|
return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
@@ -34560,7 +34197,7 @@ function documentExportStream(header2, checkpoint) {
|
|
|
34560
34197
|
function invalidHeader(message) {
|
|
34561
34198
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
34562
34199
|
}
|
|
34563
|
-
function
|
|
34200
|
+
function isRecord7(value) {
|
|
34564
34201
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
34565
34202
|
}
|
|
34566
34203
|
function configOfHeader(line) {
|
|
@@ -34576,7 +34213,7 @@ function configOfHeader(line) {
|
|
|
34576
34213
|
} catch {
|
|
34577
34214
|
invalidHeader("is not JSON");
|
|
34578
34215
|
}
|
|
34579
|
-
if (!
|
|
34216
|
+
if (!isRecord7(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
34580
34217
|
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
34581
34218
|
}
|
|
34582
34219
|
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
@@ -34669,7 +34306,7 @@ async function takeLine(state, entry) {
|
|
|
34669
34306
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
34670
34307
|
return;
|
|
34671
34308
|
}
|
|
34672
|
-
if (!
|
|
34309
|
+
if (!isRecord7(document2) || typeof document2.id !== "string" || document2.id.length === 0) {
|
|
34673
34310
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
34674
34311
|
return;
|
|
34675
34312
|
}
|
|
@@ -36331,31 +35968,6 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
|
|
|
36331
35968
|
if (needsTrailingEllipsis) snippet = `${snippet}...`;
|
|
36332
35969
|
return { snippet, visiblePositions };
|
|
36333
35970
|
}
|
|
36334
|
-
function containsPartialWord(spelling, words) {
|
|
36335
|
-
for (const word of words) {
|
|
36336
|
-
if (spelling.length > word.length && spelling.includes(word)) return true;
|
|
36337
|
-
}
|
|
36338
|
-
return false;
|
|
36339
|
-
}
|
|
36340
|
-
function partialWordMatcher(text, language, partial) {
|
|
36341
|
-
const indexed = tokenize(text, language, {
|
|
36342
|
-
stem: true,
|
|
36343
|
-
removeStopWords: true,
|
|
36344
|
-
collectSurfaces: true,
|
|
36345
|
-
customTokenizer: partial.customTokenizer,
|
|
36346
|
-
stopWordOverride: partial.stopWordOverride
|
|
36347
|
-
});
|
|
36348
|
-
const positions = /* @__PURE__ */ new Set();
|
|
36349
|
-
const spellings = /* @__PURE__ */ new Set();
|
|
36350
|
-
for (let at = 0; at < indexed.tokens.length; at++) {
|
|
36351
|
-
const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
|
|
36352
|
-
if (!containsPartialWord(spelling, partial.words)) continue;
|
|
36353
|
-
positions.add(indexed.tokens[at].position);
|
|
36354
|
-
spellings.add(spelling);
|
|
36355
|
-
}
|
|
36356
|
-
if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
|
|
36357
|
-
return (position) => positions.has(position);
|
|
36358
|
-
}
|
|
36359
35971
|
function highlightField(text, queryTokens, language, options) {
|
|
36360
35972
|
const preTag = options?.preTag ?? "<mark>";
|
|
36361
35973
|
const postTag = options?.postTag ?? "</mark>";
|
|
@@ -36392,7 +36004,7 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
36392
36004
|
matchByFieldToken.set(fieldToken, matches);
|
|
36393
36005
|
return matches;
|
|
36394
36006
|
};
|
|
36395
|
-
const matchesInside = options?.partialWords === void 0 ? null :
|
|
36007
|
+
const matchesInside = options?.partialWords === void 0 ? null : partialWordHighlighter(text, language, options.partialWords);
|
|
36396
36008
|
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
36397
36009
|
if (i >= charOffsets.length) break;
|
|
36398
36010
|
const fieldToken = fieldResult.tokens[i].token;
|
|
@@ -36442,20 +36054,7 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
|
36442
36054
|
if (params.partialWords !== true || params.exact === true) return null;
|
|
36443
36055
|
const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
|
|
36444
36056
|
if (fields.length === 0) return null;
|
|
36445
|
-
const
|
|
36446
|
-
for (const word of written) {
|
|
36447
|
-
if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
|
|
36448
|
-
}
|
|
36449
|
-
const seenTokens = /* @__PURE__ */ new Set();
|
|
36450
|
-
const words = [];
|
|
36451
|
-
for (const queryToken of queryTokens) {
|
|
36452
|
-
if (seenTokens.has(queryToken.token)) continue;
|
|
36453
|
-
seenTokens.add(queryToken.token);
|
|
36454
|
-
const word = writtenAt.get(queryToken.position);
|
|
36455
|
-
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
36456
|
-
words.push(word);
|
|
36457
|
-
if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
36458
|
-
}
|
|
36057
|
+
const words = partialWordsToHighlight(queryTokens, written);
|
|
36459
36058
|
if (words.length === 0) return null;
|
|
36460
36059
|
return {
|
|
36461
36060
|
fields: new Set(fields),
|