@delali/narsil 0.3.1-next.78 → 0.3.1-next.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/{capabilities-1lEYL3WK.d.ts → capabilities-CcTDWh_P.d.ts} +2 -2
- package/dist/{chunk-AMGDG4ZJ.mjs → chunk-23HJ5QXQ.mjs} +1 -1
- package/dist/{chunk-R6WBLBTC.mjs → chunk-4QDQEJDV.mjs} +9 -45
- package/dist/chunk-DQABW5UQ.mjs +140 -0
- package/dist/chunk-GG2FBGVZ.mjs +30 -0
- package/dist/{chunk-XSZTF4ZQ.mjs → chunk-HJASKX4F.mjs} +1 -1
- package/dist/{chunk-DFW3U5UW.mjs → chunk-K4FZ55Q2.mjs} +21 -192
- package/dist/{chunk-PJWG547Z.mjs → chunk-MEOOYKWL.mjs} +3 -3
- package/dist/{chunk-55WEJKWT.mjs → chunk-NSXQCE4K.mjs} +17 -63
- package/dist/{chunk-PXDQFYD5.mjs → chunk-OSF3SGJV.mjs} +23 -139
- package/dist/{chunk-RTR44V2A.mjs → chunk-P2PYEZZE.mjs} +586 -4
- package/dist/{chunk-Z3BXCZXS.mjs → chunk-QPYV22XH.mjs} +2 -1
- package/dist/{chunk-4344I4MR.mjs → chunk-R2XCJIO7.mjs} +335 -12
- package/dist/{chunk-IBR6634K.mjs → chunk-R6A773WK.mjs} +2 -69
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-P2BEMMOD.mjs → chunk-YGLSR7BC.mjs} +150 -203
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/{client-ZXcbSViH.d.ts → client-CsybN673.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/{config-DTe5EbWW.d.ts → config-BCNQtszf.d.ts} +1 -1
- package/dist/{config-DHL2KsfD.d.ts → config-CFonR_gY.d.ts} +1 -1
- package/dist/direct-executor-VNMQ6NIH.mjs +19 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -1
- package/dist/distribution/coordinator/in-memory.mjs +2 -1
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +15 -13
- package/dist/{engine-BjPBzWkA.d.ts → engine-B0oIF1_r.d.ts} +1 -1
- package/dist/index.browser.mjs +113 -516
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +14 -12
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/{memory-DW3OJ0im.d.ts → memory-B6SBOKlS.d.ts} +1287 -1287
- package/dist/partial-words.mjs +5 -4
- package/dist/pattern.mjs +4 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -7
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -12
- package/dist/react.d.ts +6 -6
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +10 -7
- package/dist/{tasks-BTbwBqar.d.ts → tasks-CYAQe6qd.d.ts} +1 -1
- package/dist/{thread-XBDVXKUE.mjs → thread-P6QJ2Q5O.mjs} +11 -8
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +3 -3
- package/package.json +9 -7
- package/dist/chunk-IOZPBNSG.mjs +0 -55
- package/dist/chunk-WFVOBZS4.mjs +0 -485
- package/dist/direct-executor-WB2HKU4N.mjs +0 -17
package/dist/index.browser.mjs
CHANGED
|
@@ -371,20 +371,6 @@ function appendFoldedCodePoint(codePoint, out) {
|
|
|
371
371
|
}
|
|
372
372
|
for (const foldedCodePoint of folded) out.push(foldedCodePoint);
|
|
373
373
|
}
|
|
374
|
-
function appendFoldedCodePoints(text, out) {
|
|
375
|
-
const single = singleFoldTable();
|
|
376
|
-
const multi = multiFoldTable();
|
|
377
|
-
const cursor = { index: 0, pending: null, pendingIndex: 0 };
|
|
378
|
-
for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
|
|
379
|
-
out.push(codePoint);
|
|
380
|
-
codePoint = nextFoldedCodePoint(text, cursor, single, multi);
|
|
381
|
-
}
|
|
382
|
-
}
|
|
383
|
-
function foldedCodePointCount(text) {
|
|
384
|
-
const folded = [];
|
|
385
|
-
appendFoldedCodePoints(text, folded);
|
|
386
|
-
return folded.length;
|
|
387
|
-
}
|
|
388
374
|
|
|
389
375
|
// src/core/ordering/constants.ts
|
|
390
376
|
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
@@ -797,8 +783,21 @@ function extractVectorFieldsFromSchema(schema) {
|
|
|
797
783
|
|
|
798
784
|
// src/core/partition/partial-words/registry.ts
|
|
799
785
|
var registered = null;
|
|
786
|
+
var MODULE_FUNCTIONS = [
|
|
787
|
+
"createGrowingSlot",
|
|
788
|
+
"createFrozenSlot",
|
|
789
|
+
"buildTable",
|
|
790
|
+
"expand",
|
|
791
|
+
"countSpelling",
|
|
792
|
+
"encodeSpellings",
|
|
793
|
+
"spellingCountsOf",
|
|
794
|
+
"highlightMatcher",
|
|
795
|
+
"addTermMembers",
|
|
796
|
+
"highlightWords"
|
|
797
|
+
];
|
|
800
798
|
function isPartialWordSearchModule(value) {
|
|
801
|
-
|
|
799
|
+
if (value === null || typeof value !== "object" || value.name !== "partialWords") return false;
|
|
800
|
+
return MODULE_FUNCTIONS.every((name) => typeof Reflect.get(value, name) === "function");
|
|
802
801
|
}
|
|
803
802
|
function registerPartialWordSearch(module) {
|
|
804
803
|
if (!isPartialWordSearchModule(module)) {
|
|
@@ -812,12 +811,21 @@ function registerPartialWordSearch(module) {
|
|
|
812
811
|
function availablePartialWordSearch() {
|
|
813
812
|
return registered ?? platformPartialWordSearch();
|
|
814
813
|
}
|
|
814
|
+
var PARTIAL_WORD_IMPORT_ADVICE = `Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load an index with a partial field`;
|
|
815
|
+
function partialWordModule() {
|
|
816
|
+
const module = availablePartialWordSearch();
|
|
817
|
+
if (module !== null) return module;
|
|
818
|
+
throw new NarsilError(
|
|
819
|
+
ErrorCodes.CONFIG_INVALID,
|
|
820
|
+
`This bundle contains no partial-word search code. ${PARTIAL_WORD_IMPORT_ADVICE}`
|
|
821
|
+
);
|
|
822
|
+
}
|
|
815
823
|
function requirePartialWordSearchFor(schema) {
|
|
816
824
|
const fields = partialFieldsOf(schema);
|
|
817
825
|
if (fields.length === 0 || availablePartialWordSearch() !== null) return;
|
|
818
826
|
throw new NarsilError(
|
|
819
827
|
ErrorCodes.CONFIG_INVALID,
|
|
820
|
-
`The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code.
|
|
828
|
+
`The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. ${PARTIAL_WORD_IMPORT_ADVICE}`,
|
|
821
829
|
{ field: fields[0] }
|
|
822
830
|
);
|
|
823
831
|
}
|
|
@@ -851,6 +859,25 @@ function expandPartialWords(subs, analysis, requests) {
|
|
|
851
859
|
const module = availablePartialWordSearch();
|
|
852
860
|
return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
|
|
853
861
|
}
|
|
862
|
+
function countPartialSpelling(counts, token, spelling, change) {
|
|
863
|
+
partialWordModule().countSpelling(counts, token, spelling, change);
|
|
864
|
+
}
|
|
865
|
+
function encodePartialSpellingsOf(fields, counts) {
|
|
866
|
+
return partialWordModule().encodeSpellings(fields, counts);
|
|
867
|
+
}
|
|
868
|
+
function partialSpellingsOfDocuments(documents, analysis) {
|
|
869
|
+
const module = partialWordModule();
|
|
870
|
+
return module.encodeSpellings(analysis.fields, module.spellingCountsOf(documents, analysis));
|
|
871
|
+
}
|
|
872
|
+
function partialWordHighlighter(text, language, partial) {
|
|
873
|
+
return partialWordModule().highlightMatcher(text, language, partial);
|
|
874
|
+
}
|
|
875
|
+
function addPartialWordMembers(query, expand, groupOf) {
|
|
876
|
+
partialWordModule().addTermMembers(query, expand, groupOf);
|
|
877
|
+
}
|
|
878
|
+
function partialWordsToHighlight(queryTokens, written) {
|
|
879
|
+
return partialWordModule().highlightWords(queryTokens, written);
|
|
880
|
+
}
|
|
854
881
|
|
|
855
882
|
// src/core/pattern-index/platform.browser.ts
|
|
856
883
|
function platformPatternSearch() {
|
|
@@ -859,8 +886,22 @@ function platformPatternSearch() {
|
|
|
859
886
|
|
|
860
887
|
// src/core/pattern-index/registry.ts
|
|
861
888
|
var registered2 = null;
|
|
889
|
+
var MODULE_FUNCTIONS2 = [
|
|
890
|
+
"createIndex",
|
|
891
|
+
"readArrays",
|
|
892
|
+
"mergeArrays",
|
|
893
|
+
"matchInto",
|
|
894
|
+
"checkExpression",
|
|
895
|
+
"expressionFootprint",
|
|
896
|
+
"largestFootprint",
|
|
897
|
+
"checkOperands",
|
|
898
|
+
"evaluateTests",
|
|
899
|
+
"serializeIndex",
|
|
900
|
+
"decodeIndex"
|
|
901
|
+
];
|
|
862
902
|
function isPatternSearchModule(value) {
|
|
863
|
-
|
|
903
|
+
if (value === null || typeof value !== "object" || value.name !== "pattern") return false;
|
|
904
|
+
return MODULE_FUNCTIONS2.every((name) => typeof Reflect.get(value, name) === "function");
|
|
864
905
|
}
|
|
865
906
|
function registerPatternSearch(module) {
|
|
866
907
|
if (!isPatternSearchModule(module)) {
|
|
@@ -2584,17 +2625,11 @@ function generateId() {
|
|
|
2584
2625
|
|
|
2585
2626
|
// src/core/pattern-index/constants.ts
|
|
2586
2627
|
var DEFAULT_PATTERN_WORK_CAP = 25e6;
|
|
2587
|
-
var MAX_PATTERN_TEXT_CODE_POINTS = 1024;
|
|
2588
|
-
var MAX_PATTERN_POSITIONS = 2048;
|
|
2589
2628
|
var MATCHER_STATE_BUDGET_BYTES = 1e6;
|
|
2590
|
-
var MATCHER_CODE_POINT_BYTES = 8;
|
|
2591
|
-
var MATCHER_SIMULATION_BYTES_PER_POSITION = 16;
|
|
2592
|
-
var MATCHER_STATE_OVERHEAD_BYTES = 96;
|
|
2593
2629
|
var PATTERN_WORK_FLUSH_UNITS = 65536;
|
|
2594
|
-
var PATTERN_RUN_CODE_POINTS = 3;
|
|
2595
|
-
var MAX_PATTERN_TOLERANCE = 2;
|
|
2596
2630
|
var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "contains"];
|
|
2597
2631
|
var PATTERN_EXPRESSION_OPERATORS = ["wildcard", "regex"];
|
|
2632
|
+
var PATTERN_ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
|
|
2598
2633
|
|
|
2599
2634
|
// src/geo/constants.ts
|
|
2600
2635
|
var DEG_TO_RAD = Math.PI / 180;
|
|
@@ -4124,14 +4159,6 @@ function bitsetHas(bs, index) {
|
|
|
4124
4159
|
if (word >= bs.length) return false;
|
|
4125
4160
|
return (bs[word] & 1 << (index & 31)) !== 0;
|
|
4126
4161
|
}
|
|
4127
|
-
function bitsetAnd(a, b) {
|
|
4128
|
-
const len = Math.min(a.length, b.length);
|
|
4129
|
-
const result = new Uint32Array(len);
|
|
4130
|
-
for (let i = 0; i < len; i++) {
|
|
4131
|
-
result[i] = a[i] & b[i];
|
|
4132
|
-
}
|
|
4133
|
-
return result;
|
|
4134
|
-
}
|
|
4135
4162
|
function bitsetAndInto(target, other) {
|
|
4136
4163
|
const shared2 = Math.min(target.length, other.length);
|
|
4137
4164
|
for (let i = 0; i < shared2; i++) target[i] &= other[i];
|
|
@@ -4145,18 +4172,6 @@ function bitsetAndNotInto(target, removed) {
|
|
|
4145
4172
|
const shared2 = Math.min(target.length, removed.length);
|
|
4146
4173
|
for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
|
|
4147
4174
|
}
|
|
4148
|
-
function bitsetNot(bs, capacity) {
|
|
4149
|
-
const result = new Uint32Array(bs.length);
|
|
4150
|
-
const fullWords = capacity >>> 5;
|
|
4151
|
-
for (let i = 0; i < fullWords; i++) {
|
|
4152
|
-
result[i] = ~bs[i];
|
|
4153
|
-
}
|
|
4154
|
-
const remainder = capacity & 31;
|
|
4155
|
-
if (remainder > 0 && fullWords < bs.length) {
|
|
4156
|
-
result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
|
|
4157
|
-
}
|
|
4158
|
-
return result;
|
|
4159
|
-
}
|
|
4160
4175
|
function bitsetIsEmpty(bs) {
|
|
4161
4176
|
for (let i = 0; i < bs.length; i++) {
|
|
4162
4177
|
if (bs[i] !== 0) return false;
|
|
@@ -4378,24 +4393,7 @@ function computeFacets(state, matched, config, schema) {
|
|
|
4378
4393
|
}
|
|
4379
4394
|
|
|
4380
4395
|
// src/core/pattern-index/host.ts
|
|
4381
|
-
var
|
|
4382
|
-
var foldSources = null;
|
|
4383
|
-
function sourcesByFold() {
|
|
4384
|
-
if (foldSources !== null) return foldSources;
|
|
4385
|
-
const sources = /* @__PURE__ */ new Map();
|
|
4386
|
-
const add = (source, folded) => {
|
|
4387
|
-
const known = sources.get(folded);
|
|
4388
|
-
if (known === void 0) sources.set(folded, [source]);
|
|
4389
|
-
else known.push(source);
|
|
4390
|
-
};
|
|
4391
|
-
for (const [source, folded] of singleFoldTable()) add(source, folded);
|
|
4392
|
-
foldSources = sources;
|
|
4393
|
-
return sources;
|
|
4394
|
-
}
|
|
4395
|
-
function singleFoldSources(folded) {
|
|
4396
|
-
return sourcesByFold().get(folded) ?? NO_SOURCES;
|
|
4397
|
-
}
|
|
4398
|
-
var PATTERN_HOST = { appendFoldedCodePoint, singleFoldSources };
|
|
4396
|
+
var PATTERN_HOST = { appendFoldedCodePoint, singleFolds: singleFoldTable };
|
|
4399
4397
|
function patternIndexOf(indexes, fieldPath) {
|
|
4400
4398
|
let index = indexes.get(fieldPath);
|
|
4401
4399
|
if (index === void 0) {
|
|
@@ -5238,78 +5236,6 @@ function createNumericIndex() {
|
|
|
5238
5236
|
};
|
|
5239
5237
|
}
|
|
5240
5238
|
|
|
5241
|
-
// src/core/partition/partial-words/constants.ts
|
|
5242
|
-
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
5243
|
-
|
|
5244
|
-
// src/core/partition/partial-words/spellings.ts
|
|
5245
|
-
var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
|
|
5246
|
-
function codePointCountOf(text) {
|
|
5247
|
-
let count = 0;
|
|
5248
|
-
for (let index = 0; index < text.length; count++) {
|
|
5249
|
-
index += (text.codePointAt(index) ?? 0) > LAST_BASIC_PLANE_CODE_POINT2 ? 2 : 1;
|
|
5250
|
-
}
|
|
5251
|
-
return count;
|
|
5252
|
-
}
|
|
5253
|
-
function holdsRun(spelling) {
|
|
5254
|
-
if (spelling.length > 2 * PARTIAL_WORD_RUN_CODE_POINTS) return true;
|
|
5255
|
-
if (spelling.length <= PARTIAL_WORD_RUN_CODE_POINTS) return false;
|
|
5256
|
-
return codePointCountOf(spelling) > PARTIAL_WORD_RUN_CODE_POINTS;
|
|
5257
|
-
}
|
|
5258
|
-
function countSpelling(counts, token, spelling, change) {
|
|
5259
|
-
if (!holdsRun(spelling)) return;
|
|
5260
|
-
let byToken = counts.get(token);
|
|
5261
|
-
if (byToken === void 0) {
|
|
5262
|
-
byToken = /* @__PURE__ */ new Map();
|
|
5263
|
-
counts.set(token, byToken);
|
|
5264
|
-
}
|
|
5265
|
-
const next = (byToken.get(spelling) ?? 0) + change;
|
|
5266
|
-
if (next > 0) {
|
|
5267
|
-
byToken.set(spelling, next);
|
|
5268
|
-
return;
|
|
5269
|
-
}
|
|
5270
|
-
byToken.delete(spelling);
|
|
5271
|
-
if (byToken.size === 0) counts.delete(token);
|
|
5272
|
-
}
|
|
5273
|
-
function textsOf(value) {
|
|
5274
|
-
if (typeof value === "string") return [value];
|
|
5275
|
-
if (!Array.isArray(value)) return [];
|
|
5276
|
-
return value.filter((element) => typeof element === "string");
|
|
5277
|
-
}
|
|
5278
|
-
function countDocumentSpellings(counts, document2, analysis, direction) {
|
|
5279
|
-
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
5280
|
-
const countOccurrence2 = (token, _position, surface) => {
|
|
5281
|
-
countSpelling(counts, token, surface ?? token, direction);
|
|
5282
|
-
};
|
|
5283
|
-
for (const field of analysis.fields) {
|
|
5284
|
-
for (const text of textsOf(getNestedValue(document2, field))) {
|
|
5285
|
-
visitTokens(text, analysis.language, tokenizing, countOccurrence2);
|
|
5286
|
-
}
|
|
5287
|
-
}
|
|
5288
|
-
}
|
|
5289
|
-
function spellingCountsOf(documents, analysis) {
|
|
5290
|
-
const counts = /* @__PURE__ */ new Map();
|
|
5291
|
-
for (const document2 of documents) countDocumentSpellings(counts, document2, analysis, 1);
|
|
5292
|
-
return counts;
|
|
5293
|
-
}
|
|
5294
|
-
function encodePartialSpellings(fields, counts) {
|
|
5295
|
-
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
5296
|
-
for (const [token, byToken] of counts) {
|
|
5297
|
-
for (const [spelling, count] of byToken) {
|
|
5298
|
-
encoded.tokens.push(token);
|
|
5299
|
-
encoded.spellings.push(spelling);
|
|
5300
|
-
encoded.counts.push(count);
|
|
5301
|
-
}
|
|
5302
|
-
}
|
|
5303
|
-
return encoded;
|
|
5304
|
-
}
|
|
5305
|
-
function sameFields(left, right) {
|
|
5306
|
-
if (left.length !== right.length) return false;
|
|
5307
|
-
for (let at = 0; at < left.length; at++) {
|
|
5308
|
-
if (left[at] !== right[at]) return false;
|
|
5309
|
-
}
|
|
5310
|
-
return true;
|
|
5311
|
-
}
|
|
5312
|
-
|
|
5313
5239
|
// src/core/partition/value-sizes.ts
|
|
5314
5240
|
function keepsValueSize(base) {
|
|
5315
5241
|
return base === "string" || base === "verbatim" || base.endsWith("[]");
|
|
@@ -5401,7 +5327,7 @@ function countSurface(counts, surface, token) {
|
|
|
5401
5327
|
}
|
|
5402
5328
|
function countOccurrence(counts, spellings, surface, token) {
|
|
5403
5329
|
countSurface(counts, surface, token);
|
|
5404
|
-
if (spellings !== null)
|
|
5330
|
+
if (spellings !== null) countPartialSpelling(spellings, token, surface ?? token, 1);
|
|
5405
5331
|
}
|
|
5406
5332
|
function addFieldOccurrences(occurrencesByToken, fieldTokens) {
|
|
5407
5333
|
const fieldCounts = /* @__PURE__ */ new Map();
|
|
@@ -5730,73 +5656,6 @@ var FILTER_OPERAND_COPY_DEPTH = 3;
|
|
|
5730
5656
|
var MAX_FILTER_ARRAY_SIZE = 65536;
|
|
5731
5657
|
var GEO_UNIT_METRES = { km: 1e3, mi: 1609.344, m: 1 };
|
|
5732
5658
|
|
|
5733
|
-
// src/core/pattern-index/typo-runs.ts
|
|
5734
|
-
var BASIC_PLANE_LAST = 65535;
|
|
5735
|
-
function isPatternTolerance(value) {
|
|
5736
|
-
if (value === "auto") return true;
|
|
5737
|
-
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_PATTERN_TOLERANCE;
|
|
5738
|
-
}
|
|
5739
|
-
function foldTypoText(text, host) {
|
|
5740
|
-
const folded = [];
|
|
5741
|
-
let longestFold = 0;
|
|
5742
|
-
for (let at = 0; at < text.length; at++) {
|
|
5743
|
-
const codePoint = text.codePointAt(at) ?? 0;
|
|
5744
|
-
if (codePoint > BASIC_PLANE_LAST) at++;
|
|
5745
|
-
const before = folded.length;
|
|
5746
|
-
host.appendFoldedCodePoint(codePoint, folded);
|
|
5747
|
-
longestFold = Math.max(longestFold, folded.length - before);
|
|
5748
|
-
}
|
|
5749
|
-
return { folded, longestFold };
|
|
5750
|
-
}
|
|
5751
|
-
function distinctTypoRuns(text, limit) {
|
|
5752
|
-
const { folded } = text;
|
|
5753
|
-
const seen = /* @__PURE__ */ new Set();
|
|
5754
|
-
const runs = [];
|
|
5755
|
-
for (let at = 0; at + PATTERN_RUN_CODE_POINTS <= folded.length && seen.size < limit; at++) {
|
|
5756
|
-
const key = String.fromCodePoint(folded[at], folded[at + 1], folded[at + 2]);
|
|
5757
|
-
if (seen.has(key)) continue;
|
|
5758
|
-
seen.add(key);
|
|
5759
|
-
runs.push(folded[at], folded[at + 1], folded[at + 2]);
|
|
5760
|
-
}
|
|
5761
|
-
return runs;
|
|
5762
|
-
}
|
|
5763
|
-
function typoRunsNeeded(tolerance, longestFold) {
|
|
5764
|
-
return tolerance * (2 + longestFold) + 1;
|
|
5765
|
-
}
|
|
5766
|
-
function distinctTypoRunCount(text, limit) {
|
|
5767
|
-
return distinctTypoRuns(text, limit).length / PATTERN_RUN_CODE_POINTS;
|
|
5768
|
-
}
|
|
5769
|
-
|
|
5770
|
-
// src/filters/pattern-tests.ts
|
|
5771
|
-
function excluding(context, fieldPath, excluded) {
|
|
5772
|
-
const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
|
|
5773
|
-
return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
|
|
5774
|
-
}
|
|
5775
|
-
function textsOf2(operand) {
|
|
5776
|
-
const texts = /* @__PURE__ */ new Set();
|
|
5777
|
-
if (!Array.isArray(operand)) return texts;
|
|
5778
|
-
for (const text of operand) if (typeof text === "string") texts.add(text);
|
|
5779
|
-
return texts;
|
|
5780
|
-
}
|
|
5781
|
-
function evaluatePatternTests(fieldPath, filter, context, bitsets) {
|
|
5782
|
-
const caseFold = filter.caseFold === true;
|
|
5783
|
-
const matching = (operator, texts, tolerance = 0) => {
|
|
5784
|
-
const into = createBitSet(context.capacity);
|
|
5785
|
-
for (const text of texts) context.matchPattern(fieldPath, operator, text, caseFold, tolerance, into);
|
|
5786
|
-
return into;
|
|
5787
|
-
};
|
|
5788
|
-
const eqTolerance = isPatternTolerance(filter.tolerance) ? filter.tolerance : 0;
|
|
5789
|
-
if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq], eqTolerance));
|
|
5790
|
-
if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
|
|
5791
|
-
if (filter.in !== void 0) bitsets.push(matching("eq", textsOf2(filter.in)));
|
|
5792
|
-
if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf2(filter.nin))));
|
|
5793
|
-
if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
|
|
5794
|
-
if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
|
|
5795
|
-
if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
|
|
5796
|
-
if (typeof filter.wildcard === "string") bitsets.push(matching("wildcard", [filter.wildcard]));
|
|
5797
|
-
if (typeof filter.regex === "string") bitsets.push(matching("regex", [filter.regex]));
|
|
5798
|
-
}
|
|
5799
|
-
|
|
5800
5659
|
// src/filters/field-tests.ts
|
|
5801
5660
|
var COMPARISON_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
|
|
5802
5661
|
var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
|
|
@@ -5976,7 +5835,7 @@ function narrowByFieldTests(fieldPath, f, parsed, context, narrow) {
|
|
|
5976
5835
|
narrowByTextRange(fieldPath, f, context, narrow);
|
|
5977
5836
|
if (parsed.pattern) {
|
|
5978
5837
|
const patternBits = [];
|
|
5979
|
-
|
|
5838
|
+
requirePatternSearch(fieldPath).evaluateTests(fieldPath, f, context, patternBits);
|
|
5980
5839
|
for (const matched of patternBits) narrow(matched);
|
|
5981
5840
|
} else {
|
|
5982
5841
|
narrowByValue(fieldPath, f, parsed, context, bits, narrow);
|
|
@@ -6227,50 +6086,14 @@ function requirePatternField(fieldPath, operator, type) {
|
|
|
6227
6086
|
operator
|
|
6228
6087
|
);
|
|
6229
6088
|
}
|
|
6230
|
-
function requireTextLength(fieldPath, operator, text) {
|
|
6231
|
-
if (!exceedsCodePoints(text, MAX_PATTERN_TEXT_CODE_POINTS)) return;
|
|
6232
|
-
invalid(
|
|
6233
|
-
`The text of "${operator}" on pattern field "${fieldPath}" holds more than ${MAX_PATTERN_TEXT_CODE_POINTS} code points`,
|
|
6234
|
-
fieldPath,
|
|
6235
|
-
operator
|
|
6236
|
-
);
|
|
6237
|
-
}
|
|
6238
|
-
function requirePatternText(fieldPath, operator, text, caseFold) {
|
|
6239
|
-
requireTextLength(fieldPath, operator, text);
|
|
6240
|
-
if (caseFold && foldedCodePointCount(text) > MAX_PATTERN_POSITIONS) {
|
|
6241
|
-
invalid(
|
|
6242
|
-
`The text of "${operator}" on pattern field "${fieldPath}" has a case fold of more than ${MAX_PATTERN_POSITIONS} code points, while a pattern holds at most ${MAX_PATTERN_POSITIONS} positions`,
|
|
6243
|
-
fieldPath,
|
|
6244
|
-
operator
|
|
6245
|
-
);
|
|
6246
|
-
}
|
|
6247
|
-
}
|
|
6248
|
-
var ELEMENT_OPERATORS2 = ["matchesAny", "containsAll"];
|
|
6249
6089
|
var PATTERN_OPERAND_KEYS = [
|
|
6250
6090
|
"contains",
|
|
6251
6091
|
"caseFold",
|
|
6252
6092
|
"tolerance",
|
|
6253
6093
|
...PATTERN_EXPRESSION_OPERATORS,
|
|
6254
6094
|
...PATTERN_TEST_OPERATORS,
|
|
6255
|
-
...
|
|
6095
|
+
...PATTERN_ELEMENT_OPERATORS
|
|
6256
6096
|
];
|
|
6257
|
-
function requireExpressions(fieldPath, filter, type) {
|
|
6258
|
-
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
6259
|
-
const pattern = filter[operator];
|
|
6260
|
-
if (pattern === void 0) continue;
|
|
6261
|
-
if (typeof pattern !== "string")
|
|
6262
|
-
invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
|
|
6263
|
-
requirePatternField(fieldPath, operator, type);
|
|
6264
|
-
requireTextLength(fieldPath, operator, pattern);
|
|
6265
|
-
requirePatternSearch(fieldPath).checkExpression(
|
|
6266
|
-
operator,
|
|
6267
|
-
pattern,
|
|
6268
|
-
filter.caseFold === true,
|
|
6269
|
-
fieldPath,
|
|
6270
|
-
PATTERN_HOST
|
|
6271
|
-
);
|
|
6272
|
-
}
|
|
6273
|
-
}
|
|
6274
6097
|
function requirePatternOperands(fieldPath, filter, type) {
|
|
6275
6098
|
if (filter.contains !== void 0) {
|
|
6276
6099
|
if (typeof filter.contains !== "string") {
|
|
@@ -6284,50 +6107,16 @@ function requirePatternOperands(fieldPath, filter, type) {
|
|
|
6284
6107
|
}
|
|
6285
6108
|
requirePatternField(fieldPath, "caseFold", type);
|
|
6286
6109
|
}
|
|
6287
|
-
|
|
6110
|
+
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
6111
|
+
const pattern = filter[operator];
|
|
6112
|
+
if (pattern === void 0) continue;
|
|
6113
|
+
if (typeof pattern !== "string")
|
|
6114
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
|
|
6115
|
+
requirePatternField(fieldPath, operator, type);
|
|
6116
|
+
}
|
|
6288
6117
|
if (filter.tolerance !== void 0) requirePatternField(fieldPath, "tolerance", type);
|
|
6289
6118
|
if (parsedTypeOf(type)?.pattern !== true) return;
|
|
6290
|
-
|
|
6291
|
-
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
6292
|
-
const operand = filter[operator];
|
|
6293
|
-
if (typeof operand === "string") requirePatternText(fieldPath, operator, operand, caseFold);
|
|
6294
|
-
if (!Array.isArray(operand)) continue;
|
|
6295
|
-
for (const text of operand) {
|
|
6296
|
-
if (typeof text === "string") requirePatternText(fieldPath, operator, text, caseFold);
|
|
6297
|
-
}
|
|
6298
|
-
}
|
|
6299
|
-
for (const operator of ELEMENT_OPERATORS2) {
|
|
6300
|
-
const operand = filter[operator];
|
|
6301
|
-
if (!Array.isArray(operand)) continue;
|
|
6302
|
-
for (const text of operand) {
|
|
6303
|
-
if (typeof text === "string") requirePatternText(fieldPath, operator, text, false);
|
|
6304
|
-
}
|
|
6305
|
-
}
|
|
6306
|
-
requireTolerance(fieldPath, filter);
|
|
6307
|
-
}
|
|
6308
|
-
function requireTolerance(fieldPath, filter) {
|
|
6309
|
-
const { tolerance, eq } = filter;
|
|
6310
|
-
if (tolerance === void 0) return;
|
|
6311
|
-
if (!isPatternTolerance(tolerance)) {
|
|
6312
|
-
invalid(`"tolerance" on field "${fieldPath}" must be 0, 1, 2, or "auto"`, fieldPath, "tolerance");
|
|
6313
|
-
}
|
|
6314
|
-
if (typeof eq !== "string") {
|
|
6315
|
-
invalid(
|
|
6316
|
-
`"tolerance" applies only to an "eq" test, while the filter on field "${fieldPath}" sets no "eq" with text`,
|
|
6317
|
-
fieldPath,
|
|
6318
|
-
"tolerance"
|
|
6319
|
-
);
|
|
6320
|
-
}
|
|
6321
|
-
if (tolerance === 0 || tolerance === "auto") return;
|
|
6322
|
-
const text = foldTypoText(eq, PATTERN_HOST);
|
|
6323
|
-
const requiredRuns = typoRunsNeeded(tolerance, text.longestFold);
|
|
6324
|
-
const distinctRuns = distinctTypoRunCount(text, requiredRuns);
|
|
6325
|
-
if (distinctRuns >= requiredRuns) return;
|
|
6326
|
-
throw new NarsilError(
|
|
6327
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
6328
|
-
`A tolerance of ${tolerance} on "eq" of pattern field "${fieldPath}" needs ${requiredRuns} distinct runs of three code points in the case fold of the text, while the text holds ${distinctRuns}. Lengthen the text, lower the tolerance, or set it to "auto"`,
|
|
6329
|
-
{ fieldPath, operator: "tolerance", requiredRuns, distinctRuns }
|
|
6330
|
-
);
|
|
6119
|
+
requirePatternSearch(fieldPath).checkOperands(fieldPath, filter, PATTERN_HOST);
|
|
6331
6120
|
}
|
|
6332
6121
|
function requireWellFormedOperand(fieldPath, operator, text) {
|
|
6333
6122
|
if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) return;
|
|
@@ -8923,99 +8712,18 @@ function mergeSegmentPayload(target, payload, documents) {
|
|
|
8923
8712
|
}
|
|
8924
8713
|
|
|
8925
8714
|
// src/core/partition/pattern-snapshot.ts
|
|
8926
|
-
function loadFailed(fieldPath, problem) {
|
|
8927
|
-
return new NarsilError(
|
|
8928
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
8929
|
-
`The pattern index of field "${fieldPath}" in this partition is corrupt: ${problem}`,
|
|
8930
|
-
{ field: fieldPath }
|
|
8931
|
-
);
|
|
8932
|
-
}
|
|
8933
|
-
function serializePatternIndex(state, fieldPath, index) {
|
|
8934
|
-
const resolver = state.docStore.resolver();
|
|
8935
|
-
const arrays = index.toArrays(fieldPath, null);
|
|
8936
|
-
const positionOf = /* @__PURE__ */ new Map();
|
|
8937
|
-
const docIds = [];
|
|
8938
|
-
for (const ordinal of arrays.docIds) {
|
|
8939
|
-
const externalId = resolver.toExternal(ordinal);
|
|
8940
|
-
if (externalId === void 0) continue;
|
|
8941
|
-
positionOf.set(ordinal, docIds.length);
|
|
8942
|
-
docIds.push(externalId);
|
|
8943
|
-
}
|
|
8944
|
-
const runs = /* @__PURE__ */ Object.create(null);
|
|
8945
|
-
for (let run = 0; run + 1 < arrays.offsets.length; run++) {
|
|
8946
|
-
const positions = [];
|
|
8947
|
-
for (let at = arrays.offsets[run]; at < arrays.offsets[run + 1]; at++) {
|
|
8948
|
-
const position = positionOf.get(arrays.postings[at]);
|
|
8949
|
-
if (position !== void 0) positions.push(position);
|
|
8950
|
-
}
|
|
8951
|
-
if (positions.length === 0) continue;
|
|
8952
|
-
const base = run * PATTERN_RUN_CODE_POINTS;
|
|
8953
|
-
runs[String.fromCodePoint(arrays.runs[base], arrays.runs[base + 1], arrays.runs[base + 2])] = positions;
|
|
8954
|
-
}
|
|
8955
|
-
return { docIds, runs };
|
|
8956
|
-
}
|
|
8957
8715
|
function serializePatternIndexes(state, schema) {
|
|
8958
8716
|
const patternFields = schemaFieldsOf(schema).filter((field) => field.pattern);
|
|
8959
8717
|
if (patternFields.length === 0) return void 0;
|
|
8718
|
+
const resolver = state.docStore.resolver();
|
|
8719
|
+
const toExternal = (ordinal) => resolver.toExternal(ordinal);
|
|
8960
8720
|
const serialized = /* @__PURE__ */ Object.create(null);
|
|
8961
8721
|
for (const { path: fieldPath } of patternFields) {
|
|
8962
8722
|
const index = state.patternIndexes.get(fieldPath);
|
|
8963
|
-
serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } :
|
|
8723
|
+
serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : requirePatternSearch(fieldPath).serializeIndex(fieldPath, index, toExternal);
|
|
8964
8724
|
}
|
|
8965
8725
|
return serialized;
|
|
8966
8726
|
}
|
|
8967
|
-
function appendRunCodePoints(fieldPath, key, out) {
|
|
8968
|
-
const start = out.length;
|
|
8969
|
-
for (const character of key) out.push(character.codePointAt(0) ?? 0);
|
|
8970
|
-
const count = out.length - start;
|
|
8971
|
-
if (count !== PATTERN_RUN_CODE_POINTS) {
|
|
8972
|
-
throw loadFailed(
|
|
8973
|
-
fieldPath,
|
|
8974
|
-
`the key "${key}" holds ${count} code points, while each key must hold ${PATTERN_RUN_CODE_POINTS}`
|
|
8975
|
-
);
|
|
8976
|
-
}
|
|
8977
|
-
}
|
|
8978
|
-
function requirePositions(fieldPath, key, list, documentCount) {
|
|
8979
|
-
if (!Array.isArray(list)) throw loadFailed(fieldPath, `the run "${key}" must map to a list of positions`);
|
|
8980
|
-
let previous = -1;
|
|
8981
|
-
for (const position of list) {
|
|
8982
|
-
if (!Number.isInteger(position) || position < 0 || position >= documentCount) {
|
|
8983
|
-
throw loadFailed(fieldPath, `the run "${key}" names position ${String(position)} outside its document list`);
|
|
8984
|
-
}
|
|
8985
|
-
if (position <= previous) {
|
|
8986
|
-
throw loadFailed(fieldPath, `the positions of the run "${key}" are out of ascending order`);
|
|
8987
|
-
}
|
|
8988
|
-
previous = position;
|
|
8989
|
-
}
|
|
8990
|
-
return list;
|
|
8991
|
-
}
|
|
8992
|
-
function requireDistinctDocuments(fieldPath, docIds) {
|
|
8993
|
-
const seen = /* @__PURE__ */ new Set();
|
|
8994
|
-
for (const docId of docIds) {
|
|
8995
|
-
if (seen.has(docId)) throw loadFailed(fieldPath, `the document list names "${docId}" twice`);
|
|
8996
|
-
seen.add(docId);
|
|
8997
|
-
}
|
|
8998
|
-
}
|
|
8999
|
-
function decodePatternEntry(fieldPath, entry) {
|
|
9000
|
-
requireDistinctDocuments(fieldPath, entry.docIds);
|
|
9001
|
-
const runs = [];
|
|
9002
|
-
const offsets = [0];
|
|
9003
|
-
const postings = [];
|
|
9004
|
-
for (const [key, list] of Object.entries(entry.runs)) {
|
|
9005
|
-
appendRunCodePoints(fieldPath, key, runs);
|
|
9006
|
-
for (const position of requirePositions(fieldPath, key, list, entry.docIds.length)) postings.push(position);
|
|
9007
|
-
offsets.push(postings.length);
|
|
9008
|
-
}
|
|
9009
|
-
const positions = new Uint32Array(entry.docIds.length);
|
|
9010
|
-
for (let position = 0; position < positions.length; position++) positions[position] = position;
|
|
9011
|
-
return {
|
|
9012
|
-
fieldPath,
|
|
9013
|
-
docIds: positions,
|
|
9014
|
-
runs: Uint32Array.from(runs),
|
|
9015
|
-
offsets: Uint32Array.from(offsets),
|
|
9016
|
-
postings: Uint32Array.from(postings)
|
|
9017
|
-
};
|
|
9018
|
-
}
|
|
9019
8727
|
function rebuildPatternIndex(state, fieldPath) {
|
|
9020
8728
|
for (const [docId, stored] of state.docStore.all()) {
|
|
9021
8729
|
const value = getNestedValue(stored.fields, fieldPath);
|
|
@@ -9033,7 +8741,7 @@ function loadPatternIndexes(state, serialized, schema) {
|
|
|
9033
8741
|
rebuildPatternIndex(state, fieldPath);
|
|
9034
8742
|
continue;
|
|
9035
8743
|
}
|
|
9036
|
-
const decoded =
|
|
8744
|
+
const decoded = requirePatternSearch(fieldPath).decodeIndex(fieldPath, entry);
|
|
9037
8745
|
const ordinals = entry.docIds.map((docId) => resolver.toInternal(docId) ?? -1);
|
|
9038
8746
|
patternIndexOf(state.patternIndexes, fieldPath).addArrays(decoded, (position) => ordinals[position]);
|
|
9039
8747
|
}
|
|
@@ -10273,8 +9981,6 @@ var MAX_SORT_FIELDS = 8;
|
|
|
10273
9981
|
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
10274
9982
|
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
10275
9983
|
var PREFIX_MAX_EXPANSIONS = 50;
|
|
10276
|
-
var PARTIAL_WORDS_ADDED_PER_QUERY = 50;
|
|
10277
|
-
var PARTIAL_WORD_MIN_CODE_POINTS = 3;
|
|
10278
9984
|
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
10279
9985
|
var MAX_TOLERANCE = 10;
|
|
10280
9986
|
var MAX_PREFIX_LENGTH = 1024;
|
|
@@ -10624,18 +10330,6 @@ function addCompletions(groups, partition, lastToken, typedPrefix) {
|
|
|
10624
10330
|
group.members.push({ token: completion, factor, partialFactor: factor });
|
|
10625
10331
|
}
|
|
10626
10332
|
}
|
|
10627
|
-
function addPartialWords(group, added) {
|
|
10628
|
-
const membersByToken = /* @__PURE__ */ new Map();
|
|
10629
|
-
for (const member of group.members) membersByToken.set(member.token, member);
|
|
10630
|
-
for (const word of added) {
|
|
10631
|
-
const member = membersByToken.get(word.token);
|
|
10632
|
-
if (member === void 0) {
|
|
10633
|
-
group.members.push({ token: word.token, factor: 0, partialFactor: word.factor });
|
|
10634
|
-
} else if (word.factor > member.partialFactor) {
|
|
10635
|
-
member.partialFactor = word.factor;
|
|
10636
|
-
}
|
|
10637
|
-
}
|
|
10638
|
-
}
|
|
10639
10333
|
function partialWordAnalysisOf(fields, language, analysis) {
|
|
10640
10334
|
return {
|
|
10641
10335
|
fields,
|
|
@@ -10654,29 +10348,6 @@ function searchesAPartialField(params, partialFields) {
|
|
|
10654
10348
|
}
|
|
10655
10349
|
return false;
|
|
10656
10350
|
}
|
|
10657
|
-
function partialWordRequests(params, queryTokens, written, prefixToken) {
|
|
10658
|
-
const eligible = [];
|
|
10659
|
-
for (const queryToken of queryTokens) {
|
|
10660
|
-
const word = written.get(queryToken.position);
|
|
10661
|
-
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
10662
|
-
eligible.push({ token: queryToken.token, written: word });
|
|
10663
|
-
if (eligible.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
10664
|
-
}
|
|
10665
|
-
const count = eligible.length;
|
|
10666
|
-
if (count === 0) return [];
|
|
10667
|
-
const evenShare = Math.floor(PARTIAL_WORDS_ADDED_PER_QUERY / count);
|
|
10668
|
-
const remainder = PARTIAL_WORDS_ADDED_PER_QUERY % count;
|
|
10669
|
-
return eligible.map((entry, order) => ({
|
|
10670
|
-
token: entry.token,
|
|
10671
|
-
request: {
|
|
10672
|
-
written: entry.written,
|
|
10673
|
-
ownToken: entry.token,
|
|
10674
|
-
share: evenShare + (order < remainder ? 1 : 0),
|
|
10675
|
-
tolerance: entry.token === prefixToken ? 0 : params.tolerance ?? 0,
|
|
10676
|
-
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
10677
|
-
}
|
|
10678
|
-
}));
|
|
10679
|
-
}
|
|
10680
10351
|
function resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis) {
|
|
10681
10352
|
const term = params.term;
|
|
10682
10353
|
if (term === void 0 || params.exact === true) return {};
|
|
@@ -10689,16 +10360,18 @@ function resolveTermGroups(partition, params, language, schema, queryTokens, las
|
|
|
10689
10360
|
if (usesPrefix) addCompletions(groups, partition, lastToken, written.get(lastToken.position));
|
|
10690
10361
|
const prefixToken = groups.has(lastToken.token) ? lastToken.token : void 0;
|
|
10691
10362
|
if (usesPartialWords) {
|
|
10692
|
-
|
|
10693
|
-
|
|
10694
|
-
|
|
10695
|
-
|
|
10363
|
+
addPartialWordMembers(
|
|
10364
|
+
{
|
|
10365
|
+
queryTokens,
|
|
10366
|
+
written,
|
|
10367
|
+
prefixToken,
|
|
10368
|
+
tolerance: params.tolerance ?? 0,
|
|
10369
|
+
prefixLength: params.prefixLength,
|
|
10370
|
+
analysis: partialWordAnalysisOf(partialFields, language, analysis)
|
|
10371
|
+
},
|
|
10372
|
+
(partialAnalysis, requests) => partition.expandPartialWords(partialAnalysis, requests),
|
|
10373
|
+
(token, typoMatches) => groupFor(groups, token, typoMatches)
|
|
10696
10374
|
);
|
|
10697
|
-
for (let at = 0; at < requests.length; at++) {
|
|
10698
|
-
if (added[at].length === 0) continue;
|
|
10699
|
-
const token = requests[at].token;
|
|
10700
|
-
addPartialWords(groupFor(groups, token, token !== prefixToken), added[at]);
|
|
10701
|
-
}
|
|
10702
10375
|
}
|
|
10703
10376
|
const termGroups = [...groups.values()];
|
|
10704
10377
|
if (termGroups.length === 0) return {};
|
|
@@ -19248,6 +18921,15 @@ function createFrozenTombstones() {
|
|
|
19248
18921
|
};
|
|
19249
18922
|
}
|
|
19250
18923
|
|
|
18924
|
+
// src/core/partition/partial-words/spellings.ts
|
|
18925
|
+
function sameFields(left, right) {
|
|
18926
|
+
if (left.length !== right.length) return false;
|
|
18927
|
+
for (let at = 0; at < left.length; at++) {
|
|
18928
|
+
if (left[at] !== right[at]) return false;
|
|
18929
|
+
}
|
|
18930
|
+
return true;
|
|
18931
|
+
}
|
|
18932
|
+
|
|
19251
18933
|
// src/core/partition/frozen/shared-snapshot.ts
|
|
19252
18934
|
function sharedUint32(source) {
|
|
19253
18935
|
const copy = new Uint32Array(new SharedArrayBuffer(source.length * 4));
|
|
@@ -19291,7 +18973,7 @@ function sharePartialWordTable(table) {
|
|
|
19291
18973
|
function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
|
|
19292
18974
|
let source = spellings;
|
|
19293
18975
|
if (source === null && analysis !== null && analysis.fields.length > 0) {
|
|
19294
|
-
source =
|
|
18976
|
+
source = partialSpellingsOfDocuments(storedDocumentsOf(documentData), analysis);
|
|
19295
18977
|
}
|
|
19296
18978
|
if (source === null || source.fields.length === 0) return null;
|
|
19297
18979
|
const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
|
|
@@ -20158,7 +19840,7 @@ function mergePartialWords(inputs, mergedTokenCount, analysis) {
|
|
|
20158
19840
|
}
|
|
20159
19841
|
const counts = /* @__PURE__ */ new Map();
|
|
20160
19842
|
for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
|
|
20161
|
-
return { spellings:
|
|
19843
|
+
return { spellings: encodePartialSpellingsOf(analysis.fields, counts), reused: null };
|
|
20162
19844
|
}
|
|
20163
19845
|
function mergeFrozenSegments(segments, analysis) {
|
|
20164
19846
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
@@ -20504,7 +20186,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
20504
20186
|
const addToken = (token, position, surface) => {
|
|
20505
20187
|
postings.addToken(token, positionOffset + position);
|
|
20506
20188
|
surfaces.count(surface, token);
|
|
20507
|
-
if (fieldSpellings !== null)
|
|
20189
|
+
if (fieldSpellings !== null) countPartialSpelling(fieldSpellings, token, surface ?? token, 1);
|
|
20508
20190
|
};
|
|
20509
20191
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
20510
20192
|
const document2 = documents[ordinal].document;
|
|
@@ -20538,7 +20220,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
20538
20220
|
fieldLengthColumns: lengths.columns,
|
|
20539
20221
|
totalFieldLengths: lengths.totalsByName(),
|
|
20540
20222
|
surfaceForms: surfaces.encode(),
|
|
20541
|
-
partialSpellings: partialFields.length === 0 ? null :
|
|
20223
|
+
partialSpellings: partialFields.length === 0 ? null : encodePartialSpellingsOf(partialFields, spellings),
|
|
20542
20224
|
...encodeFieldIndexes(filters, identity)
|
|
20543
20225
|
};
|
|
20544
20226
|
}
|
|
@@ -22112,7 +21794,7 @@ async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, wha
|
|
|
22112
21794
|
checksum});
|
|
22113
21795
|
return { chunks: [header2, ...payloadChunks], byteLength: HEADER_SIZE + payloadLength, checksum };
|
|
22114
21796
|
}
|
|
22115
|
-
function
|
|
21797
|
+
function loadFailed(message, details) {
|
|
22116
21798
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, message, details);
|
|
22117
21799
|
}
|
|
22118
21800
|
async function gunzipWithinLimit(compressed, what) {
|
|
@@ -22125,7 +21807,7 @@ async function gunzipWithinLimit(compressed, what) {
|
|
|
22125
21807
|
try {
|
|
22126
21808
|
step = await reader.read();
|
|
22127
21809
|
} catch (error) {
|
|
22128
|
-
|
|
21810
|
+
loadFailed(`The ${what} holds a gzip payload that fails to inflate`, {
|
|
22129
21811
|
cause: error instanceof Error ? error.message : String(error)
|
|
22130
21812
|
});
|
|
22131
21813
|
}
|
|
@@ -22133,7 +21815,7 @@ async function gunzipWithinLimit(compressed, what) {
|
|
|
22133
21815
|
total += step.value.byteLength;
|
|
22134
21816
|
if (total > MAX_FILE_BYTES) {
|
|
22135
21817
|
await reader.cancel();
|
|
22136
|
-
|
|
21818
|
+
loadFailed(`The ${what} inflates past the file size limit of ${MAX_FILE_BYTES} bytes`);
|
|
22137
21819
|
}
|
|
22138
21820
|
chunks.push(step.value);
|
|
22139
21821
|
}
|
|
@@ -22162,7 +21844,7 @@ async function openChecksummedEnvelope(data, expected) {
|
|
|
22162
21844
|
);
|
|
22163
21845
|
}
|
|
22164
21846
|
if (header2.flags.compressionEnabled && !expected.allowsCompression) {
|
|
22165
|
-
|
|
21847
|
+
loadFailed(`The ${what} is compressed, while its format stores it raw`);
|
|
22166
21848
|
}
|
|
22167
21849
|
if (header2.flags.compressionEnabled && header2.flags.compressionAlgorithm !== "gzip") {
|
|
22168
21850
|
throw new NarsilError(
|
|
@@ -22170,9 +21852,9 @@ async function openChecksummedEnvelope(data, expected) {
|
|
|
22170
21852
|
`The ${what} uses ${header2.flags.compressionAlgorithm} compression, which this build cannot read`
|
|
22171
21853
|
);
|
|
22172
21854
|
}
|
|
22173
|
-
if (!header2.flags.checksumPresent)
|
|
21855
|
+
if (!header2.flags.checksumPresent) loadFailed(`The ${what} carries no checksum, while its format requires one`);
|
|
22174
21856
|
if (data.byteLength !== HEADER_SIZE + header2.payloadLength) {
|
|
22175
|
-
|
|
21857
|
+
loadFailed(
|
|
22176
21858
|
`The ${what} holds ${data.byteLength} bytes where its header gives ${HEADER_SIZE + header2.payloadLength}`,
|
|
22177
21859
|
{ bytes: data.byteLength, expected: HEADER_SIZE + header2.payloadLength }
|
|
22178
21860
|
);
|
|
@@ -30974,65 +30656,18 @@ function stopIdleSweep(state) {
|
|
|
30974
30656
|
}
|
|
30975
30657
|
|
|
30976
30658
|
// src/core/pattern-index/matcher-budget.ts
|
|
30977
|
-
var POSITIONS_PER_WORD = 32;
|
|
30978
|
-
function positionWords(positions) {
|
|
30979
|
-
return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
|
|
30980
|
-
}
|
|
30981
|
-
function smallestMatcherBytes(positions) {
|
|
30982
|
-
return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
|
|
30983
|
-
}
|
|
30984
30659
|
function widestPatternSplit(footprintBytes) {
|
|
30985
30660
|
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / Math.max(1, footprintBytes)));
|
|
30986
30661
|
}
|
|
30987
30662
|
|
|
30988
30663
|
// src/core/pattern-index/query-positions.ts
|
|
30989
|
-
function isRecord6(value) {
|
|
30990
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
30991
|
-
}
|
|
30992
|
-
function literalFootprint(text, caseFold) {
|
|
30993
|
-
if (typeof text !== "string") return null;
|
|
30994
|
-
return smallestMatcherBytes(caseFold ? foldedCodePointCount(text) : codePointCount(text));
|
|
30995
|
-
}
|
|
30996
|
-
function larger(left, right) {
|
|
30997
|
-
if (left === null) return right;
|
|
30998
|
-
if (right === null) return left;
|
|
30999
|
-
return Math.max(left, right);
|
|
31000
|
-
}
|
|
31001
|
-
function largestTestOf(filter, module) {
|
|
31002
|
-
const caseFold = filter.caseFold === true;
|
|
31003
|
-
let largest = null;
|
|
31004
|
-
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
31005
|
-
const operand = filter[operator];
|
|
31006
|
-
if (operand === void 0) continue;
|
|
31007
|
-
const texts = Array.isArray(operand) ? operand : [operand];
|
|
31008
|
-
for (const text of texts) largest = larger(largest, literalFootprint(text, caseFold));
|
|
31009
|
-
}
|
|
31010
|
-
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
31011
|
-
const pattern = filter[operator];
|
|
31012
|
-
if (typeof pattern !== "string") continue;
|
|
31013
|
-
largest = larger(largest, module.expressionFootprint(operator, pattern, caseFold, PATTERN_HOST));
|
|
31014
|
-
}
|
|
31015
|
-
return largest;
|
|
31016
|
-
}
|
|
31017
|
-
function largestIn(expression, patternFields, module) {
|
|
31018
|
-
let largest = null;
|
|
31019
|
-
if (isRecord6(expression.fields)) {
|
|
31020
|
-
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
31021
|
-
if (patternFields.has(fieldPath) && isRecord6(filter)) largest = larger(largest, largestTestOf(filter, module));
|
|
31022
|
-
}
|
|
31023
|
-
}
|
|
31024
|
-
const nested = [...expression.and ?? [], ...expression.or ?? []];
|
|
31025
|
-
if (expression.not !== void 0) nested.push(expression.not);
|
|
31026
|
-
for (const child of nested) largest = larger(largest, largestIn(child, patternFields, module));
|
|
31027
|
-
return largest;
|
|
31028
|
-
}
|
|
31029
30664
|
function largestPatternFootprint(expression, schema) {
|
|
31030
30665
|
if (expression === void 0) return null;
|
|
31031
30666
|
const module = availablePatternSearch();
|
|
31032
30667
|
if (module === null) return null;
|
|
31033
30668
|
const patternFields = /* @__PURE__ */ new Set();
|
|
31034
30669
|
for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
|
|
31035
|
-
return patternFields.size === 0 ? null :
|
|
30670
|
+
return patternFields.size === 0 ? null : module.largestFootprint(expression, patternFields, PATTERN_HOST);
|
|
31036
30671
|
}
|
|
31037
30672
|
|
|
31038
30673
|
// src/engine/orchestration/search.ts
|
|
@@ -34270,7 +33905,7 @@ function refuse(path, reason) {
|
|
|
34270
33905
|
field: path
|
|
34271
33906
|
});
|
|
34272
33907
|
}
|
|
34273
|
-
function
|
|
33908
|
+
function isRecord6(value) {
|
|
34274
33909
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
34275
33910
|
}
|
|
34276
33911
|
function startsWithTagPrefix(key) {
|
|
@@ -34337,7 +33972,7 @@ function untagTag(tag, value, path, depth) {
|
|
|
34337
33972
|
case NUMBER_TAG:
|
|
34338
33973
|
return numberOf(value, path);
|
|
34339
33974
|
case OBJECT_TAG:
|
|
34340
|
-
if (!
|
|
33975
|
+
if (!isRecord6(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
34341
33976
|
return untagMap(value, path, depth);
|
|
34342
33977
|
default:
|
|
34343
33978
|
return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
@@ -34560,7 +34195,7 @@ function documentExportStream(header2, checkpoint) {
|
|
|
34560
34195
|
function invalidHeader(message) {
|
|
34561
34196
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
34562
34197
|
}
|
|
34563
|
-
function
|
|
34198
|
+
function isRecord7(value) {
|
|
34564
34199
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
34565
34200
|
}
|
|
34566
34201
|
function configOfHeader(line) {
|
|
@@ -34576,7 +34211,7 @@ function configOfHeader(line) {
|
|
|
34576
34211
|
} catch {
|
|
34577
34212
|
invalidHeader("is not JSON");
|
|
34578
34213
|
}
|
|
34579
|
-
if (!
|
|
34214
|
+
if (!isRecord7(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
34580
34215
|
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
34581
34216
|
}
|
|
34582
34217
|
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
@@ -34669,7 +34304,7 @@ async function takeLine(state, entry) {
|
|
|
34669
34304
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
34670
34305
|
return;
|
|
34671
34306
|
}
|
|
34672
|
-
if (!
|
|
34307
|
+
if (!isRecord7(document2) || typeof document2.id !== "string" || document2.id.length === 0) {
|
|
34673
34308
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
34674
34309
|
return;
|
|
34675
34310
|
}
|
|
@@ -36331,31 +35966,6 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
|
|
|
36331
35966
|
if (needsTrailingEllipsis) snippet = `${snippet}...`;
|
|
36332
35967
|
return { snippet, visiblePositions };
|
|
36333
35968
|
}
|
|
36334
|
-
function containsPartialWord(spelling, words) {
|
|
36335
|
-
for (const word of words) {
|
|
36336
|
-
if (spelling.length > word.length && spelling.includes(word)) return true;
|
|
36337
|
-
}
|
|
36338
|
-
return false;
|
|
36339
|
-
}
|
|
36340
|
-
function partialWordMatcher(text, language, partial) {
|
|
36341
|
-
const indexed = tokenize(text, language, {
|
|
36342
|
-
stem: true,
|
|
36343
|
-
removeStopWords: true,
|
|
36344
|
-
collectSurfaces: true,
|
|
36345
|
-
customTokenizer: partial.customTokenizer,
|
|
36346
|
-
stopWordOverride: partial.stopWordOverride
|
|
36347
|
-
});
|
|
36348
|
-
const positions = /* @__PURE__ */ new Set();
|
|
36349
|
-
const spellings = /* @__PURE__ */ new Set();
|
|
36350
|
-
for (let at = 0; at < indexed.tokens.length; at++) {
|
|
36351
|
-
const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
|
|
36352
|
-
if (!containsPartialWord(spelling, partial.words)) continue;
|
|
36353
|
-
positions.add(indexed.tokens[at].position);
|
|
36354
|
-
spellings.add(spelling);
|
|
36355
|
-
}
|
|
36356
|
-
if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
|
|
36357
|
-
return (position) => positions.has(position);
|
|
36358
|
-
}
|
|
36359
35969
|
function highlightField(text, queryTokens, language, options) {
|
|
36360
35970
|
const preTag = options?.preTag ?? "<mark>";
|
|
36361
35971
|
const postTag = options?.postTag ?? "</mark>";
|
|
@@ -36392,7 +36002,7 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
36392
36002
|
matchByFieldToken.set(fieldToken, matches);
|
|
36393
36003
|
return matches;
|
|
36394
36004
|
};
|
|
36395
|
-
const matchesInside = options?.partialWords === void 0 ? null :
|
|
36005
|
+
const matchesInside = options?.partialWords === void 0 ? null : partialWordHighlighter(text, language, options.partialWords);
|
|
36396
36006
|
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
36397
36007
|
if (i >= charOffsets.length) break;
|
|
36398
36008
|
const fieldToken = fieldResult.tokens[i].token;
|
|
@@ -36442,20 +36052,7 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
|
36442
36052
|
if (params.partialWords !== true || params.exact === true) return null;
|
|
36443
36053
|
const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
|
|
36444
36054
|
if (fields.length === 0) return null;
|
|
36445
|
-
const
|
|
36446
|
-
for (const word of written) {
|
|
36447
|
-
if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
|
|
36448
|
-
}
|
|
36449
|
-
const seenTokens = /* @__PURE__ */ new Set();
|
|
36450
|
-
const words = [];
|
|
36451
|
-
for (const queryToken of queryTokens) {
|
|
36452
|
-
if (seenTokens.has(queryToken.token)) continue;
|
|
36453
|
-
seenTokens.add(queryToken.token);
|
|
36454
|
-
const word = writtenAt.get(queryToken.position);
|
|
36455
|
-
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
36456
|
-
words.push(word);
|
|
36457
|
-
if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
36458
|
-
}
|
|
36055
|
+
const words = partialWordsToHighlight(queryTokens, written);
|
|
36459
36056
|
if (words.length === 0) return null;
|
|
36460
36057
|
return {
|
|
36461
36058
|
fields: new Set(fields),
|