@delali/narsil 0.3.1-next.73 → 0.3.1-next.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{capabilities-CYeAWYxQ.d.ts → capabilities-BHNksicw.d.ts} +16 -2
- package/dist/{chunk-PU43WDY4.mjs → chunk-2E35UM3G.mjs} +1 -13
- package/dist/{chunk-AK3BLGCI.mjs → chunk-356UOTK2.mjs} +15 -48
- package/dist/chunk-7ODNRREZ.mjs +2050 -0
- package/dist/chunk-AIUOAN7Y.mjs +658 -0
- package/dist/{chunk-WVDFG45M.mjs → chunk-AMGDG4ZJ.mjs} +3 -2
- package/dist/{chunk-AFKH4ZK3.mjs → chunk-BT62MUM5.mjs} +1 -1
- package/dist/{chunk-LPBWCHBE.mjs → chunk-BT6BI4GV.mjs} +42 -4
- package/dist/{chunk-LJYTW7GT.mjs → chunk-IBR6634K.mjs} +1 -1
- package/dist/chunk-KEHK2O6P.mjs +13 -0
- package/dist/{chunk-CVFAFZSN.mjs → chunk-LOARLB4F.mjs} +46 -34
- package/dist/{chunk-3YKWSO54.mjs → chunk-NAMGDK45.mjs} +228 -649
- package/dist/{chunk-2WLD2NFW.mjs → chunk-PXDY5VRT.mjs} +3 -3
- package/dist/chunk-UOJIV2NG.mjs +593 -0
- package/dist/chunk-V3B24XR7.mjs +54 -0
- package/dist/{chunk-ZULSEZCE.mjs → chunk-WL2GDSCV.mjs} +100 -19
- package/dist/{chunk-THMSFOJO.mjs → chunk-YOVS4EV5.mjs} +582 -598
- package/dist/{client-D698E5ge.d.ts → client-D4sXFS9f.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/{config-Bv8etwUZ.d.ts → config-B-GRonqw.d.ts} +2 -2
- package/dist/{config-QXU3HMpq.d.ts → config-C1kbq1X2.d.ts} +2 -2
- package/dist/direct-executor-AVNWB73C.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +23 -13
- package/dist/{engine-D_OzKZ-6.d.ts → engine-DcGSd5vh.d.ts} +2 -2
- package/dist/index.browser.mjs +1272 -834
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +14 -11
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-B4jMmygw.d.ts → memory-M9kPmYP8.d.ts} +1257 -1178
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +11 -0
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +33 -13
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -11
- package/dist/react.d.ts +7 -7
- package/dist/{schema-BBOCdhND.d.ts → schema-DgyDijhf.d.ts} +6 -2
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +36 -11
- package/dist/{tasks-DpgCpW1x.d.ts → tasks-A8se0J98.d.ts} +1 -1
- package/dist/{thread-3LNVA43V.mjs → thread-BSIKT6ED.mjs} +16 -9
- package/dist/{types-BiUmaiWT.d.ts → types-ChrVy4lH.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -3
- package/dist/workers/entry.mjs +3 -3
- package/package.json +16 -7
- package/dist/chunk-XW25X6AU.mjs +0 -614
- package/dist/chunk-ZKRV5M2P.mjs +0 -75
- package/dist/direct-executor-5SRDFZAS.mjs +0 -13
package/dist/index.browser.mjs
CHANGED
|
@@ -209,11 +209,303 @@ function resolveIndexAnalysis(config) {
|
|
|
209
209
|
};
|
|
210
210
|
}
|
|
211
211
|
|
|
212
|
-
// src/core/
|
|
213
|
-
function
|
|
212
|
+
// src/core/partition/partial-words/platform.browser.ts
|
|
213
|
+
function platformPartialWordSearch() {
|
|
214
214
|
return null;
|
|
215
215
|
}
|
|
216
216
|
|
|
217
|
+
// src/core/ordering/code-points.ts
|
|
218
|
+
function exceedsCodePoints(value, maximum) {
|
|
219
|
+
if (value.length <= maximum) return false;
|
|
220
|
+
let counted = 0;
|
|
221
|
+
let i = 0;
|
|
222
|
+
while (i < value.length) {
|
|
223
|
+
const unit = value.charCodeAt(i);
|
|
224
|
+
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
225
|
+
const low = value.charCodeAt(i + 1);
|
|
226
|
+
if (low >= 56320 && low < 57344) i++;
|
|
227
|
+
}
|
|
228
|
+
i++;
|
|
229
|
+
counted++;
|
|
230
|
+
if (counted > maximum) return true;
|
|
231
|
+
}
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
var SURROGATE_UNIT = /[\ud800-\udfff]/;
|
|
235
|
+
function holdsUnpairedSurrogate(value) {
|
|
236
|
+
if (!SURROGATE_UNIT.test(value)) return false;
|
|
237
|
+
for (let i = 0; i < value.length; i++) {
|
|
238
|
+
const unit = value.charCodeAt(i);
|
|
239
|
+
if (unit < 55296 || unit >= 57344) continue;
|
|
240
|
+
if (unit >= 56320) return true;
|
|
241
|
+
const low = value.charCodeAt(i + 1);
|
|
242
|
+
if (!(low >= 56320 && low < 57344)) return true;
|
|
243
|
+
i++;
|
|
244
|
+
}
|
|
245
|
+
return false;
|
|
246
|
+
}
|
|
247
|
+
function codePointCount(value) {
|
|
248
|
+
let counted = 0;
|
|
249
|
+
for (let i = 0; i < value.length; i++) {
|
|
250
|
+
const unit = value.charCodeAt(i);
|
|
251
|
+
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
252
|
+
const low = value.charCodeAt(i + 1);
|
|
253
|
+
if (low >= 56320 && low < 57344) i++;
|
|
254
|
+
}
|
|
255
|
+
counted++;
|
|
256
|
+
}
|
|
257
|
+
return counted;
|
|
258
|
+
}
|
|
259
|
+
function adjustUtf16Unit(unit) {
|
|
260
|
+
if (unit < 55296) return unit;
|
|
261
|
+
if (unit < 57344) return unit + 8192;
|
|
262
|
+
return unit - 2048;
|
|
263
|
+
}
|
|
264
|
+
function compareCodePoints(a, b) {
|
|
265
|
+
const shorter = a.length < b.length ? a.length : b.length;
|
|
266
|
+
let i = 0;
|
|
267
|
+
while (i < shorter && a.charCodeAt(i) === b.charCodeAt(i)) i++;
|
|
268
|
+
if (i === shorter) return a.length - b.length;
|
|
269
|
+
return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// src/core/ordering/fold-table.ts
|
|
273
|
+
var SINGLE_FOLDS = "1t,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2j,lj;b,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;y,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3d;1,1;2,1;2,1;2,-7g;2,5u;1,1;2,1;2,5q;1,1;2,5p;1,5p;1,1;3,27;1,5m;1,5n;1,1;2,5p;1,5r;2,5v;1,5t;1,1;4,5v;1,5x;2,5y;1,1;2,1;2,1;2,62;1,1;2,62;3,1;2,62;1,1;2,61;1,61;1,1;2,1;2,63;1,1;4,1;8,2;1,1;2,2;1,1;2,2;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,2;1,1;2,1;2,-2p;1,-1k;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;8,8bv;1,1;2,-4j;1,8bs;3,1;2,-5f;1,1x;1,1z;1,1;2,1;2,1;2,1;2,1;6v,38;17,1;2,1;4,1;9,38;7,12;2,11;1,11;1,11;2,1s;2,1r;1,1r;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;n,1;d,8;1,-u;1,-p;4,-f;1,-m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1i;1,-1c;3,-1o;1,-1s;2,1;2,-7;1,1;3,-3m;1,-3m;1,-3m;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1d,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,f;1,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;28a,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;2,5ls;6,5ls;mj,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1oj,-4su;1,-4st;1,-4sk;1,-4si;1,-4si;1,-4sj;1,-4sc;1,-4ro;1,r7n;1,1;7,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;3,-2bk;1,-2bk;1,-2bk;8x,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;7,-1m;5,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;b,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;c,-8;2,-8;2,-8;2,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;21,-8;1,-8;1,-22;1,-22;3,-5j9;a,-2e;1,-2e;1,-2e;1,-2e;d,-8;1,-8;1,-2s;1,-2s;d,-8;1,-8;1,-34;1,-34;1,-7;c,-3k;1,-3k;1,-3i;1,-3i;8b,-5st;4,-6gv;1,-6di;7,s;1a,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;k,1;mr,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1f5,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1d,1;2,-8af;1,-2xy;1,-89z;3,1;2,1;2,1;2,-8bg;1,-8al;1,-8bj;1,-8bi;2,1;3,1;9,-8cf;1,-8cf;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;9,1;2,1;5,1;nym,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;k,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3s,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;b,1;2,1;2,-r9g;1,1;2,1;2,1;2,1;2,1;5,1;2,-wmg;3,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wn8;1,-wnj;1,-wnf;1,-wn5;1,-wn8;2,-wlu;1,-wmi;1,-wlx;1,ps;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1c;1,-wn7;1,-raw;1,1;2,1;2,-wo7;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wu9;p,1;or,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;ggy,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;xy,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;3t,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;4d,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1d7,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;4e,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;27v,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;gw1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1t,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;o5k,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;";
|
|
274
|
+
var MULTI_FOLDS = "67,37,37;29,2x,lj;p,jg,32;4n,2y,lo;bk,qh,lk,ld;w,qt,lk,ld;d3,12d,136;4y7,2w,mp;1,38,lk;1,3b,lm;1,3d,lm;1,2p,ji;4,37,37;4y,qt,lv;2,qt,lv,lc;2,qt,lv,ld;2,qt,lv,n6;16,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;3,67k,qh;1,q9,qh;1,q4,qh;2,q9,n6;1,q9,n6,qh;5,q9,qh;6,67o,qh;1,qf,qh;1,q6,qh;2,qf,n6;1,qf,n6,qh;5,qf,qh;6,qh,lk,lc;1,qh,lk,ld;3,qh,n6;1,qh,lk,n6;b,qt,lk,lc;1,qt,lk,ld;1,qp,lv;2,qt,n6;1,qt,lk,n6;b,67w,qh;1,qx,qh;1,r2,qh;2,qx,n6;1,qx,n6,qh;5,qx,qh;179g,2u,2u;1,2u,2x;1,2u,30;1,2u,2u,2x;1,2u,2u,30;1,37,38;1,37,38;d,12s,12u;1,12s,12d;1,12s,12j;1,132,12u;1,12s,12l;";
|
|
275
|
+
var singleTable = null;
|
|
276
|
+
var multiTable = null;
|
|
277
|
+
function singleFoldTable() {
|
|
278
|
+
if (singleTable === null) {
|
|
279
|
+
singleTable = /* @__PURE__ */ new Map();
|
|
280
|
+
let source = 0;
|
|
281
|
+
for (const entry of SINGLE_FOLDS.split(";")) {
|
|
282
|
+
if (entry.length === 0) continue;
|
|
283
|
+
const [delta, offset] = entry.split(",");
|
|
284
|
+
source += Number.parseInt(delta, 36);
|
|
285
|
+
singleTable.set(source, source + Number.parseInt(offset, 36));
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
return singleTable;
|
|
289
|
+
}
|
|
290
|
+
function multiFoldTable() {
|
|
291
|
+
if (multiTable === null) {
|
|
292
|
+
multiTable = /* @__PURE__ */ new Map();
|
|
293
|
+
let source = 0;
|
|
294
|
+
for (const entry of MULTI_FOLDS.split(";")) {
|
|
295
|
+
if (entry.length === 0) continue;
|
|
296
|
+
const parts = entry.split(",");
|
|
297
|
+
source += Number.parseInt(parts[0], 36);
|
|
298
|
+
multiTable.set(
|
|
299
|
+
source,
|
|
300
|
+
parts.slice(1).map((cp) => Number.parseInt(cp, 36))
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return multiTable;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// src/core/ordering/fold-compare.ts
|
|
308
|
+
var EXHAUSTED = -1;
|
|
309
|
+
function foldOf(codePoint, single, multi) {
|
|
310
|
+
if (codePoint < 128) return codePoint >= 65 && codePoint <= 90 ? codePoint + 32 : codePoint;
|
|
311
|
+
return single.get(codePoint) ?? multi.get(codePoint) ?? codePoint;
|
|
312
|
+
}
|
|
313
|
+
function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
314
|
+
if (cursor.pending !== null) {
|
|
315
|
+
const cp2 = cursor.pending[cursor.pendingIndex];
|
|
316
|
+
cursor.pendingIndex++;
|
|
317
|
+
if (cursor.pendingIndex === cursor.pending.length) {
|
|
318
|
+
cursor.pending = null;
|
|
319
|
+
cursor.pendingIndex = 0;
|
|
320
|
+
}
|
|
321
|
+
return cp2;
|
|
322
|
+
}
|
|
323
|
+
if (cursor.index >= text.length) return EXHAUSTED;
|
|
324
|
+
let cp = text.charCodeAt(cursor.index);
|
|
325
|
+
cursor.index++;
|
|
326
|
+
if (cp >= 55296 && cp < 56320 && cursor.index < text.length) {
|
|
327
|
+
const low = text.charCodeAt(cursor.index);
|
|
328
|
+
if (low >= 56320 && low < 57344) {
|
|
329
|
+
cp = 65536 + (cp - 55296 << 10) + (low - 56320);
|
|
330
|
+
cursor.index++;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const folded = foldOf(cp, single, multi);
|
|
334
|
+
if (typeof folded === "number") return folded;
|
|
335
|
+
if (folded.length > 1) {
|
|
336
|
+
cursor.pending = folded;
|
|
337
|
+
cursor.pendingIndex = 1;
|
|
338
|
+
}
|
|
339
|
+
return folded[0];
|
|
340
|
+
}
|
|
341
|
+
function compareCaseFolded(a, b) {
|
|
342
|
+
const shorter = a.length < b.length ? a.length : b.length;
|
|
343
|
+
let start = 0;
|
|
344
|
+
while (start < shorter && a.charCodeAt(start) === b.charCodeAt(start)) start++;
|
|
345
|
+
if (start === a.length && start === b.length) return 0;
|
|
346
|
+
if (start > 0) {
|
|
347
|
+
const previous = a.charCodeAt(start - 1);
|
|
348
|
+
if (previous >= 55296 && previous < 56320) start--;
|
|
349
|
+
}
|
|
350
|
+
const single = singleFoldTable();
|
|
351
|
+
const multi = multiFoldTable();
|
|
352
|
+
const aCursor = { index: start, pending: null, pendingIndex: 0 };
|
|
353
|
+
const bCursor = { index: start, pending: null, pendingIndex: 0 };
|
|
354
|
+
for (; ; ) {
|
|
355
|
+
const aCp = nextFoldedCodePoint(a, aCursor, single, multi);
|
|
356
|
+
const bCp = nextFoldedCodePoint(b, bCursor, single, multi);
|
|
357
|
+
if (aCp !== bCp) return aCp - bCp;
|
|
358
|
+
if (aCp === EXHAUSTED) return 0;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
function compareSortStrings(a, b) {
|
|
362
|
+
const folded = compareCaseFolded(a, b);
|
|
363
|
+
if (folded !== 0) return folded;
|
|
364
|
+
return compareCodePoints(a, b);
|
|
365
|
+
}
|
|
366
|
+
function appendFoldedCodePoint(codePoint, out) {
|
|
367
|
+
const folded = foldOf(codePoint, singleFoldTable(), multiFoldTable());
|
|
368
|
+
if (typeof folded === "number") {
|
|
369
|
+
out.push(folded);
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
for (const foldedCodePoint of folded) out.push(foldedCodePoint);
|
|
373
|
+
}
|
|
374
|
+
function appendFoldedCodePoints(text, out) {
|
|
375
|
+
const single = singleFoldTable();
|
|
376
|
+
const multi = multiFoldTable();
|
|
377
|
+
const cursor = { index: 0, pending: null, pendingIndex: 0 };
|
|
378
|
+
for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
|
|
379
|
+
out.push(codePoint);
|
|
380
|
+
codePoint = nextFoldedCodePoint(text, cursor, single, multi);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
function foldedCodePointCount(text) {
|
|
384
|
+
const folded = [];
|
|
385
|
+
appendFoldedCodePoints(text, folded);
|
|
386
|
+
return folded.length;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// src/core/ordering/constants.ts
|
|
390
|
+
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
391
|
+
|
|
392
|
+
// src/core/ordering/sort-values.ts
|
|
393
|
+
var SORT_MODES = ["min", "max", "avg", "median"];
|
|
394
|
+
function isSortMode(value) {
|
|
395
|
+
return value === "min" || value === "max" || value === "avg" || value === "median";
|
|
396
|
+
}
|
|
397
|
+
function defaultSortMode(direction) {
|
|
398
|
+
return direction === "desc" ? "max" : "min";
|
|
399
|
+
}
|
|
400
|
+
function sortModeOf(entry) {
|
|
401
|
+
return entry.mode ?? defaultSortMode(entry.direction);
|
|
402
|
+
}
|
|
403
|
+
function truncateSortString(value) {
|
|
404
|
+
if (value.length <= SORT_VALUE_MAX_CODE_POINTS) return value;
|
|
405
|
+
let index = 0;
|
|
406
|
+
let counted = 0;
|
|
407
|
+
while (index < value.length && counted < SORT_VALUE_MAX_CODE_POINTS) {
|
|
408
|
+
const unit = value.charCodeAt(index);
|
|
409
|
+
if (unit >= 55296 && unit < 56320 && index + 1 < value.length) {
|
|
410
|
+
const low = value.charCodeAt(index + 1);
|
|
411
|
+
if (low >= 56320 && low < 57344) index++;
|
|
412
|
+
}
|
|
413
|
+
index++;
|
|
414
|
+
counted++;
|
|
415
|
+
}
|
|
416
|
+
return value.slice(0, index);
|
|
417
|
+
}
|
|
418
|
+
function toComparableSortValue(value) {
|
|
419
|
+
if (typeof value === "string") return truncateSortString(value);
|
|
420
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
421
|
+
if (typeof value === "boolean") return value;
|
|
422
|
+
return null;
|
|
423
|
+
}
|
|
424
|
+
function numericReduction(values, mode) {
|
|
425
|
+
const numbers = [];
|
|
426
|
+
for (const element of values) {
|
|
427
|
+
if (typeof element === "number" && Number.isFinite(element)) numbers.push(element);
|
|
428
|
+
}
|
|
429
|
+
if (numbers.length === 0) return null;
|
|
430
|
+
if (mode === "avg") {
|
|
431
|
+
let mean = 0;
|
|
432
|
+
for (let index = 0; index < numbers.length; index++) {
|
|
433
|
+
mean += (numbers[index] - mean) / (index + 1);
|
|
434
|
+
}
|
|
435
|
+
return mean;
|
|
436
|
+
}
|
|
437
|
+
numbers.sort((a, b) => a - b);
|
|
438
|
+
const middle = numbers.length >> 1;
|
|
439
|
+
if (numbers.length % 2 === 1) return numbers[middle];
|
|
440
|
+
return numbers[middle - 1] / 2 + numbers[middle] / 2;
|
|
441
|
+
}
|
|
442
|
+
function toReducedSortValue(value, mode) {
|
|
443
|
+
if (!Array.isArray(value)) return toComparableSortValue(value);
|
|
444
|
+
if (mode === "avg" || mode === "median") return numericReduction(value, mode);
|
|
445
|
+
let chosen = null;
|
|
446
|
+
for (const element of value) {
|
|
447
|
+
const comparable = toComparableSortValue(element);
|
|
448
|
+
if (comparable === null) continue;
|
|
449
|
+
if (chosen === null) {
|
|
450
|
+
chosen = comparable;
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
const comparison = comparePresentValues(comparable, chosen);
|
|
454
|
+
if (mode === "min" ? comparison < 0 : comparison > 0) chosen = comparable;
|
|
455
|
+
}
|
|
456
|
+
return chosen;
|
|
457
|
+
}
|
|
458
|
+
function readSortField(document2, path) {
|
|
459
|
+
const segments = path.split(".");
|
|
460
|
+
let current = document2;
|
|
461
|
+
for (const segment of segments) {
|
|
462
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
463
|
+
if (!Object.hasOwn(current, segment)) return void 0;
|
|
464
|
+
current = current[segment];
|
|
465
|
+
}
|
|
466
|
+
return current;
|
|
467
|
+
}
|
|
468
|
+
function typeRank(value) {
|
|
469
|
+
if (typeof value === "number") return 0;
|
|
470
|
+
if (typeof value === "string") return 1;
|
|
471
|
+
return 2;
|
|
472
|
+
}
|
|
473
|
+
function comparePresentValues(a, b) {
|
|
474
|
+
const rankDifference = typeRank(a) - typeRank(b);
|
|
475
|
+
if (rankDifference !== 0) return rankDifference;
|
|
476
|
+
if (typeof a === "number" && typeof b === "number") return a - b;
|
|
477
|
+
if (typeof a === "string" && typeof b === "string") return compareSortStrings(a, b);
|
|
478
|
+
if (a === b) return 0;
|
|
479
|
+
return a ? 1 : -1;
|
|
480
|
+
}
|
|
481
|
+
function compareComparableValues(a, b, direction) {
|
|
482
|
+
if (a === null && b === null) return 0;
|
|
483
|
+
if (a === null) return 1;
|
|
484
|
+
if (b === null) return -1;
|
|
485
|
+
const comparison = comparePresentValues(a, b);
|
|
486
|
+
if (comparison === 0) return 0;
|
|
487
|
+
return direction === "desc" ? -comparison : comparison;
|
|
488
|
+
}
|
|
489
|
+
function compareComparableKeys(aKey, bKey, directions) {
|
|
490
|
+
for (let i = 0; i < directions.length; i++) {
|
|
491
|
+
const comparison = compareComparableValues(aKey[i] ?? null, bKey[i] ?? null, directions[i]);
|
|
492
|
+
if (comparison !== 0) return comparison;
|
|
493
|
+
}
|
|
494
|
+
return 0;
|
|
495
|
+
}
|
|
496
|
+
function compareSortValues(aValues, bValues, directions) {
|
|
497
|
+
for (let i = 0; i < directions.length; i++) {
|
|
498
|
+
const a = toComparableSortValue(aValues[i]);
|
|
499
|
+
const b = toComparableSortValue(bValues[i]);
|
|
500
|
+
if (a === null && b === null) continue;
|
|
501
|
+
if (a === null) return 1;
|
|
502
|
+
if (b === null) return -1;
|
|
503
|
+
const comparison = comparePresentValues(a, b);
|
|
504
|
+
if (comparison !== 0) return directions[i] === "desc" ? -comparison : comparison;
|
|
505
|
+
}
|
|
506
|
+
return 0;
|
|
507
|
+
}
|
|
508
|
+
|
|
217
509
|
// src/schema/constants.ts
|
|
218
510
|
var MAX_NESTING_DEPTH = 4;
|
|
219
511
|
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
@@ -477,6 +769,18 @@ function schemaFieldsOf(schema) {
|
|
|
477
769
|
parsedSchemas.set(schema, frozen);
|
|
478
770
|
return frozen;
|
|
479
771
|
}
|
|
772
|
+
var partialFieldsBySchema = /* @__PURE__ */ new WeakMap();
|
|
773
|
+
function partialFieldsOf(schema) {
|
|
774
|
+
const cached = partialFieldsBySchema.get(schema);
|
|
775
|
+
if (cached !== void 0) return cached;
|
|
776
|
+
const paths = [];
|
|
777
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
778
|
+
if (field.partial) paths.push(field.path);
|
|
779
|
+
}
|
|
780
|
+
const frozen = Object.freeze(paths.sort(compareCodePoints));
|
|
781
|
+
partialFieldsBySchema.set(schema, frozen);
|
|
782
|
+
return frozen;
|
|
783
|
+
}
|
|
480
784
|
function extractVectorFieldsFromSchema(schema) {
|
|
481
785
|
const result = /* @__PURE__ */ new Map();
|
|
482
786
|
for (const field of schemaFieldsOf(schema)) {
|
|
@@ -485,10 +789,70 @@ function extractVectorFieldsFromSchema(schema) {
|
|
|
485
789
|
return result;
|
|
486
790
|
}
|
|
487
791
|
|
|
488
|
-
// src/core/
|
|
792
|
+
// src/core/partition/partial-words/registry.ts
|
|
489
793
|
var registered = null;
|
|
794
|
+
function isPartialWordSearchModule(value) {
|
|
795
|
+
return value !== null && typeof value === "object" && value.name === "partialWords" && "createGrowingSlot" in value && typeof value.createGrowingSlot === "function" && "createFrozenSlot" in value && typeof value.createFrozenSlot === "function" && "buildTable" in value && typeof value.buildTable === "function" && "expand" in value && typeof value.expand === "function";
|
|
796
|
+
}
|
|
797
|
+
function registerPartialWordSearch(module) {
|
|
798
|
+
if (!isPartialWordSearchModule(module)) {
|
|
799
|
+
throw new NarsilError(
|
|
800
|
+
ErrorCodes.CONFIG_INVALID,
|
|
801
|
+
'Pass registerPartialWordSearch the partialWordSearch object that "@delali/narsil/partial-words" exports'
|
|
802
|
+
);
|
|
803
|
+
}
|
|
804
|
+
registered = module;
|
|
805
|
+
}
|
|
806
|
+
function availablePartialWordSearch() {
|
|
807
|
+
return registered ?? platformPartialWordSearch();
|
|
808
|
+
}
|
|
809
|
+
function requirePartialWordSearchFor(schema) {
|
|
810
|
+
const fields = partialFieldsOf(schema);
|
|
811
|
+
if (fields.length === 0 || availablePartialWordSearch() !== null) return;
|
|
812
|
+
throw new NarsilError(
|
|
813
|
+
ErrorCodes.CONFIG_INVALID,
|
|
814
|
+
`The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load the index`,
|
|
815
|
+
{ field: fields[0] }
|
|
816
|
+
);
|
|
817
|
+
}
|
|
818
|
+
var EMPTY_GROWING_SLOT = {
|
|
819
|
+
readerFor: () => null,
|
|
820
|
+
spellings: () => null,
|
|
821
|
+
bytes: () => 0,
|
|
822
|
+
afterDocument: () => void 0,
|
|
823
|
+
afterMerge: () => void 0,
|
|
824
|
+
forgetSpellings: () => void 0,
|
|
825
|
+
clear: () => void 0
|
|
826
|
+
};
|
|
827
|
+
var EMPTY_FROZEN_SLOT = {
|
|
828
|
+
readerFor: () => null,
|
|
829
|
+
spellings: () => null,
|
|
830
|
+
bytes: () => 0,
|
|
831
|
+
afterRemoval: () => void 0
|
|
832
|
+
};
|
|
833
|
+
function createGrowingPartialWords(view) {
|
|
834
|
+
return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
|
|
835
|
+
}
|
|
836
|
+
function createFrozenPartialWords(initial, tokenTable, documents) {
|
|
837
|
+
return availablePartialWordSearch()?.createFrozenSlot(initial, tokenTable, documents) ?? EMPTY_FROZEN_SLOT;
|
|
838
|
+
}
|
|
839
|
+
function buildPartialWordTableFrom(source, tokenTable) {
|
|
840
|
+
return availablePartialWordSearch()?.buildTable(source, tokenTable) ?? null;
|
|
841
|
+
}
|
|
842
|
+
function expandPartialWords(subs, analysis, requests) {
|
|
843
|
+
const module = availablePartialWordSearch();
|
|
844
|
+
return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
// src/core/pattern-index/platform.browser.ts
|
|
848
|
+
function platformPatternSearch() {
|
|
849
|
+
return null;
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
// src/core/pattern-index/registry.ts
|
|
853
|
+
var registered2 = null;
|
|
490
854
|
function isPatternSearchModule(value) {
|
|
491
|
-
return value !== null && typeof value === "object" && value.name === "pattern" && "createIndex" in value && typeof value.createIndex === "function" && "readArrays" in value && typeof value.readArrays === "function" && "mergeArrays" in value && typeof value.mergeArrays === "function" && "matchInto" in value && typeof value.matchInto === "function";
|
|
855
|
+
return value !== null && typeof value === "object" && value.name === "pattern" && "createIndex" in value && typeof value.createIndex === "function" && "readArrays" in value && typeof value.readArrays === "function" && "mergeArrays" in value && typeof value.mergeArrays === "function" && "matchInto" in value && typeof value.matchInto === "function" && "checkExpression" in value && typeof value.checkExpression === "function" && "expressionFootprint" in value && typeof value.expressionFootprint === "function";
|
|
492
856
|
}
|
|
493
857
|
function registerPatternSearch(module) {
|
|
494
858
|
if (!isPatternSearchModule(module)) {
|
|
@@ -497,10 +861,10 @@ function registerPatternSearch(module) {
|
|
|
497
861
|
'registerPatternSearch takes the patternSearch object that "@delali/narsil/pattern" exports'
|
|
498
862
|
);
|
|
499
863
|
}
|
|
500
|
-
|
|
864
|
+
registered2 = module;
|
|
501
865
|
}
|
|
502
866
|
function availablePatternSearch() {
|
|
503
|
-
return
|
|
867
|
+
return registered2 ?? platformPatternSearch();
|
|
504
868
|
}
|
|
505
869
|
function requirePatternSearchFor(schema) {
|
|
506
870
|
for (const { path, pattern } of schemaFieldsOf(schema)) {
|
|
@@ -2268,61 +2632,7 @@ var MATCHER_STATE_OVERHEAD_BYTES = 96;
|
|
|
2268
2632
|
var PATTERN_WORK_FLUSH_UNITS = 65536;
|
|
2269
2633
|
var PATTERN_RUN_CODE_POINTS = 3;
|
|
2270
2634
|
var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "contains"];
|
|
2271
|
-
|
|
2272
|
-
// src/core/ordering/code-points.ts
|
|
2273
|
-
function exceedsCodePoints(value, maximum) {
|
|
2274
|
-
if (value.length <= maximum) return false;
|
|
2275
|
-
let counted = 0;
|
|
2276
|
-
let i = 0;
|
|
2277
|
-
while (i < value.length) {
|
|
2278
|
-
const unit = value.charCodeAt(i);
|
|
2279
|
-
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
2280
|
-
const low = value.charCodeAt(i + 1);
|
|
2281
|
-
if (low >= 56320 && low < 57344) i++;
|
|
2282
|
-
}
|
|
2283
|
-
i++;
|
|
2284
|
-
counted++;
|
|
2285
|
-
if (counted > maximum) return true;
|
|
2286
|
-
}
|
|
2287
|
-
return false;
|
|
2288
|
-
}
|
|
2289
|
-
var SURROGATE_UNIT = /[\ud800-\udfff]/;
|
|
2290
|
-
function holdsUnpairedSurrogate(value) {
|
|
2291
|
-
if (!SURROGATE_UNIT.test(value)) return false;
|
|
2292
|
-
for (let i = 0; i < value.length; i++) {
|
|
2293
|
-
const unit = value.charCodeAt(i);
|
|
2294
|
-
if (unit < 55296 || unit >= 57344) continue;
|
|
2295
|
-
if (unit >= 56320) return true;
|
|
2296
|
-
const low = value.charCodeAt(i + 1);
|
|
2297
|
-
if (!(low >= 56320 && low < 57344)) return true;
|
|
2298
|
-
i++;
|
|
2299
|
-
}
|
|
2300
|
-
return false;
|
|
2301
|
-
}
|
|
2302
|
-
function codePointCount(value) {
|
|
2303
|
-
let counted = 0;
|
|
2304
|
-
for (let i = 0; i < value.length; i++) {
|
|
2305
|
-
const unit = value.charCodeAt(i);
|
|
2306
|
-
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
2307
|
-
const low = value.charCodeAt(i + 1);
|
|
2308
|
-
if (low >= 56320 && low < 57344) i++;
|
|
2309
|
-
}
|
|
2310
|
-
counted++;
|
|
2311
|
-
}
|
|
2312
|
-
return counted;
|
|
2313
|
-
}
|
|
2314
|
-
function adjustUtf16Unit(unit) {
|
|
2315
|
-
if (unit < 55296) return unit;
|
|
2316
|
-
if (unit < 57344) return unit + 8192;
|
|
2317
|
-
return unit - 2048;
|
|
2318
|
-
}
|
|
2319
|
-
function compareCodePoints(a, b) {
|
|
2320
|
-
const shorter = a.length < b.length ? a.length : b.length;
|
|
2321
|
-
let i = 0;
|
|
2322
|
-
while (i < shorter && a.charCodeAt(i) === b.charCodeAt(i)) i++;
|
|
2323
|
-
if (i === shorter) return a.length - b.length;
|
|
2324
|
-
return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
|
|
2325
|
-
}
|
|
2635
|
+
var PATTERN_EXPRESSION_OPERATORS = ["wildcard", "regex"];
|
|
2326
2636
|
|
|
2327
2637
|
// src/geo/constants.ts
|
|
2328
2638
|
var DEG_TO_RAD = Math.PI / 180;
|
|
@@ -2845,243 +3155,6 @@ function validateVectorPromotion(config) {
|
|
|
2845
3155
|
}
|
|
2846
3156
|
}
|
|
2847
3157
|
|
|
2848
|
-
// src/core/ordering/fold-table.ts
|
|
2849
|
-
var SINGLE_FOLDS = "1t,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2j,lj;b,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;y,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3d;1,1;2,1;2,1;2,-7g;2,5u;1,1;2,1;2,5q;1,1;2,5p;1,5p;1,1;3,27;1,5m;1,5n;1,1;2,5p;1,5r;2,5v;1,5t;1,1;4,5v;1,5x;2,5y;1,1;2,1;2,1;2,62;1,1;2,62;3,1;2,62;1,1;2,61;1,61;1,1;2,1;2,63;1,1;4,1;8,2;1,1;2,2;1,1;2,2;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,2;1,1;2,1;2,-2p;1,-1k;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;8,8bv;1,1;2,-4j;1,8bs;3,1;2,-5f;1,1x;1,1z;1,1;2,1;2,1;2,1;2,1;6v,38;17,1;2,1;4,1;9,38;7,12;2,11;1,11;1,11;2,1s;2,1r;1,1r;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;n,1;d,8;1,-u;1,-p;4,-f;1,-m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1i;1,-1c;3,-1o;1,-1s;2,1;2,-7;1,1;3,-3m;1,-3m;1,-3m;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1d,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,f;1,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;28a,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;2,5ls;6,5ls;mj,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1oj,-4su;1,-4st;1,-4sk;1,-4si;1,-4si;1,-4sj;1,-4sc;1,-4ro;1,r7n;1,1;7,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;3,-2bk;1,-2bk;1,-2bk;8x,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;7,-1m;5,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;b,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;c,-8;2,-8;2,-8;2,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;21,-8;1,-8;1,-22;1,-22;3,-5j9;a,-2e;1,-2e;1,-2e;1,-2e;d,-8;1,-8;1,-2s;1,-2s;d,-8;1,-8;1,-34;1,-34;1,-7;c,-3k;1,-3k;1,-3i;1,-3i;8b,-5st;4,-6gv;1,-6di;7,s;1a,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;k,1;mr,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1f5,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1d,1;2,-8af;1,-2xy;1,-89z;3,1;2,1;2,1;2,-8bg;1,-8al;1,-8bj;1,-8bi;2,1;3,1;9,-8cf;1,-8cf;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;9,1;2,1;5,1;nym,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;k,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3s,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;b,1;2,1;2,-r9g;1,1;2,1;2,1;2,1;2,1;5,1;2,-wmg;3,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wn8;1,-wnj;1,-wnf;1,-wn5;1,-wn8;2,-wlu;1,-wmi;1,-wlx;1,ps;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1c;1,-wn7;1,-raw;1,1;2,1;2,-wo7;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wu9;p,1;or,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;ggy,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;xy,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;3t,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;4d,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1d7,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;4e,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;27v,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;gw1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1t,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;o5k,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;";
|
|
2850
|
-
var MULTI_FOLDS = "67,37,37;29,2x,lj;p,jg,32;4n,2y,lo;bk,qh,lk,ld;w,qt,lk,ld;d3,12d,136;4y7,2w,mp;1,38,lk;1,3b,lm;1,3d,lm;1,2p,ji;4,37,37;4y,qt,lv;2,qt,lv,lc;2,qt,lv,ld;2,qt,lv,n6;16,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;3,67k,qh;1,q9,qh;1,q4,qh;2,q9,n6;1,q9,n6,qh;5,q9,qh;6,67o,qh;1,qf,qh;1,q6,qh;2,qf,n6;1,qf,n6,qh;5,qf,qh;6,qh,lk,lc;1,qh,lk,ld;3,qh,n6;1,qh,lk,n6;b,qt,lk,lc;1,qt,lk,ld;1,qp,lv;2,qt,n6;1,qt,lk,n6;b,67w,qh;1,qx,qh;1,r2,qh;2,qx,n6;1,qx,n6,qh;5,qx,qh;179g,2u,2u;1,2u,2x;1,2u,30;1,2u,2u,2x;1,2u,2u,30;1,37,38;1,37,38;d,12s,12u;1,12s,12d;1,12s,12j;1,132,12u;1,12s,12l;";
|
|
2851
|
-
var singleTable = null;
|
|
2852
|
-
var multiTable = null;
|
|
2853
|
-
function singleFoldTable() {
|
|
2854
|
-
if (singleTable === null) {
|
|
2855
|
-
singleTable = /* @__PURE__ */ new Map();
|
|
2856
|
-
let source = 0;
|
|
2857
|
-
for (const entry of SINGLE_FOLDS.split(";")) {
|
|
2858
|
-
if (entry.length === 0) continue;
|
|
2859
|
-
const [delta, offset] = entry.split(",");
|
|
2860
|
-
source += Number.parseInt(delta, 36);
|
|
2861
|
-
singleTable.set(source, source + Number.parseInt(offset, 36));
|
|
2862
|
-
}
|
|
2863
|
-
}
|
|
2864
|
-
return singleTable;
|
|
2865
|
-
}
|
|
2866
|
-
function multiFoldTable() {
|
|
2867
|
-
if (multiTable === null) {
|
|
2868
|
-
multiTable = /* @__PURE__ */ new Map();
|
|
2869
|
-
let source = 0;
|
|
2870
|
-
for (const entry of MULTI_FOLDS.split(";")) {
|
|
2871
|
-
if (entry.length === 0) continue;
|
|
2872
|
-
const parts = entry.split(",");
|
|
2873
|
-
source += Number.parseInt(parts[0], 36);
|
|
2874
|
-
multiTable.set(
|
|
2875
|
-
source,
|
|
2876
|
-
parts.slice(1).map((cp) => Number.parseInt(cp, 36))
|
|
2877
|
-
);
|
|
2878
|
-
}
|
|
2879
|
-
}
|
|
2880
|
-
return multiTable;
|
|
2881
|
-
}
|
|
2882
|
-
|
|
2883
|
-
// src/core/ordering/fold-compare.ts
|
|
2884
|
-
var EXHAUSTED = -1;
|
|
2885
|
-
function foldOf(codePoint, single, multi) {
|
|
2886
|
-
if (codePoint < 128) return codePoint >= 65 && codePoint <= 90 ? codePoint + 32 : codePoint;
|
|
2887
|
-
return single.get(codePoint) ?? multi.get(codePoint) ?? codePoint;
|
|
2888
|
-
}
|
|
2889
|
-
function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
2890
|
-
if (cursor.pending !== null) {
|
|
2891
|
-
const cp2 = cursor.pending[cursor.pendingIndex];
|
|
2892
|
-
cursor.pendingIndex++;
|
|
2893
|
-
if (cursor.pendingIndex === cursor.pending.length) {
|
|
2894
|
-
cursor.pending = null;
|
|
2895
|
-
cursor.pendingIndex = 0;
|
|
2896
|
-
}
|
|
2897
|
-
return cp2;
|
|
2898
|
-
}
|
|
2899
|
-
if (cursor.index >= text.length) return EXHAUSTED;
|
|
2900
|
-
let cp = text.charCodeAt(cursor.index);
|
|
2901
|
-
cursor.index++;
|
|
2902
|
-
if (cp >= 55296 && cp < 56320 && cursor.index < text.length) {
|
|
2903
|
-
const low = text.charCodeAt(cursor.index);
|
|
2904
|
-
if (low >= 56320 && low < 57344) {
|
|
2905
|
-
cp = 65536 + (cp - 55296 << 10) + (low - 56320);
|
|
2906
|
-
cursor.index++;
|
|
2907
|
-
}
|
|
2908
|
-
}
|
|
2909
|
-
const folded = foldOf(cp, single, multi);
|
|
2910
|
-
if (typeof folded === "number") return folded;
|
|
2911
|
-
if (folded.length > 1) {
|
|
2912
|
-
cursor.pending = folded;
|
|
2913
|
-
cursor.pendingIndex = 1;
|
|
2914
|
-
}
|
|
2915
|
-
return folded[0];
|
|
2916
|
-
}
|
|
2917
|
-
function compareCaseFolded(a, b) {
|
|
2918
|
-
const shorter = a.length < b.length ? a.length : b.length;
|
|
2919
|
-
let start = 0;
|
|
2920
|
-
while (start < shorter && a.charCodeAt(start) === b.charCodeAt(start)) start++;
|
|
2921
|
-
if (start === a.length && start === b.length) return 0;
|
|
2922
|
-
if (start > 0) {
|
|
2923
|
-
const previous = a.charCodeAt(start - 1);
|
|
2924
|
-
if (previous >= 55296 && previous < 56320) start--;
|
|
2925
|
-
}
|
|
2926
|
-
const single = singleFoldTable();
|
|
2927
|
-
const multi = multiFoldTable();
|
|
2928
|
-
const aCursor = { index: start, pending: null, pendingIndex: 0 };
|
|
2929
|
-
const bCursor = { index: start, pending: null, pendingIndex: 0 };
|
|
2930
|
-
for (; ; ) {
|
|
2931
|
-
const aCp = nextFoldedCodePoint(a, aCursor, single, multi);
|
|
2932
|
-
const bCp = nextFoldedCodePoint(b, bCursor, single, multi);
|
|
2933
|
-
if (aCp !== bCp) return aCp - bCp;
|
|
2934
|
-
if (aCp === EXHAUSTED) return 0;
|
|
2935
|
-
}
|
|
2936
|
-
}
|
|
2937
|
-
function compareSortStrings(a, b) {
|
|
2938
|
-
const folded = compareCaseFolded(a, b);
|
|
2939
|
-
if (folded !== 0) return folded;
|
|
2940
|
-
return compareCodePoints(a, b);
|
|
2941
|
-
}
|
|
2942
|
-
function appendFoldedCodePoint(codePoint, out) {
|
|
2943
|
-
const folded = foldOf(codePoint, singleFoldTable(), multiFoldTable());
|
|
2944
|
-
if (typeof folded === "number") {
|
|
2945
|
-
out.push(folded);
|
|
2946
|
-
return;
|
|
2947
|
-
}
|
|
2948
|
-
for (const foldedCodePoint of folded) out.push(foldedCodePoint);
|
|
2949
|
-
}
|
|
2950
|
-
function appendFoldedCodePoints(text, out) {
|
|
2951
|
-
const single = singleFoldTable();
|
|
2952
|
-
const multi = multiFoldTable();
|
|
2953
|
-
const cursor = { index: 0, pending: null, pendingIndex: 0 };
|
|
2954
|
-
for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
|
|
2955
|
-
out.push(codePoint);
|
|
2956
|
-
codePoint = nextFoldedCodePoint(text, cursor, single, multi);
|
|
2957
|
-
}
|
|
2958
|
-
}
|
|
2959
|
-
function foldedCodePointCount(text) {
|
|
2960
|
-
const folded = [];
|
|
2961
|
-
appendFoldedCodePoints(text, folded);
|
|
2962
|
-
return folded.length;
|
|
2963
|
-
}
|
|
2964
|
-
|
|
2965
|
-
// src/core/ordering/constants.ts
|
|
2966
|
-
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
2967
|
-
|
|
2968
|
-
// src/core/ordering/sort-values.ts
|
|
2969
|
-
var SORT_MODES = ["min", "max", "avg", "median"];
|
|
2970
|
-
function isSortMode(value) {
|
|
2971
|
-
return value === "min" || value === "max" || value === "avg" || value === "median";
|
|
2972
|
-
}
|
|
2973
|
-
function defaultSortMode(direction) {
|
|
2974
|
-
return direction === "desc" ? "max" : "min";
|
|
2975
|
-
}
|
|
2976
|
-
function sortModeOf(entry) {
|
|
2977
|
-
return entry.mode ?? defaultSortMode(entry.direction);
|
|
2978
|
-
}
|
|
2979
|
-
function truncateSortString(value) {
|
|
2980
|
-
if (value.length <= SORT_VALUE_MAX_CODE_POINTS) return value;
|
|
2981
|
-
let index = 0;
|
|
2982
|
-
let counted = 0;
|
|
2983
|
-
while (index < value.length && counted < SORT_VALUE_MAX_CODE_POINTS) {
|
|
2984
|
-
const unit = value.charCodeAt(index);
|
|
2985
|
-
if (unit >= 55296 && unit < 56320 && index + 1 < value.length) {
|
|
2986
|
-
const low = value.charCodeAt(index + 1);
|
|
2987
|
-
if (low >= 56320 && low < 57344) index++;
|
|
2988
|
-
}
|
|
2989
|
-
index++;
|
|
2990
|
-
counted++;
|
|
2991
|
-
}
|
|
2992
|
-
return value.slice(0, index);
|
|
2993
|
-
}
|
|
2994
|
-
function toComparableSortValue(value) {
|
|
2995
|
-
if (typeof value === "string") return truncateSortString(value);
|
|
2996
|
-
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
2997
|
-
if (typeof value === "boolean") return value;
|
|
2998
|
-
return null;
|
|
2999
|
-
}
|
|
3000
|
-
function numericReduction(values, mode) {
|
|
3001
|
-
const numbers = [];
|
|
3002
|
-
for (const element of values) {
|
|
3003
|
-
if (typeof element === "number" && Number.isFinite(element)) numbers.push(element);
|
|
3004
|
-
}
|
|
3005
|
-
if (numbers.length === 0) return null;
|
|
3006
|
-
if (mode === "avg") {
|
|
3007
|
-
let mean = 0;
|
|
3008
|
-
for (let index = 0; index < numbers.length; index++) {
|
|
3009
|
-
mean += (numbers[index] - mean) / (index + 1);
|
|
3010
|
-
}
|
|
3011
|
-
return mean;
|
|
3012
|
-
}
|
|
3013
|
-
numbers.sort((a, b) => a - b);
|
|
3014
|
-
const middle = numbers.length >> 1;
|
|
3015
|
-
if (numbers.length % 2 === 1) return numbers[middle];
|
|
3016
|
-
return numbers[middle - 1] / 2 + numbers[middle] / 2;
|
|
3017
|
-
}
|
|
3018
|
-
function toReducedSortValue(value, mode) {
|
|
3019
|
-
if (!Array.isArray(value)) return toComparableSortValue(value);
|
|
3020
|
-
if (mode === "avg" || mode === "median") return numericReduction(value, mode);
|
|
3021
|
-
let chosen = null;
|
|
3022
|
-
for (const element of value) {
|
|
3023
|
-
const comparable = toComparableSortValue(element);
|
|
3024
|
-
if (comparable === null) continue;
|
|
3025
|
-
if (chosen === null) {
|
|
3026
|
-
chosen = comparable;
|
|
3027
|
-
continue;
|
|
3028
|
-
}
|
|
3029
|
-
const comparison = comparePresentValues(comparable, chosen);
|
|
3030
|
-
if (mode === "min" ? comparison < 0 : comparison > 0) chosen = comparable;
|
|
3031
|
-
}
|
|
3032
|
-
return chosen;
|
|
3033
|
-
}
|
|
3034
|
-
function readSortField(document2, path) {
|
|
3035
|
-
const segments = path.split(".");
|
|
3036
|
-
let current = document2;
|
|
3037
|
-
for (const segment of segments) {
|
|
3038
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
3039
|
-
if (!Object.hasOwn(current, segment)) return void 0;
|
|
3040
|
-
current = current[segment];
|
|
3041
|
-
}
|
|
3042
|
-
return current;
|
|
3043
|
-
}
|
|
3044
|
-
function typeRank(value) {
|
|
3045
|
-
if (typeof value === "number") return 0;
|
|
3046
|
-
if (typeof value === "string") return 1;
|
|
3047
|
-
return 2;
|
|
3048
|
-
}
|
|
3049
|
-
function comparePresentValues(a, b) {
|
|
3050
|
-
const rankDifference = typeRank(a) - typeRank(b);
|
|
3051
|
-
if (rankDifference !== 0) return rankDifference;
|
|
3052
|
-
if (typeof a === "number" && typeof b === "number") return a - b;
|
|
3053
|
-
if (typeof a === "string" && typeof b === "string") return compareSortStrings(a, b);
|
|
3054
|
-
if (a === b) return 0;
|
|
3055
|
-
return a ? 1 : -1;
|
|
3056
|
-
}
|
|
3057
|
-
function compareComparableValues(a, b, direction) {
|
|
3058
|
-
if (a === null && b === null) return 0;
|
|
3059
|
-
if (a === null) return 1;
|
|
3060
|
-
if (b === null) return -1;
|
|
3061
|
-
const comparison = comparePresentValues(a, b);
|
|
3062
|
-
if (comparison === 0) return 0;
|
|
3063
|
-
return direction === "desc" ? -comparison : comparison;
|
|
3064
|
-
}
|
|
3065
|
-
function compareComparableKeys(aKey, bKey, directions) {
|
|
3066
|
-
for (let i = 0; i < directions.length; i++) {
|
|
3067
|
-
const comparison = compareComparableValues(aKey[i] ?? null, bKey[i] ?? null, directions[i]);
|
|
3068
|
-
if (comparison !== 0) return comparison;
|
|
3069
|
-
}
|
|
3070
|
-
return 0;
|
|
3071
|
-
}
|
|
3072
|
-
function compareSortValues(aValues, bValues, directions) {
|
|
3073
|
-
for (let i = 0; i < directions.length; i++) {
|
|
3074
|
-
const a = toComparableSortValue(aValues[i]);
|
|
3075
|
-
const b = toComparableSortValue(bValues[i]);
|
|
3076
|
-
if (a === null && b === null) continue;
|
|
3077
|
-
if (a === null) return 1;
|
|
3078
|
-
if (b === null) return -1;
|
|
3079
|
-
const comparison = comparePresentValues(a, b);
|
|
3080
|
-
if (comparison !== 0) return directions[i] === "desc" ? -comparison : comparison;
|
|
3081
|
-
}
|
|
3082
|
-
return 0;
|
|
3083
|
-
}
|
|
3084
|
-
|
|
3085
3158
|
// src/core/document-store.ts
|
|
3086
3159
|
function detachedView(view) {
|
|
3087
3160
|
if (view.byteOffset === 0 && view.byteLength === view.buffer.byteLength) return view;
|
|
@@ -3436,8 +3509,6 @@ function compactList(list) {
|
|
|
3436
3509
|
}
|
|
3437
3510
|
}
|
|
3438
3511
|
writeIdx++;
|
|
3439
|
-
} else {
|
|
3440
|
-
list.totalTermFrequency -= list.termFrequencies[i];
|
|
3441
3512
|
}
|
|
3442
3513
|
}
|
|
3443
3514
|
list.docIds.length = writeIdx;
|
|
@@ -3462,8 +3533,6 @@ function compactDocEntries(list, internalId) {
|
|
|
3462
3533
|
}
|
|
3463
3534
|
}
|
|
3464
3535
|
writeIdx++;
|
|
3465
|
-
} else {
|
|
3466
|
-
list.totalTermFrequency -= list.termFrequencies[i];
|
|
3467
3536
|
}
|
|
3468
3537
|
}
|
|
3469
3538
|
list.docIds.length = writeIdx;
|
|
@@ -3549,12 +3618,13 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
3549
3618
|
}
|
|
3550
3619
|
list.length++;
|
|
3551
3620
|
},
|
|
3552
|
-
remove(token, internalId) {
|
|
3621
|
+
remove(token, internalId, occurrences) {
|
|
3553
3622
|
const list = index.get(token);
|
|
3554
3623
|
if (!list || !list.docIdSet.has(internalId)) return;
|
|
3555
3624
|
list.structureRevision++;
|
|
3556
3625
|
list.docIdSet.delete(internalId);
|
|
3557
3626
|
list.deletedDocs.add(internalId);
|
|
3627
|
+
list.totalTermFrequency = Math.max(0, list.totalTermFrequency - occurrences);
|
|
3558
3628
|
if (list.docIdSet.size === 0) {
|
|
3559
3629
|
index.delete(token);
|
|
3560
3630
|
untrackToken(token);
|
|
@@ -3587,6 +3657,9 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
3587
3657
|
lookup(token) {
|
|
3588
3658
|
return index.get(token);
|
|
3589
3659
|
},
|
|
3660
|
+
documentFrequency(token) {
|
|
3661
|
+
return index.get(token)?.docIdSet.size ?? 0;
|
|
3662
|
+
},
|
|
3590
3663
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
3591
3664
|
if (tolerance === 0) {
|
|
3592
3665
|
const exact = index.get(token);
|
|
@@ -4334,7 +4407,24 @@ function computeFacets(state, matched, config, schema) {
|
|
|
4334
4407
|
}
|
|
4335
4408
|
|
|
4336
4409
|
// src/core/pattern-index/host.ts
|
|
4337
|
-
var
|
|
4410
|
+
var NO_SOURCES = [];
|
|
4411
|
+
var foldSources = null;
|
|
4412
|
+
function sourcesByFold() {
|
|
4413
|
+
if (foldSources !== null) return foldSources;
|
|
4414
|
+
const sources = /* @__PURE__ */ new Map();
|
|
4415
|
+
const add = (source, folded) => {
|
|
4416
|
+
const known = sources.get(folded);
|
|
4417
|
+
if (known === void 0) sources.set(folded, [source]);
|
|
4418
|
+
else known.push(source);
|
|
4419
|
+
};
|
|
4420
|
+
for (const [source, folded] of singleFoldTable()) add(source, folded);
|
|
4421
|
+
foldSources = sources;
|
|
4422
|
+
return sources;
|
|
4423
|
+
}
|
|
4424
|
+
function singleFoldSources(folded) {
|
|
4425
|
+
return sourcesByFold().get(folded) ?? NO_SOURCES;
|
|
4426
|
+
}
|
|
4427
|
+
var PATTERN_HOST = { appendFoldedCodePoint, singleFoldSources };
|
|
4338
4428
|
function patternIndexOf(indexes, fieldPath) {
|
|
4339
4429
|
let index = indexes.get(fieldPath);
|
|
4340
4430
|
if (index === void 0) {
|
|
@@ -5177,6 +5267,53 @@ function createNumericIndex() {
|
|
|
5177
5267
|
};
|
|
5178
5268
|
}
|
|
5179
5269
|
|
|
5270
|
+
// src/core/partition/partial-words/spellings.ts
|
|
5271
|
+
function countSpelling(counts, token, spelling, change) {
|
|
5272
|
+
let byToken = counts.get(token);
|
|
5273
|
+
if (byToken === void 0) {
|
|
5274
|
+
byToken = /* @__PURE__ */ new Map();
|
|
5275
|
+
counts.set(token, byToken);
|
|
5276
|
+
}
|
|
5277
|
+
const next = (byToken.get(spelling) ?? 0) + change;
|
|
5278
|
+
if (next > 0) {
|
|
5279
|
+
byToken.set(spelling, next);
|
|
5280
|
+
return;
|
|
5281
|
+
}
|
|
5282
|
+
byToken.delete(spelling);
|
|
5283
|
+
if (byToken.size === 0) counts.delete(token);
|
|
5284
|
+
}
|
|
5285
|
+
function textsOf(value) {
|
|
5286
|
+
if (typeof value === "string") return [value];
|
|
5287
|
+
if (!Array.isArray(value)) return [];
|
|
5288
|
+
return value.filter((element) => typeof element === "string");
|
|
5289
|
+
}
|
|
5290
|
+
function countDocumentSpellings(counts, document2, analysis, direction) {
|
|
5291
|
+
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
5292
|
+
for (const field of analysis.fields) {
|
|
5293
|
+
for (const text of textsOf(getNestedValue(document2, field))) {
|
|
5294
|
+
for (const occurrence of tokenizeIterator(text, analysis.language, tokenizing)) {
|
|
5295
|
+
countSpelling(counts, occurrence.token, occurrence.surface ?? occurrence.token, direction);
|
|
5296
|
+
}
|
|
5297
|
+
}
|
|
5298
|
+
}
|
|
5299
|
+
}
|
|
5300
|
+
function spellingCountsOf(documents, analysis) {
|
|
5301
|
+
const counts = /* @__PURE__ */ new Map();
|
|
5302
|
+
for (const document2 of documents) countDocumentSpellings(counts, document2, analysis, 1);
|
|
5303
|
+
return counts;
|
|
5304
|
+
}
|
|
5305
|
+
function encodePartialSpellings(fields, counts) {
|
|
5306
|
+
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
5307
|
+
for (const [token, byToken] of counts) {
|
|
5308
|
+
for (const [spelling, count] of byToken) {
|
|
5309
|
+
encoded.tokens.push(token);
|
|
5310
|
+
encoded.spellings.push(spelling);
|
|
5311
|
+
encoded.counts.push(count);
|
|
5312
|
+
}
|
|
5313
|
+
}
|
|
5314
|
+
return encoded;
|
|
5315
|
+
}
|
|
5316
|
+
|
|
5180
5317
|
// src/core/partition/value-sizes.ts
|
|
5181
5318
|
function keepsValueSize(base) {
|
|
5182
5319
|
return base === "string" || base === "verbatim" || base.endsWith("[]");
|
|
@@ -5250,6 +5387,13 @@ function resolveInternalId(state, docId) {
|
|
|
5250
5387
|
}
|
|
5251
5388
|
return internalId;
|
|
5252
5389
|
}
|
|
5390
|
+
function emptyIndexedText(fields) {
|
|
5391
|
+
const holdsPartialField = fields.some((field) => field.partial);
|
|
5392
|
+
return { fieldLengths: {}, tokensByField: {}, partialSpellings: holdsPartialField ? /* @__PURE__ */ new Map() : null };
|
|
5393
|
+
}
|
|
5394
|
+
function partialSpellingsFor(field, indexed) {
|
|
5395
|
+
return field.partial ? indexed.partialSpellings : null;
|
|
5396
|
+
}
|
|
5253
5397
|
function countSurface(counts, surface, token) {
|
|
5254
5398
|
if (surface === void 0 || surface.length === 0) return;
|
|
5255
5399
|
const existing = counts.get(surface);
|
|
@@ -5259,6 +5403,18 @@ function countSurface(counts, surface, token) {
|
|
|
5259
5403
|
counts.set(surface, { token, count: 1 });
|
|
5260
5404
|
}
|
|
5261
5405
|
}
|
|
5406
|
+
function countOccurrence(counts, spellings, surface, token) {
|
|
5407
|
+
countSurface(counts, surface, token);
|
|
5408
|
+
if (spellings !== null) countSpelling(spellings, token, surface ?? token, 1);
|
|
5409
|
+
}
|
|
5410
|
+
function addFieldOccurrences(occurrencesByToken, fieldTokens) {
|
|
5411
|
+
const fieldCounts = /* @__PURE__ */ new Map();
|
|
5412
|
+
for (const token of fieldTokens) fieldCounts.set(token, (fieldCounts.get(token) ?? 0) + 1);
|
|
5413
|
+
for (const [token, count] of fieldCounts) {
|
|
5414
|
+
const stored = count > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : count;
|
|
5415
|
+
occurrencesByToken.set(token, (occurrencesByToken.get(token) ?? 0) + stored);
|
|
5416
|
+
}
|
|
5417
|
+
}
|
|
5262
5418
|
function applySurfaceCounts(state, counts, direction) {
|
|
5263
5419
|
for (const [surface, entry] of counts) {
|
|
5264
5420
|
if (direction === 1) {
|
|
@@ -5268,7 +5424,7 @@ function applySurfaceCounts(state, counts, direction) {
|
|
|
5268
5424
|
}
|
|
5269
5425
|
}
|
|
5270
5426
|
}
|
|
5271
|
-
function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField) {
|
|
5427
|
+
function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField, spellings = null) {
|
|
5272
5428
|
const fieldTokenList = [];
|
|
5273
5429
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
|
|
5274
5430
|
const opts = tokenizeOptions(options);
|
|
@@ -5284,7 +5440,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
5284
5440
|
tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
|
|
5285
5441
|
}
|
|
5286
5442
|
fieldTokenList.push(t.token);
|
|
5287
|
-
|
|
5443
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
5288
5444
|
}
|
|
5289
5445
|
for (const [token, freq] of tokenFreqs) {
|
|
5290
5446
|
state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex2, freq.positions);
|
|
@@ -5294,7 +5450,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
5294
5450
|
for (const t of tokenizeIterator(text, language, opts)) {
|
|
5295
5451
|
tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
|
|
5296
5452
|
fieldTokenList.push(t.token);
|
|
5297
|
-
|
|
5453
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
5298
5454
|
}
|
|
5299
5455
|
for (const [token, count] of tokenCounts) {
|
|
5300
5456
|
state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
|
|
@@ -5304,7 +5460,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
5304
5460
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
5305
5461
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5306
5462
|
}
|
|
5307
|
-
function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
|
|
5463
|
+
function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField, spellings = null) {
|
|
5308
5464
|
const fieldTokenList = [];
|
|
5309
5465
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
|
|
5310
5466
|
const opts = tokenizeOptions(options);
|
|
@@ -5323,7 +5479,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5323
5479
|
tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
|
|
5324
5480
|
}
|
|
5325
5481
|
fieldTokenList.push(t.token);
|
|
5326
|
-
|
|
5482
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
5327
5483
|
itemTokenCount++;
|
|
5328
5484
|
}
|
|
5329
5485
|
positionOffset += itemTokenCount;
|
|
@@ -5338,7 +5494,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5338
5494
|
for (const t of tokenizeIterator(item, language, opts)) {
|
|
5339
5495
|
tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
|
|
5340
5496
|
fieldTokenList.push(t.token);
|
|
5341
|
-
|
|
5497
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
5342
5498
|
}
|
|
5343
5499
|
}
|
|
5344
5500
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
@@ -5350,17 +5506,28 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5350
5506
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5351
5507
|
}
|
|
5352
5508
|
function indexDocument(state, docId, document2, fields, language, options) {
|
|
5353
|
-
const
|
|
5354
|
-
const tokensByField =
|
|
5509
|
+
const indexed = emptyIndexedText(fields);
|
|
5510
|
+
const { fieldLengths, tokensByField } = indexed;
|
|
5355
5511
|
const internalId = resolveInternalId(state, docId);
|
|
5356
|
-
for (const
|
|
5512
|
+
for (const field of fields) {
|
|
5513
|
+
const { path: fieldPath, base, pattern } = field;
|
|
5357
5514
|
const value = getNestedValue(document2, fieldPath);
|
|
5358
5515
|
if (value === void 0 || value === null) continue;
|
|
5359
5516
|
recordValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
5360
5517
|
ensureFieldIndex(state, fieldPath, base);
|
|
5361
5518
|
if (pattern) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
|
|
5362
5519
|
if (base === "string") {
|
|
5363
|
-
indexStringField(
|
|
5520
|
+
indexStringField(
|
|
5521
|
+
state,
|
|
5522
|
+
internalId,
|
|
5523
|
+
fieldPath,
|
|
5524
|
+
value,
|
|
5525
|
+
language,
|
|
5526
|
+
options,
|
|
5527
|
+
fieldLengths,
|
|
5528
|
+
tokensByField,
|
|
5529
|
+
partialSpellingsFor(field, indexed)
|
|
5530
|
+
);
|
|
5364
5531
|
} else if (base === "number") {
|
|
5365
5532
|
getOrCreateNumericIndex(state, fieldPath).insert(internalId, value);
|
|
5366
5533
|
} else if (base === "boolean") {
|
|
@@ -5379,7 +5546,8 @@ function indexDocument(state, docId, document2, fields, language, options) {
|
|
|
5379
5546
|
language,
|
|
5380
5547
|
options,
|
|
5381
5548
|
fieldLengths,
|
|
5382
|
-
tokensByField
|
|
5549
|
+
tokensByField,
|
|
5550
|
+
partialSpellingsFor(field, indexed)
|
|
5383
5551
|
);
|
|
5384
5552
|
} else if (base === "number[]") {
|
|
5385
5553
|
const numIdx = getOrCreateNumericIndex(state, fieldPath);
|
|
@@ -5394,15 +5562,17 @@ function indexDocument(state, docId, document2, fields, language, options) {
|
|
|
5394
5562
|
}
|
|
5395
5563
|
}
|
|
5396
5564
|
}
|
|
5397
|
-
return
|
|
5565
|
+
return indexed;
|
|
5398
5566
|
}
|
|
5399
5567
|
function removeFromIndexes(state, docId, storedDoc, schemaFields, language, options) {
|
|
5400
|
-
const
|
|
5401
|
-
const tokensByField =
|
|
5568
|
+
const removed = emptyIndexedText(schemaFields);
|
|
5569
|
+
const { fieldLengths, tokensByField } = removed;
|
|
5402
5570
|
const fields = storedDoc.fields;
|
|
5403
5571
|
const internalId = resolveInternalId(state, docId);
|
|
5404
5572
|
const opts = tokenizeOptions(options);
|
|
5405
|
-
|
|
5573
|
+
const occurrencesByToken = /* @__PURE__ */ new Map();
|
|
5574
|
+
for (const field of schemaFields) {
|
|
5575
|
+
const { path: fieldPath, base, pattern } = field;
|
|
5406
5576
|
const value = getNestedValue(fields, fieldPath);
|
|
5407
5577
|
if (value === void 0 || value === null) continue;
|
|
5408
5578
|
forgetValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
@@ -5410,16 +5580,15 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
5410
5580
|
if (base === "string") {
|
|
5411
5581
|
const result = tokenize(value, language, opts);
|
|
5412
5582
|
fieldLengths[fieldPath] = result.tokens.length;
|
|
5413
|
-
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
5414
5583
|
const fieldTokenList = [];
|
|
5415
5584
|
const surfaceCounts = /* @__PURE__ */ new Map();
|
|
5585
|
+
const spellings = partialSpellingsFor(field, removed);
|
|
5416
5586
|
for (let i = 0; i < result.tokens.length; i++) {
|
|
5417
5587
|
const t = result.tokens[i];
|
|
5418
|
-
uniqueTokens.add(t.token);
|
|
5419
5588
|
fieldTokenList.push(t.token);
|
|
5420
|
-
|
|
5589
|
+
countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
|
|
5421
5590
|
}
|
|
5422
|
-
|
|
5591
|
+
addFieldOccurrences(occurrencesByToken, fieldTokenList);
|
|
5423
5592
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
5424
5593
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5425
5594
|
} else if (base === "number") {
|
|
@@ -5432,20 +5601,19 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
5432
5601
|
state.geoIndexes.get(fieldPath)?.remove(internalId);
|
|
5433
5602
|
} else if (base === "string[]") {
|
|
5434
5603
|
const arr = value;
|
|
5435
|
-
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
5436
5604
|
const fieldTokenList = [];
|
|
5437
5605
|
const surfaceCounts = /* @__PURE__ */ new Map();
|
|
5606
|
+
const spellings = partialSpellingsFor(field, removed);
|
|
5438
5607
|
for (const item of arr) {
|
|
5439
5608
|
const result = tokenize(item, language, opts);
|
|
5440
5609
|
for (let i = 0; i < result.tokens.length; i++) {
|
|
5441
5610
|
const t = result.tokens[i];
|
|
5442
|
-
uniqueTokens.add(t.token);
|
|
5443
5611
|
fieldTokenList.push(t.token);
|
|
5444
|
-
|
|
5612
|
+
countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
|
|
5445
5613
|
}
|
|
5446
5614
|
}
|
|
5447
5615
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
5448
|
-
|
|
5616
|
+
addFieldOccurrences(occurrencesByToken, fieldTokenList);
|
|
5449
5617
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
5450
5618
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5451
5619
|
} else if (base === "number[]") {
|
|
@@ -5465,7 +5633,8 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
5465
5633
|
}
|
|
5466
5634
|
}
|
|
5467
5635
|
}
|
|
5468
|
-
|
|
5636
|
+
for (const [token, occurrences] of occurrencesByToken) state.invertedIdx.remove(token, internalId, occurrences);
|
|
5637
|
+
return removed;
|
|
5469
5638
|
}
|
|
5470
5639
|
|
|
5471
5640
|
// src/core/partition/field-updates.ts
|
|
@@ -5568,7 +5737,7 @@ function excluding(context, fieldPath, excluded) {
|
|
|
5568
5737
|
const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
|
|
5569
5738
|
return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
|
|
5570
5739
|
}
|
|
5571
|
-
function
|
|
5740
|
+
function textsOf2(operand) {
|
|
5572
5741
|
const texts = /* @__PURE__ */ new Set();
|
|
5573
5742
|
if (!Array.isArray(operand)) return texts;
|
|
5574
5743
|
for (const text of operand) if (typeof text === "string") texts.add(text);
|
|
@@ -5583,11 +5752,13 @@ function evaluatePatternTests(fieldPath, filter, context, bitsets) {
|
|
|
5583
5752
|
};
|
|
5584
5753
|
if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq]));
|
|
5585
5754
|
if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
|
|
5586
|
-
if (filter.in !== void 0) bitsets.push(matching("eq",
|
|
5587
|
-
if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq",
|
|
5755
|
+
if (filter.in !== void 0) bitsets.push(matching("eq", textsOf2(filter.in)));
|
|
5756
|
+
if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf2(filter.nin))));
|
|
5588
5757
|
if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
|
|
5589
5758
|
if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
|
|
5590
5759
|
if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
|
|
5760
|
+
if (typeof filter.wildcard === "string") bitsets.push(matching("wildcard", [filter.wildcard]));
|
|
5761
|
+
if (typeof filter.regex === "string") bitsets.push(matching("regex", [filter.regex]));
|
|
5591
5762
|
}
|
|
5592
5763
|
|
|
5593
5764
|
// src/filters/field-tests.ts
|
|
@@ -5792,6 +5963,8 @@ var FIELD_FILTER_OPERATORS = [
|
|
|
5792
5963
|
"startsWith",
|
|
5793
5964
|
"endsWith",
|
|
5794
5965
|
"contains",
|
|
5966
|
+
"wildcard",
|
|
5967
|
+
"regex",
|
|
5795
5968
|
"caseFold",
|
|
5796
5969
|
"containsAll",
|
|
5797
5970
|
"matchesAny",
|
|
@@ -6007,14 +6180,16 @@ function requirePatternField(fieldPath, operator, type) {
|
|
|
6007
6180
|
operator
|
|
6008
6181
|
);
|
|
6009
6182
|
}
|
|
6183
|
+
function requireTextLength(fieldPath, operator, text) {
|
|
6184
|
+
if (!exceedsCodePoints(text, MAX_PATTERN_TEXT_CODE_POINTS)) return;
|
|
6185
|
+
invalid(
|
|
6186
|
+
`The text of "${operator}" on pattern field "${fieldPath}" holds more than ${MAX_PATTERN_TEXT_CODE_POINTS} code points`,
|
|
6187
|
+
fieldPath,
|
|
6188
|
+
operator
|
|
6189
|
+
);
|
|
6190
|
+
}
|
|
6010
6191
|
function requirePatternText(fieldPath, operator, text, caseFold) {
|
|
6011
|
-
|
|
6012
|
-
invalid(
|
|
6013
|
-
`The text of "${operator}" on pattern field "${fieldPath}" holds more than ${MAX_PATTERN_TEXT_CODE_POINTS} code points`,
|
|
6014
|
-
fieldPath,
|
|
6015
|
-
operator
|
|
6016
|
-
);
|
|
6017
|
-
}
|
|
6192
|
+
requireTextLength(fieldPath, operator, text);
|
|
6018
6193
|
if (caseFold && foldedCodePointCount(text) > MAX_PATTERN_POSITIONS) {
|
|
6019
6194
|
invalid(
|
|
6020
6195
|
`The text of "${operator}" on pattern field "${fieldPath}" has a case fold of more than ${MAX_PATTERN_POSITIONS} code points, while a pattern holds at most ${MAX_PATTERN_POSITIONS} positions`,
|
|
@@ -6024,6 +6199,23 @@ function requirePatternText(fieldPath, operator, text, caseFold) {
|
|
|
6024
6199
|
}
|
|
6025
6200
|
}
|
|
6026
6201
|
var ELEMENT_OPERATORS2 = ["matchesAny", "containsAll"];
|
|
6202
|
+
function requireExpressions(fieldPath, filter, type) {
|
|
6203
|
+
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
6204
|
+
const pattern = filter[operator];
|
|
6205
|
+
if (pattern === void 0) continue;
|
|
6206
|
+
if (typeof pattern !== "string")
|
|
6207
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
|
|
6208
|
+
requirePatternField(fieldPath, operator, type);
|
|
6209
|
+
requireTextLength(fieldPath, operator, pattern);
|
|
6210
|
+
requirePatternSearch(fieldPath).checkExpression(
|
|
6211
|
+
operator,
|
|
6212
|
+
pattern,
|
|
6213
|
+
filter.caseFold === true,
|
|
6214
|
+
fieldPath,
|
|
6215
|
+
PATTERN_HOST
|
|
6216
|
+
);
|
|
6217
|
+
}
|
|
6218
|
+
}
|
|
6027
6219
|
function requirePatternOperands(fieldPath, filter, type) {
|
|
6028
6220
|
if (filter.contains !== void 0) {
|
|
6029
6221
|
if (typeof filter.contains !== "string") {
|
|
@@ -6037,6 +6229,7 @@ function requirePatternOperands(fieldPath, filter, type) {
|
|
|
6037
6229
|
}
|
|
6038
6230
|
requirePatternField(fieldPath, "caseFold", type);
|
|
6039
6231
|
}
|
|
6232
|
+
requireExpressions(fieldPath, filter, type);
|
|
6040
6233
|
if (parsedTypeOf(type)?.pattern !== true) return;
|
|
6041
6234
|
const caseFold = filter.caseFold === true;
|
|
6042
6235
|
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
@@ -7016,6 +7209,67 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
7016
7209
|
};
|
|
7017
7210
|
}
|
|
7018
7211
|
|
|
7212
|
+
// src/core/scorer.ts
|
|
7213
|
+
var DEFAULT_K1 = 1.2;
|
|
7214
|
+
var DEFAULT_B = 0.75;
|
|
7215
|
+
function computeIDF(docFrequency, totalDocs) {
|
|
7216
|
+
if (totalDocs === 0) return 0;
|
|
7217
|
+
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
7218
|
+
}
|
|
7219
|
+
function bm25PruningSound(params) {
|
|
7220
|
+
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7221
|
+
const b = params?.b ?? DEFAULT_B;
|
|
7222
|
+
return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
|
|
7223
|
+
}
|
|
7224
|
+
function resolveBM25Params(params) {
|
|
7225
|
+
return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
|
|
7226
|
+
}
|
|
7227
|
+
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
7228
|
+
if (avgFieldLength === 0) return 0;
|
|
7229
|
+
const numerator = termFrequency * (k1 + 1);
|
|
7230
|
+
const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
|
|
7231
|
+
if (denominator === 0) return 0;
|
|
7232
|
+
return idf * (numerator / denominator);
|
|
7233
|
+
}
|
|
7234
|
+
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
7235
|
+
if (totalDocs === 0) return 0;
|
|
7236
|
+
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7237
|
+
const b = params?.b ?? DEFAULT_B;
|
|
7238
|
+
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
7239
|
+
}
|
|
7240
|
+
|
|
7241
|
+
// src/core/partition/field-scoring.ts
|
|
7242
|
+
function loadFieldScoring(state, fields, boost, averageFieldLengths) {
|
|
7243
|
+
const fieldNames = state.fieldNameTable.names;
|
|
7244
|
+
const scoring = {
|
|
7245
|
+
searchable: new Uint8Array(fieldNames.length),
|
|
7246
|
+
boosts: new Float64Array(fieldNames.length),
|
|
7247
|
+
averageLengths: new Float64Array(fieldNames.length),
|
|
7248
|
+
lengthColumns: new Array(fieldNames.length).fill(null)
|
|
7249
|
+
};
|
|
7250
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
7251
|
+
const fieldName = fieldNames[fieldIndex];
|
|
7252
|
+
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
7253
|
+
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
7254
|
+
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
7255
|
+
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
7256
|
+
}
|
|
7257
|
+
return scoring;
|
|
7258
|
+
}
|
|
7259
|
+
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
7260
|
+
const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
|
|
7261
|
+
if (column === null || internalId >= column.length) return averageLength;
|
|
7262
|
+
const stored = column[internalId];
|
|
7263
|
+
return stored > 0 ? stored : averageLength;
|
|
7264
|
+
}
|
|
7265
|
+
function bestSearchable(searchable, values) {
|
|
7266
|
+
let best = 0;
|
|
7267
|
+
for (let index = 0; index < searchable.length; index++) {
|
|
7268
|
+
if (searchable[index] === 1 && values[index] > best) best = values[index];
|
|
7269
|
+
}
|
|
7270
|
+
return best;
|
|
7271
|
+
}
|
|
7272
|
+
|
|
7019
7273
|
// src/core/partition/posting-columns.ts
|
|
7020
7274
|
function postingColumns(list) {
|
|
7021
7275
|
const deletedDocs = list.deletedDocs;
|
|
@@ -7029,39 +7283,174 @@ function postingColumns(list) {
|
|
|
7029
7283
|
};
|
|
7030
7284
|
}
|
|
7031
7285
|
|
|
7286
|
+
// src/core/partition/scoring.ts
|
|
7287
|
+
var EMPTY_COMPONENTS = Object.freeze({});
|
|
7288
|
+
function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
|
|
7289
|
+
const existing = components.get(internalId);
|
|
7290
|
+
if (existing) {
|
|
7291
|
+
existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
|
|
7292
|
+
existing.fieldLengths[fieldName] = fieldLength;
|
|
7293
|
+
existing.idf[token] = idf;
|
|
7294
|
+
return;
|
|
7295
|
+
}
|
|
7296
|
+
components.set(internalId, {
|
|
7297
|
+
termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
|
|
7298
|
+
fieldLengths: { [fieldName]: fieldLength },
|
|
7299
|
+
idf: { [token]: idf }
|
|
7300
|
+
});
|
|
7301
|
+
}
|
|
7302
|
+
function mergeGroupComponents(components, internalId, contribution) {
|
|
7303
|
+
const existing = components.get(internalId);
|
|
7304
|
+
if (existing) {
|
|
7305
|
+
Object.assign(existing.termFrequencies, contribution.termFrequencies);
|
|
7306
|
+
Object.assign(existing.fieldLengths, contribution.fieldLengths);
|
|
7307
|
+
existing.idf[contribution.token] = contribution.idf;
|
|
7308
|
+
return;
|
|
7309
|
+
}
|
|
7310
|
+
components.set(internalId, {
|
|
7311
|
+
termFrequencies: contribution.termFrequencies,
|
|
7312
|
+
fieldLengths: contribution.fieldLengths,
|
|
7313
|
+
idf: { [contribution.token]: contribution.idf }
|
|
7314
|
+
});
|
|
7315
|
+
}
|
|
7316
|
+
|
|
7317
|
+
// src/core/partition/group-scoring.ts
|
|
7318
|
+
function partialFieldMarks(fieldNames, partialFields) {
|
|
7319
|
+
const marks = new Uint8Array(fieldNames.length);
|
|
7320
|
+
if (partialFields === void 0 || partialFields.length === 0) return marks;
|
|
7321
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
7322
|
+
if (partialFields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
|
|
7323
|
+
}
|
|
7324
|
+
return marks;
|
|
7325
|
+
}
|
|
7326
|
+
function termGroupOf(groups, token) {
|
|
7327
|
+
if (groups === void 0) return void 0;
|
|
7328
|
+
for (const group of groups) {
|
|
7329
|
+
if (group.token === token) return group;
|
|
7330
|
+
}
|
|
7331
|
+
return void 0;
|
|
7332
|
+
}
|
|
7333
|
+
function typoMatchesFor(index, token, group, exact, tolerance, prefixLength) {
|
|
7334
|
+
if (group !== void 0) {
|
|
7335
|
+
if (!group.typoMatches || exact || tolerance === 0) return [];
|
|
7336
|
+
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token);
|
|
7337
|
+
}
|
|
7338
|
+
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
7339
|
+
const postingList = index.lookup(token);
|
|
7340
|
+
return postingList ? [{ token, postingList }] : [];
|
|
7341
|
+
}
|
|
7342
|
+
function resolveGroupMatches(context, group) {
|
|
7343
|
+
let groupDocumentFrequency = 0;
|
|
7344
|
+
const found = [];
|
|
7345
|
+
for (const member of group.members) {
|
|
7346
|
+
const documents = context.globalFrequency(member.token) ?? context.index.documentFrequency(member.token);
|
|
7347
|
+
if (documents > groupDocumentFrequency) groupDocumentFrequency = documents;
|
|
7348
|
+
const postingList = context.index.lookup(member.token);
|
|
7349
|
+
if (postingList === void 0) continue;
|
|
7350
|
+
found.push({ token: member.token, factor: member.factor, partialFactor: member.partialFactor, postingList });
|
|
7351
|
+
}
|
|
7352
|
+
if (found.length === 0) return [];
|
|
7353
|
+
const idf = computeIDF(groupDocumentFrequency, context.totalDocs);
|
|
7354
|
+
return found.map((entry) => ({ ...entry, idf }));
|
|
7355
|
+
}
|
|
7356
|
+
function groupPostingCount(matches) {
|
|
7357
|
+
let total = 0;
|
|
7358
|
+
for (const match of matches) total += match.postingList.length;
|
|
7359
|
+
return total;
|
|
7360
|
+
}
|
|
7361
|
+
function computeGroupContributions(context, matches, collect) {
|
|
7362
|
+
const { fields, fieldNames, filterBitset, partialMarks, k1, b } = context;
|
|
7363
|
+
const scoresAreZero = context.totalDocs === 0;
|
|
7364
|
+
const best = /* @__PURE__ */ new Map();
|
|
7365
|
+
for (const match of matches) {
|
|
7366
|
+
const perTerm = /* @__PURE__ */ new Map();
|
|
7367
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
7368
|
+
match.postingList
|
|
7369
|
+
);
|
|
7370
|
+
for (let pi = 0; pi < count; pi++) {
|
|
7371
|
+
const internalId = docIds[pi];
|
|
7372
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7373
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
7374
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
7375
|
+
if (fields.searchable[fieldIndex] === 0) continue;
|
|
7376
|
+
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
7377
|
+
if (factor === 0) continue;
|
|
7378
|
+
const fieldName = fieldNames[fieldIndex];
|
|
7379
|
+
const termFrequency = termFrequencies[pi];
|
|
7380
|
+
const fieldBoost = fields.boosts[fieldIndex];
|
|
7381
|
+
const avgLen = fields.averageLengths[fieldIndex];
|
|
7382
|
+
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
7383
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * factor;
|
|
7384
|
+
const existing = perTerm.get(internalId);
|
|
7385
|
+
if (existing) {
|
|
7386
|
+
existing.score += termScore;
|
|
7387
|
+
if (collect) {
|
|
7388
|
+
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
7389
|
+
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
7390
|
+
}
|
|
7391
|
+
} else if (collect) {
|
|
7392
|
+
perTerm.set(internalId, {
|
|
7393
|
+
score: termScore,
|
|
7394
|
+
token: match.token,
|
|
7395
|
+
idf: match.idf,
|
|
7396
|
+
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
7397
|
+
fieldLengths: { [fieldName]: actualFieldLength }
|
|
7398
|
+
});
|
|
7399
|
+
} else {
|
|
7400
|
+
perTerm.set(internalId, {
|
|
7401
|
+
score: termScore,
|
|
7402
|
+
token: match.token,
|
|
7403
|
+
idf: match.idf,
|
|
7404
|
+
termFrequencies: EMPTY_COMPONENTS,
|
|
7405
|
+
fieldLengths: EMPTY_COMPONENTS
|
|
7406
|
+
});
|
|
7407
|
+
}
|
|
7408
|
+
}
|
|
7409
|
+
for (const [internalId, contribution] of perTerm) {
|
|
7410
|
+
const current = best.get(internalId);
|
|
7411
|
+
if (!current || contribution.score > current.score) {
|
|
7412
|
+
best.set(internalId, contribution);
|
|
7413
|
+
}
|
|
7414
|
+
}
|
|
7415
|
+
}
|
|
7416
|
+
return best;
|
|
7417
|
+
}
|
|
7418
|
+
|
|
7032
7419
|
// src/core/partition/matches.ts
|
|
7033
|
-
function markPostingList(matched, list,
|
|
7420
|
+
function markPostingList(matched, list, markedFields, filterBitset) {
|
|
7034
7421
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(list);
|
|
7035
7422
|
for (let pi = 0; pi < count; pi++) {
|
|
7036
7423
|
const internalId = docIds[pi];
|
|
7037
7424
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7038
7425
|
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
7039
|
-
if (
|
|
7426
|
+
if (markedFields[fieldNameIndices[pi]] === 0) continue;
|
|
7040
7427
|
bitsetSet(matched, internalId);
|
|
7041
7428
|
}
|
|
7042
7429
|
}
|
|
7430
|
+
function searchedFieldMarks(fieldNames, fields) {
|
|
7431
|
+
const marks = new Uint8Array(fieldNames.length);
|
|
7432
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
7433
|
+
if (fields === void 0 || fields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
|
|
7434
|
+
}
|
|
7435
|
+
return marks;
|
|
7436
|
+
}
|
|
7043
7437
|
function searchFulltextMatches(state, params) {
|
|
7044
|
-
const { queryTokens,
|
|
7438
|
+
const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
7045
7439
|
const fieldNames = state.fieldNameTable.names;
|
|
7046
7440
|
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
7441
|
+
const searched = searchedFieldMarks(fieldNames, fields);
|
|
7442
|
+
const partial = partialFieldMarks(fieldNames, params.partialFields);
|
|
7443
|
+
const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
|
|
7047
7444
|
for (const qt of queryTokens) {
|
|
7048
|
-
|
|
7049
|
-
|
|
7050
|
-
|
|
7051
|
-
if (seen.has(term)) continue;
|
|
7052
|
-
seen.add(term);
|
|
7053
|
-
const postingList = state.invertedIdx.lookup(term);
|
|
7054
|
-
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
7055
|
-
}
|
|
7056
|
-
continue;
|
|
7445
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
7446
|
+
for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
7447
|
+
markPostingList(matched, match.postingList, searched, filterBitset);
|
|
7057
7448
|
}
|
|
7058
|
-
if (
|
|
7059
|
-
|
|
7060
|
-
|
|
7061
|
-
continue;
|
|
7062
|
-
|
|
7063
|
-
for (const match of state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength)) {
|
|
7064
|
-
markPostingList(matched, match.postingList, fieldNames, fields, filterBitset);
|
|
7449
|
+
if (group === void 0) continue;
|
|
7450
|
+
for (const member of group.members) {
|
|
7451
|
+
const postingList = state.invertedIdx.lookup(member.token);
|
|
7452
|
+
if (postingList === void 0) continue;
|
|
7453
|
+
markPostingList(matched, postingList, member.factor > 0 ? searched : searchedPartial, filterBitset);
|
|
7065
7454
|
}
|
|
7066
7455
|
}
|
|
7067
7456
|
const docStore = state.docStore;
|
|
@@ -7149,6 +7538,7 @@ function estimatePartitionBytes(state) {
|
|
|
7149
7538
|
bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7150
7539
|
bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7151
7540
|
for (const index of state.patternIndexes.values()) bytes += index.bytes();
|
|
7541
|
+
bytes += state.partialWords.bytes();
|
|
7152
7542
|
return bytes + sortColumnBytes(state);
|
|
7153
7543
|
}
|
|
7154
7544
|
|
|
@@ -7280,6 +7670,7 @@ function mergeSegmentState(target, source) {
|
|
|
7280
7670
|
mergeFieldIndexes(target, source, idMapping);
|
|
7281
7671
|
mergeStatistics(target, source);
|
|
7282
7672
|
mergeSurfaceForms(target, source);
|
|
7673
|
+
target.partialWords.afterMerge(source.partialWords.spellings(), [...idMapping.values()]);
|
|
7283
7674
|
target.sortColumns = null;
|
|
7284
7675
|
target.scoreBuffer = null;
|
|
7285
7676
|
}
|
|
@@ -7339,70 +7730,10 @@ function rebuildTextIndex(state, schema, language, options) {
|
|
|
7339
7730
|
}
|
|
7340
7731
|
} finally {
|
|
7341
7732
|
state.invertedIdx.endBatch();
|
|
7733
|
+
state.partialWords.forgetSpellings();
|
|
7342
7734
|
}
|
|
7343
7735
|
}
|
|
7344
7736
|
|
|
7345
|
-
// src/core/scorer.ts
|
|
7346
|
-
var DEFAULT_K1 = 1.2;
|
|
7347
|
-
var DEFAULT_B = 0.75;
|
|
7348
|
-
function computeIDF(docFrequency, totalDocs) {
|
|
7349
|
-
if (totalDocs === 0) return 0;
|
|
7350
|
-
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
7351
|
-
}
|
|
7352
|
-
function bm25PruningSound(params) {
|
|
7353
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7354
|
-
const b = params?.b ?? DEFAULT_B;
|
|
7355
|
-
return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
|
|
7356
|
-
}
|
|
7357
|
-
function resolveBM25Params(params) {
|
|
7358
|
-
return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
|
|
7359
|
-
}
|
|
7360
|
-
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
7361
|
-
if (avgFieldLength === 0) return 0;
|
|
7362
|
-
const numerator = termFrequency * (k1 + 1);
|
|
7363
|
-
const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
|
|
7364
|
-
if (denominator === 0) return 0;
|
|
7365
|
-
return idf * (numerator / denominator);
|
|
7366
|
-
}
|
|
7367
|
-
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
7368
|
-
if (totalDocs === 0) return 0;
|
|
7369
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7370
|
-
const b = params?.b ?? DEFAULT_B;
|
|
7371
|
-
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
7372
|
-
}
|
|
7373
|
-
|
|
7374
|
-
// src/core/partition/field-scoring.ts
|
|
7375
|
-
function loadFieldScoring(state, fields, boost, averageFieldLengths) {
|
|
7376
|
-
const fieldNames = state.fieldNameTable.names;
|
|
7377
|
-
const scoring = {
|
|
7378
|
-
searchable: new Uint8Array(fieldNames.length),
|
|
7379
|
-
boosts: new Float64Array(fieldNames.length),
|
|
7380
|
-
averageLengths: new Float64Array(fieldNames.length),
|
|
7381
|
-
lengthColumns: new Array(fieldNames.length).fill(null)
|
|
7382
|
-
};
|
|
7383
|
-
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
7384
|
-
const fieldName = fieldNames[fieldIndex];
|
|
7385
|
-
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
7386
|
-
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
7387
|
-
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
7388
|
-
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
7389
|
-
}
|
|
7390
|
-
return scoring;
|
|
7391
|
-
}
|
|
7392
|
-
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
7393
|
-
const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
|
|
7394
|
-
if (column === null || internalId >= column.length) return averageLength;
|
|
7395
|
-
const stored = column[internalId];
|
|
7396
|
-
return stored > 0 ? stored : averageLength;
|
|
7397
|
-
}
|
|
7398
|
-
function bestSearchable(searchable, values) {
|
|
7399
|
-
let best = 0;
|
|
7400
|
-
for (let index = 0; index < searchable.length; index++) {
|
|
7401
|
-
if (searchable[index] === 1 && values[index] > best) best = values[index];
|
|
7402
|
-
}
|
|
7403
|
-
return best;
|
|
7404
|
-
}
|
|
7405
|
-
|
|
7406
7737
|
// src/core/partition/block-bounds.ts
|
|
7407
7738
|
var ABSENT_FIELD_LENGTH_BOUND = 0;
|
|
7408
7739
|
var boundsByList = /* @__PURE__ */ new WeakMap();
|
|
@@ -7498,37 +7829,6 @@ function blockBoundsFor(list, columns) {
|
|
|
7498
7829
|
return built;
|
|
7499
7830
|
}
|
|
7500
7831
|
|
|
7501
|
-
// src/core/partition/scoring.ts
|
|
7502
|
-
var EMPTY_COMPONENTS = Object.freeze({});
|
|
7503
|
-
function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
|
|
7504
|
-
const existing = components.get(internalId);
|
|
7505
|
-
if (existing) {
|
|
7506
|
-
existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
|
|
7507
|
-
existing.fieldLengths[fieldName] = fieldLength;
|
|
7508
|
-
existing.idf[token] = idf;
|
|
7509
|
-
return;
|
|
7510
|
-
}
|
|
7511
|
-
components.set(internalId, {
|
|
7512
|
-
termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
|
|
7513
|
-
fieldLengths: { [fieldName]: fieldLength },
|
|
7514
|
-
idf: { [token]: idf }
|
|
7515
|
-
});
|
|
7516
|
-
}
|
|
7517
|
-
function mergePrefixComponents(components, internalId, contribution) {
|
|
7518
|
-
const existing = components.get(internalId);
|
|
7519
|
-
if (existing) {
|
|
7520
|
-
Object.assign(existing.termFrequencies, contribution.termFrequencies);
|
|
7521
|
-
Object.assign(existing.fieldLengths, contribution.fieldLengths);
|
|
7522
|
-
existing.idf[contribution.token] = contribution.idf;
|
|
7523
|
-
return;
|
|
7524
|
-
}
|
|
7525
|
-
components.set(internalId, {
|
|
7526
|
-
termFrequencies: contribution.termFrequencies,
|
|
7527
|
-
fieldLengths: contribution.fieldLengths,
|
|
7528
|
-
idf: { [contribution.token]: contribution.idf }
|
|
7529
|
-
});
|
|
7530
|
-
}
|
|
7531
|
-
|
|
7532
7832
|
// src/core/partition/top-k-heap.ts
|
|
7533
7833
|
function candidateWorse(a, b) {
|
|
7534
7834
|
if (a.score !== b.score) return a.score < b.score;
|
|
@@ -7692,7 +7992,7 @@ function topKFromBuffer(buffer, k, resolver, components) {
|
|
|
7692
7992
|
// src/core/partition/multi-term-topk.ts
|
|
7693
7993
|
function prunableMultiTermLists(params, index) {
|
|
7694
7994
|
if (params.queryTokens.length < 2) return null;
|
|
7695
|
-
if (params.
|
|
7995
|
+
if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
|
|
7696
7996
|
if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
|
|
7697
7997
|
if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
|
|
7698
7998
|
if (params.collectComponents !== false) return null;
|
|
@@ -7864,87 +8164,6 @@ function multiTermTopK(request) {
|
|
|
7864
8164
|
return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
|
|
7865
8165
|
}
|
|
7866
8166
|
|
|
7867
|
-
// src/core/partition/prefix-scoring.ts
|
|
7868
|
-
function resolvePrefixMatches(context, token, expansionTerms) {
|
|
7869
|
-
const found = [];
|
|
7870
|
-
const seen = /* @__PURE__ */ new Set();
|
|
7871
|
-
for (const term of [token, ...expansionTerms]) {
|
|
7872
|
-
if (seen.has(term)) continue;
|
|
7873
|
-
seen.add(term);
|
|
7874
|
-
const postingList = context.index.lookup(term);
|
|
7875
|
-
if (!postingList) continue;
|
|
7876
|
-
found.push({ token: term, postingList, docFreq: context.documentFrequency(term, postingList) });
|
|
7877
|
-
}
|
|
7878
|
-
if (found.length === 0) return [];
|
|
7879
|
-
let blendedDf = 0;
|
|
7880
|
-
for (const f of found) {
|
|
7881
|
-
if (f.docFreq > blendedDf) blendedDf = f.docFreq;
|
|
7882
|
-
}
|
|
7883
|
-
const blendedIdf = computeIDF(blendedDf, context.totalDocs);
|
|
7884
|
-
return found.map((f) => ({
|
|
7885
|
-
token: f.token,
|
|
7886
|
-
factor: Math.min(1, token.length / f.token.length),
|
|
7887
|
-
postingList: f.postingList,
|
|
7888
|
-
docFreq: blendedDf,
|
|
7889
|
-
idf: blendedIdf
|
|
7890
|
-
}));
|
|
7891
|
-
}
|
|
7892
|
-
function computePrefixContributions(context, matches, collect) {
|
|
7893
|
-
const { fields, fieldNames, filterBitset, k1, b } = context;
|
|
7894
|
-
const scoresAreZero = context.totalDocs === 0;
|
|
7895
|
-
const best = /* @__PURE__ */ new Map();
|
|
7896
|
-
for (const match of matches) {
|
|
7897
|
-
const perTerm = /* @__PURE__ */ new Map();
|
|
7898
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
7899
|
-
match.postingList
|
|
7900
|
-
);
|
|
7901
|
-
for (let pi = 0; pi < count; pi++) {
|
|
7902
|
-
const internalId = docIds[pi];
|
|
7903
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7904
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
7905
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
7906
|
-
if (fields.searchable[fieldIndex] === 0) continue;
|
|
7907
|
-
const fieldName = fieldNames[fieldIndex];
|
|
7908
|
-
const termFrequency = termFrequencies[pi];
|
|
7909
|
-
const fieldBoost = fields.boosts[fieldIndex];
|
|
7910
|
-
const avgLen = fields.averageLengths[fieldIndex];
|
|
7911
|
-
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
7912
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * match.factor;
|
|
7913
|
-
const existing = perTerm.get(internalId);
|
|
7914
|
-
if (existing) {
|
|
7915
|
-
existing.score += termScore;
|
|
7916
|
-
if (collect) {
|
|
7917
|
-
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
7918
|
-
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
7919
|
-
}
|
|
7920
|
-
} else if (collect) {
|
|
7921
|
-
perTerm.set(internalId, {
|
|
7922
|
-
score: termScore,
|
|
7923
|
-
token: match.token,
|
|
7924
|
-
idf: match.idf,
|
|
7925
|
-
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
7926
|
-
fieldLengths: { [fieldName]: actualFieldLength }
|
|
7927
|
-
});
|
|
7928
|
-
} else {
|
|
7929
|
-
perTerm.set(internalId, {
|
|
7930
|
-
score: termScore,
|
|
7931
|
-
token: match.token,
|
|
7932
|
-
idf: match.idf,
|
|
7933
|
-
termFrequencies: EMPTY_COMPONENTS,
|
|
7934
|
-
fieldLengths: EMPTY_COMPONENTS
|
|
7935
|
-
});
|
|
7936
|
-
}
|
|
7937
|
-
}
|
|
7938
|
-
for (const [internalId, contribution] of perTerm) {
|
|
7939
|
-
const current = best.get(internalId);
|
|
7940
|
-
if (!current || contribution.score > current.score) {
|
|
7941
|
-
best.set(internalId, contribution);
|
|
7942
|
-
}
|
|
7943
|
-
}
|
|
7944
|
-
}
|
|
7945
|
-
return best;
|
|
7946
|
-
}
|
|
7947
|
-
|
|
7948
8167
|
// src/core/partition/single-term-topk.ts
|
|
7949
8168
|
function fieldsCoverEvery(fields, fieldNames) {
|
|
7950
8169
|
if (fields === void 0) return true;
|
|
@@ -7955,7 +8174,7 @@ function fieldsCoverEvery(fields, fieldNames) {
|
|
|
7955
8174
|
}
|
|
7956
8175
|
function prunableSingleTermList(params, index, fieldNames) {
|
|
7957
8176
|
if (params.queryTokens.length !== 1) return null;
|
|
7958
|
-
if (params.
|
|
8177
|
+
if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
|
|
7959
8178
|
if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
|
|
7960
8179
|
if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
|
|
7961
8180
|
if (params.collectComponents !== false) return null;
|
|
@@ -8065,15 +8284,11 @@ function singleTermTopK(request) {
|
|
|
8065
8284
|
function globalDocFreqFor(docFreqs, term, fallback) {
|
|
8066
8285
|
return Object.hasOwn(docFreqs, term) ? docFreqs[term] : fallback;
|
|
8067
8286
|
}
|
|
8068
|
-
function matchesFor(index, token, exact, tolerance, prefixLength) {
|
|
8069
|
-
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
8070
|
-
const postingList = index.lookup(token);
|
|
8071
|
-
return postingList ? [{ token, postingList }] : [];
|
|
8072
|
-
}
|
|
8073
8287
|
function searchFulltext(state, params) {
|
|
8074
8288
|
const {
|
|
8075
8289
|
queryTokens,
|
|
8076
|
-
|
|
8290
|
+
termGroups,
|
|
8291
|
+
partialFields,
|
|
8077
8292
|
fields,
|
|
8078
8293
|
boost,
|
|
8079
8294
|
tolerance = 0,
|
|
@@ -8105,19 +8320,75 @@ function searchFulltext(state, params) {
|
|
|
8105
8320
|
function documentFrequency(token, list) {
|
|
8106
8321
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
8107
8322
|
}
|
|
8108
|
-
const
|
|
8323
|
+
const groupContext = {
|
|
8109
8324
|
index: state.invertedIdx,
|
|
8110
8325
|
fields: fieldScoring,
|
|
8326
|
+
partialMarks: partialFieldMarks(fieldNames, partialFields),
|
|
8111
8327
|
fieldNames,
|
|
8112
8328
|
totalDocs,
|
|
8113
8329
|
k1,
|
|
8114
8330
|
b,
|
|
8115
8331
|
filterBitset,
|
|
8116
|
-
|
|
8332
|
+
globalFrequency(token) {
|
|
8333
|
+
if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
|
|
8334
|
+
return globalStats.docFrequencies[token];
|
|
8335
|
+
}
|
|
8117
8336
|
};
|
|
8118
|
-
function
|
|
8337
|
+
function mergeGroupContribution(internalId, contribution) {
|
|
8119
8338
|
addScore(scoreBuffer, internalId, contribution.score);
|
|
8120
|
-
if (components !== null)
|
|
8339
|
+
if (components !== null) mergeGroupComponents(components, internalId, contribution);
|
|
8340
|
+
}
|
|
8341
|
+
function resolveMatch(token, postingList) {
|
|
8342
|
+
const docFreq = documentFrequency(token, postingList);
|
|
8343
|
+
return { token, docFreq, idf: computeIDF(docFreq, totalDocs), postingList };
|
|
8344
|
+
}
|
|
8345
|
+
function scorePostings(match, onlyScored) {
|
|
8346
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
8347
|
+
match.postingList
|
|
8348
|
+
);
|
|
8349
|
+
if (!onlyScored && !hasDeleted && filterBitset === void 0 && components === null) {
|
|
8350
|
+
for (let pi = 0; pi < count; pi++) {
|
|
8351
|
+
const internalId = docIds[pi];
|
|
8352
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
8353
|
+
if (searchable[fieldIndex] === 0) continue;
|
|
8354
|
+
const avgLen = averageLengths[fieldIndex];
|
|
8355
|
+
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8356
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], match.idf, actualFieldLength, avgLen, k1, b);
|
|
8357
|
+
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8358
|
+
}
|
|
8359
|
+
return;
|
|
8360
|
+
}
|
|
8361
|
+
for (let pi = 0; pi < count; pi++) {
|
|
8362
|
+
const internalId = docIds[pi];
|
|
8363
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8364
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
8365
|
+
if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
|
|
8366
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
8367
|
+
if (searchable[fieldIndex] === 0) continue;
|
|
8368
|
+
const termFrequency = termFrequencies[pi];
|
|
8369
|
+
const avgLen = averageLengths[fieldIndex];
|
|
8370
|
+
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8371
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
|
|
8372
|
+
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8373
|
+
if (components !== null) {
|
|
8374
|
+
recordComponents(
|
|
8375
|
+
components,
|
|
8376
|
+
internalId,
|
|
8377
|
+
fieldNames[fieldIndex],
|
|
8378
|
+
match.token,
|
|
8379
|
+
termFrequency,
|
|
8380
|
+
actualFieldLength,
|
|
8381
|
+
match.idf
|
|
8382
|
+
);
|
|
8383
|
+
}
|
|
8384
|
+
}
|
|
8385
|
+
}
|
|
8386
|
+
function scoreGroup(group, onlyScored) {
|
|
8387
|
+
if (group === null) return;
|
|
8388
|
+
for (const [internalId, contribution] of computeGroupContributions(groupContext, group, collectComponents)) {
|
|
8389
|
+
if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
|
|
8390
|
+
mergeGroupContribution(internalId, contribution);
|
|
8391
|
+
}
|
|
8121
8392
|
}
|
|
8122
8393
|
const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
|
|
8123
8394
|
if (prunableList !== null && maxResults !== void 0) {
|
|
@@ -8147,127 +8418,30 @@ function searchFulltext(state, params) {
|
|
|
8147
8418
|
}
|
|
8148
8419
|
if (useIntersection) {
|
|
8149
8420
|
const resolved = [];
|
|
8150
|
-
let prefixMatches = [];
|
|
8151
8421
|
for (const qt of queryTokens) {
|
|
8152
|
-
|
|
8153
|
-
|
|
8154
|
-
|
|
8155
|
-
for (const m of prefixMatches) {
|
|
8156
|
-
totalPostings2 += m.postingList.length;
|
|
8157
|
-
}
|
|
8158
|
-
resolved.push({ token: qt.token, matches: [], totalPostings: totalPostings2, isPrefix: true });
|
|
8159
|
-
continue;
|
|
8160
|
-
}
|
|
8161
|
-
const rawMatches = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
|
|
8162
|
-
let totalPostings = 0;
|
|
8422
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
8423
|
+
const groupMatches = group === void 0 ? null : resolveGroupMatches(groupContext, group);
|
|
8424
|
+
let totalPostings = groupMatches === null ? 0 : groupPostingCount(groupMatches);
|
|
8163
8425
|
const matches = [];
|
|
8164
|
-
for (const m of
|
|
8165
|
-
const docFreq = documentFrequency(m.token, m.postingList);
|
|
8166
|
-
const idf = computeIDF(docFreq, totalDocs);
|
|
8426
|
+
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
8167
8427
|
totalPostings += m.postingList.length;
|
|
8168
|
-
matches.push(
|
|
8428
|
+
matches.push(resolveMatch(m.token, m.postingList));
|
|
8169
8429
|
}
|
|
8170
|
-
resolved.push({ token: qt.token, matches, totalPostings });
|
|
8430
|
+
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
8171
8431
|
}
|
|
8172
8432
|
resolved.sort((a, b2) => a.totalPostings - b2.totalPostings);
|
|
8173
8433
|
for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
|
|
8174
|
-
|
|
8175
|
-
|
|
8176
|
-
|
|
8177
|
-
if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
|
|
8178
|
-
mergePrefixContribution(internalId, contribution);
|
|
8179
|
-
}
|
|
8180
|
-
continue;
|
|
8181
|
-
}
|
|
8182
|
-
for (const match of resolved[tokenIndex].matches) {
|
|
8183
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
8184
|
-
match.postingList
|
|
8185
|
-
);
|
|
8186
|
-
for (let pi = 0; pi < count; pi++) {
|
|
8187
|
-
const internalId = docIds[pi];
|
|
8188
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8189
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
8190
|
-
if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
|
|
8191
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
8192
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
8193
|
-
const termFrequency = termFrequencies[pi];
|
|
8194
|
-
const fieldBoost = boosts[fieldIndex];
|
|
8195
|
-
const avgLen = averageLengths[fieldIndex];
|
|
8196
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8197
|
-
let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
|
|
8198
|
-
termScore *= fieldBoost;
|
|
8199
|
-
addScore(scoreBuffer, internalId, termScore);
|
|
8200
|
-
if (components !== null) {
|
|
8201
|
-
recordComponents(
|
|
8202
|
-
components,
|
|
8203
|
-
internalId,
|
|
8204
|
-
fieldNames[fieldIndex],
|
|
8205
|
-
match.token,
|
|
8206
|
-
termFrequency,
|
|
8207
|
-
actualFieldLength,
|
|
8208
|
-
match.idf
|
|
8209
|
-
);
|
|
8210
|
-
}
|
|
8211
|
-
}
|
|
8212
|
-
}
|
|
8434
|
+
const onlyScored = tokenIndex > 0;
|
|
8435
|
+
for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
|
|
8436
|
+
scoreGroup(resolved[tokenIndex].group, onlyScored);
|
|
8213
8437
|
}
|
|
8214
8438
|
} else {
|
|
8215
8439
|
for (const qt of queryTokens) {
|
|
8216
|
-
|
|
8217
|
-
|
|
8218
|
-
|
|
8219
|
-
resolvePrefixMatches(prefixContext, qt.token, prefixExpansion.terms),
|
|
8220
|
-
collectComponents
|
|
8221
|
-
);
|
|
8222
|
-
for (const [internalId, contribution] of contributions) {
|
|
8223
|
-
mergePrefixContribution(internalId, contribution);
|
|
8224
|
-
}
|
|
8225
|
-
continue;
|
|
8226
|
-
}
|
|
8227
|
-
const matchingPostings = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
|
|
8228
|
-
for (const match of matchingPostings) {
|
|
8229
|
-
const idf = computeIDF(documentFrequency(match.token, match.postingList), totalDocs);
|
|
8230
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
8231
|
-
match.postingList
|
|
8232
|
-
);
|
|
8233
|
-
if (!hasDeleted && filterBitset === void 0 && components === null) {
|
|
8234
|
-
for (let pi = 0; pi < count; pi++) {
|
|
8235
|
-
const internalId = docIds[pi];
|
|
8236
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
8237
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
8238
|
-
const avgLen = averageLengths[fieldIndex];
|
|
8239
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8240
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], idf, actualFieldLength, avgLen, k1, b);
|
|
8241
|
-
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8242
|
-
}
|
|
8243
|
-
continue;
|
|
8244
|
-
}
|
|
8245
|
-
for (let pi = 0; pi < count; pi++) {
|
|
8246
|
-
const internalId = docIds[pi];
|
|
8247
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8248
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
8249
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
8250
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
8251
|
-
const termFrequency = termFrequencies[pi];
|
|
8252
|
-
const fieldBoost = boosts[fieldIndex];
|
|
8253
|
-
const avgLen = averageLengths[fieldIndex];
|
|
8254
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8255
|
-
let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, idf, actualFieldLength, avgLen, k1, b);
|
|
8256
|
-
termScore *= fieldBoost;
|
|
8257
|
-
addScore(scoreBuffer, internalId, termScore);
|
|
8258
|
-
if (components !== null) {
|
|
8259
|
-
recordComponents(
|
|
8260
|
-
components,
|
|
8261
|
-
internalId,
|
|
8262
|
-
fieldNames[fieldIndex],
|
|
8263
|
-
match.token,
|
|
8264
|
-
termFrequency,
|
|
8265
|
-
actualFieldLength,
|
|
8266
|
-
idf
|
|
8267
|
-
);
|
|
8268
|
-
}
|
|
8269
|
-
}
|
|
8440
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
8441
|
+
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
8442
|
+
scorePostings(resolveMatch(m.token, m.postingList), false);
|
|
8270
8443
|
}
|
|
8444
|
+
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
|
|
8271
8445
|
}
|
|
8272
8446
|
}
|
|
8273
8447
|
const totalMatched = scoreBuffer.touchedCount;
|
|
@@ -8476,6 +8650,7 @@ function encodeSegmentState(state) {
|
|
|
8476
8650
|
totalFieldLengths: fieldLengths.totals,
|
|
8477
8651
|
docFrequencies,
|
|
8478
8652
|
surfaceForms: state.surfaceRegistry.size() === 0 ? null : state.surfaceRegistry.serialize(),
|
|
8653
|
+
partialSpellings: state.partialWords.spellings(),
|
|
8479
8654
|
...encodeFieldIndexes(state, remap)
|
|
8480
8655
|
};
|
|
8481
8656
|
}
|
|
@@ -8594,6 +8769,8 @@ function mergeSegmentPayload(target, payload, documents) {
|
|
|
8594
8769
|
target.surfaceRegistry.add(surface, value[1], value[0]);
|
|
8595
8770
|
}
|
|
8596
8771
|
}
|
|
8772
|
+
const mergedIds = Array.from({ length: payload.docIds.length }, (_, at) => ordinalBase + at);
|
|
8773
|
+
target.partialWords.afterMerge(payload.partialSpellings, mergedIds);
|
|
8597
8774
|
target.sortColumns = null;
|
|
8598
8775
|
target.scoreBuffer = null;
|
|
8599
8776
|
}
|
|
@@ -8883,6 +9060,7 @@ function deserializePartition(state, data, clearFn, schema) {
|
|
|
8883
9060
|
}
|
|
8884
9061
|
loadPatternIndexes(state, data.fieldIndexes.pattern, schema);
|
|
8885
9062
|
rebuildSizeIndexes(state, schema);
|
|
9063
|
+
state.partialWords.forgetSpellings();
|
|
8886
9064
|
}
|
|
8887
9065
|
|
|
8888
9066
|
// src/core/partition/suggestions.ts
|
|
@@ -8928,27 +9106,33 @@ function suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit) {
|
|
|
8928
9106
|
if (results.length > limit) results.length = limit;
|
|
8929
9107
|
return results;
|
|
8930
9108
|
}
|
|
8931
|
-
function
|
|
9109
|
+
function partitionDocumentFrequency(subs, token) {
|
|
9110
|
+
let documents = 0;
|
|
9111
|
+
for (const sub of subs) documents += sub.invertedIdx.documentFrequency(token);
|
|
9112
|
+
return documents;
|
|
9113
|
+
}
|
|
9114
|
+
function expandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
|
|
8932
9115
|
if (maxExpansions <= 0 || surfacePrefix.length === 0) return [];
|
|
8933
|
-
const
|
|
8934
|
-
|
|
8935
|
-
|
|
8936
|
-
|
|
8937
|
-
|
|
8938
|
-
|
|
8939
|
-
|
|
8940
|
-
|
|
8941
|
-
|
|
8942
|
-
|
|
8943
|
-
if (dfByToken.has(suggestion.term)) continue;
|
|
8944
|
-
if (verbatimOccurrences(state, suggestion.term) === 0) continue;
|
|
8945
|
-
dfByToken.set(suggestion.term, suggestion.documentFrequency);
|
|
9116
|
+
const candidates = /* @__PURE__ */ new Set();
|
|
9117
|
+
const holdsSurfaceForms = subs.some((sub) => sub.surfaceRegistry.size() > 0);
|
|
9118
|
+
const termPrefixes = !holdsSurfaceForms && stemmedToken !== surfacePrefix && stemmedToken.length > 0 ? [surfacePrefix, stemmedToken] : [surfacePrefix];
|
|
9119
|
+
for (const sub of subs) {
|
|
9120
|
+
for (const candidate of sub.surfaceRegistry.candidatesForPrefix(surfacePrefix)) candidates.add(candidate.token);
|
|
9121
|
+
for (const prefix of termPrefixes) {
|
|
9122
|
+
for (const suggestion of sub.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
9123
|
+
if (candidates.has(suggestion.term)) continue;
|
|
9124
|
+
if (verbatimOccurrences(sub, suggestion.term) > 0) candidates.add(suggestion.term);
|
|
9125
|
+
}
|
|
8946
9126
|
}
|
|
8947
9127
|
}
|
|
8948
|
-
const
|
|
8949
|
-
|
|
8950
|
-
|
|
8951
|
-
|
|
9128
|
+
const ranked = [];
|
|
9129
|
+
for (const token of candidates) {
|
|
9130
|
+
const documents = partitionDocumentFrequency(subs, token);
|
|
9131
|
+
if (documents > 0) ranked.push({ token, documents });
|
|
9132
|
+
}
|
|
9133
|
+
ranked.sort((a, b) => b.documents - a.documents || compareCodePoints(a.token, b.token));
|
|
9134
|
+
if (ranked.length > maxExpansions) ranked.length = maxExpansions;
|
|
9135
|
+
return ranked.map((entry) => entry.token);
|
|
8952
9136
|
}
|
|
8953
9137
|
|
|
8954
9138
|
// src/core/partition/index.ts
|
|
@@ -8965,9 +9149,10 @@ function partitionStateOf(partition) {
|
|
|
8965
9149
|
}
|
|
8966
9150
|
function createPartitionIndex(partitionId, trackPositions = true) {
|
|
8967
9151
|
const fieldNameTable = { names: [], indexMap: /* @__PURE__ */ new Map() };
|
|
9152
|
+
const docStore = createDocumentStore();
|
|
8968
9153
|
const state = {
|
|
8969
9154
|
invertedIdx: createInvertedIndex(fieldNameTable),
|
|
8970
|
-
docStore
|
|
9155
|
+
docStore,
|
|
8971
9156
|
stats: createPartitionStats(),
|
|
8972
9157
|
surfaceRegistry: createSurfaceRegistry(),
|
|
8973
9158
|
numericIndexes: /* @__PURE__ */ new Map(),
|
|
@@ -8977,6 +9162,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8977
9162
|
patternIndexes: /* @__PURE__ */ new Map(),
|
|
8978
9163
|
sizeIndexes: /* @__PURE__ */ new Map(),
|
|
8979
9164
|
fieldNameTable,
|
|
9165
|
+
partialWords: createGrowingPartialWords({ docStore }),
|
|
8980
9166
|
flatSchemaCache: null,
|
|
8981
9167
|
lastSchemaRef: null,
|
|
8982
9168
|
trackPositions,
|
|
@@ -8997,6 +9183,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8997
9183
|
state.geoIndexes.clear();
|
|
8998
9184
|
state.patternIndexes.clear();
|
|
8999
9185
|
state.sizeIndexes.clear();
|
|
9186
|
+
state.partialWords.clear();
|
|
9000
9187
|
state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
|
|
9001
9188
|
state.flatSchemaCache = null;
|
|
9002
9189
|
state.lastSchemaRef = null;
|
|
@@ -9025,7 +9212,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9025
9212
|
}
|
|
9026
9213
|
}
|
|
9027
9214
|
const internalId = state.docStore.ensureInternalId(docId);
|
|
9028
|
-
const { fieldLengths, tokensByField } = indexDocument(
|
|
9215
|
+
const { fieldLengths, tokensByField, partialSpellings } = indexDocument(
|
|
9029
9216
|
state,
|
|
9030
9217
|
docId,
|
|
9031
9218
|
document2,
|
|
@@ -9039,6 +9226,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9039
9226
|
state.docStore.store(docId, document2, fieldLengths);
|
|
9040
9227
|
}
|
|
9041
9228
|
state.stats.addDocument(fieldLengths, tokensByField);
|
|
9229
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, 1);
|
|
9042
9230
|
recordSortValues(state, internalId, document2);
|
|
9043
9231
|
},
|
|
9044
9232
|
remove(docId, schema, language, options) {
|
|
@@ -9049,7 +9237,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9049
9237
|
partitionId
|
|
9050
9238
|
});
|
|
9051
9239
|
}
|
|
9052
|
-
const { fieldLengths, tokensByField } = removeFromIndexes(
|
|
9240
|
+
const { fieldLengths, tokensByField, partialSpellings } = removeFromIndexes(
|
|
9053
9241
|
state,
|
|
9054
9242
|
docId,
|
|
9055
9243
|
stored,
|
|
@@ -9060,6 +9248,9 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9060
9248
|
const internalId = state.docStore.getInternalId(docId);
|
|
9061
9249
|
state.docStore.remove(docId);
|
|
9062
9250
|
state.stats.removeDocument(fieldLengths, tokensByField);
|
|
9251
|
+
if (internalId !== void 0) {
|
|
9252
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, -1);
|
|
9253
|
+
}
|
|
9063
9254
|
forgetSortValues(state, internalId);
|
|
9064
9255
|
},
|
|
9065
9256
|
beginBatch() {
|
|
@@ -9095,29 +9286,19 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9095
9286
|
const fields = schemaFieldsOf(schema);
|
|
9096
9287
|
const needsTextReindex = textFieldsChanged(stored.fields, document2, fields);
|
|
9097
9288
|
if (needsTextReindex) {
|
|
9098
|
-
const
|
|
9099
|
-
|
|
9100
|
-
docId,
|
|
9101
|
-
stored,
|
|
9102
|
-
fields,
|
|
9103
|
-
language,
|
|
9104
|
-
options
|
|
9105
|
-
);
|
|
9106
|
-
state.stats.removeDocument(oldFieldLengths, oldTokens);
|
|
9289
|
+
const previous = removeFromIndexes(state, docId, stored, fields, language, options);
|
|
9290
|
+
state.stats.removeDocument(previous.fieldLengths, previous.tokensByField);
|
|
9107
9291
|
const previousInternalId = state.docStore.getInternalId(docId);
|
|
9292
|
+
if (previousInternalId !== void 0) {
|
|
9293
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), previousInternalId, previous.partialSpellings, -1);
|
|
9294
|
+
}
|
|
9108
9295
|
state.docStore.remove(docId);
|
|
9109
9296
|
forgetSortValues(state, previousInternalId);
|
|
9110
9297
|
const internalId = state.docStore.ensureInternalId(docId);
|
|
9111
|
-
const
|
|
9112
|
-
|
|
9113
|
-
|
|
9114
|
-
|
|
9115
|
-
fields,
|
|
9116
|
-
language,
|
|
9117
|
-
options
|
|
9118
|
-
);
|
|
9119
|
-
state.docStore.store(docId, document2, newFieldLengths);
|
|
9120
|
-
state.stats.addDocument(newFieldLengths, newTokens);
|
|
9298
|
+
const next = indexDocument(state, docId, document2, fields, language, options);
|
|
9299
|
+
state.docStore.store(docId, document2, next.fieldLengths);
|
|
9300
|
+
state.stats.addDocument(next.fieldLengths, next.tokensByField);
|
|
9301
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, next.partialSpellings, 1);
|
|
9121
9302
|
recordSortValues(state, internalId, document2);
|
|
9122
9303
|
} else {
|
|
9123
9304
|
updateFieldIndexOnly(state, docId, stored.fields, document2, fields);
|
|
@@ -9188,7 +9369,13 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9188
9369
|
return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
|
|
9189
9370
|
},
|
|
9190
9371
|
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
9191
|
-
return expandTermPrefix(state, surfacePrefix, stemmedToken, maxExpansions);
|
|
9372
|
+
return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
|
|
9373
|
+
},
|
|
9374
|
+
expandPartialWords(analysis, requests) {
|
|
9375
|
+
return expandPartialWords([state], analysis, requests);
|
|
9376
|
+
},
|
|
9377
|
+
preparePartialWords(analysis) {
|
|
9378
|
+
expandPartialWords([state], analysis, []);
|
|
9192
9379
|
},
|
|
9193
9380
|
serialize(indexName, totalPartitions, language, schema) {
|
|
9194
9381
|
return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
|
|
@@ -9939,6 +10126,8 @@ var MAX_SORT_FIELDS = 8;
|
|
|
9939
10126
|
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
9940
10127
|
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
9941
10128
|
var PREFIX_MAX_EXPANSIONS = 50;
|
|
10129
|
+
var PARTIAL_WORDS_ADDED_PER_QUERY = 50;
|
|
10130
|
+
var PARTIAL_WORD_MIN_CODE_POINTS = 3;
|
|
9942
10131
|
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
9943
10132
|
var MAX_TOLERANCE = 10;
|
|
9944
10133
|
var MAX_PREFIX_LENGTH = 1024;
|
|
@@ -10256,36 +10445,120 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
|
|
|
10256
10445
|
return { paginated: sliced, nextCursor };
|
|
10257
10446
|
}
|
|
10258
10447
|
|
|
10259
|
-
// src/search/
|
|
10260
|
-
function
|
|
10261
|
-
const term = params.term;
|
|
10262
|
-
if (term === void 0) return void 0;
|
|
10448
|
+
// src/search/term-groups.ts
|
|
10449
|
+
function writtenWordsByPosition(term, language, analysis) {
|
|
10263
10450
|
const unstemmed = tokenize(term, language, {
|
|
10264
10451
|
stem: false,
|
|
10265
10452
|
removeStopWords: true,
|
|
10266
|
-
customTokenizer:
|
|
10267
|
-
stopWordOverride:
|
|
10453
|
+
customTokenizer: analysis?.customTokenizer,
|
|
10454
|
+
stopWordOverride: analysis?.stopWords
|
|
10268
10455
|
});
|
|
10269
|
-
|
|
10270
|
-
for (const
|
|
10271
|
-
if (
|
|
10272
|
-
|
|
10273
|
-
|
|
10456
|
+
const written = /* @__PURE__ */ new Map();
|
|
10457
|
+
for (const token of unstemmed.tokens) {
|
|
10458
|
+
if (!written.has(token.position)) written.set(token.position, token.token);
|
|
10459
|
+
}
|
|
10460
|
+
return written;
|
|
10461
|
+
}
|
|
10462
|
+
function groupFor(groups, token, typoMatches) {
|
|
10463
|
+
let group = groups.get(token);
|
|
10464
|
+
if (group === void 0) {
|
|
10465
|
+
group = { token, members: [{ token, factor: 1, partialFactor: 1 }], typoMatches };
|
|
10466
|
+
groups.set(token, group);
|
|
10467
|
+
}
|
|
10468
|
+
return group;
|
|
10469
|
+
}
|
|
10470
|
+
function addCompletions(groups, partition, lastToken, typedPrefix) {
|
|
10471
|
+
if (typedPrefix === void 0 || typedPrefix.length === 0) return;
|
|
10472
|
+
const group = groupFor(groups, lastToken.token, false);
|
|
10473
|
+
const ownLength = codePointCount(lastToken.token);
|
|
10474
|
+
for (const completion of partition.expandTermPrefix(typedPrefix, lastToken.token, PREFIX_MAX_EXPANSIONS)) {
|
|
10475
|
+
if (completion === lastToken.token) continue;
|
|
10476
|
+
const factor = Math.min(1, ownLength / codePointCount(completion));
|
|
10477
|
+
group.members.push({ token: completion, factor, partialFactor: factor });
|
|
10478
|
+
}
|
|
10479
|
+
}
|
|
10480
|
+
function addPartialWords(group, added) {
|
|
10481
|
+
const membersByToken = /* @__PURE__ */ new Map();
|
|
10482
|
+
for (const member of group.members) membersByToken.set(member.token, member);
|
|
10483
|
+
for (const word of added) {
|
|
10484
|
+
const member = membersByToken.get(word.token);
|
|
10485
|
+
if (member === void 0) {
|
|
10486
|
+
group.members.push({ token: word.token, factor: 0, partialFactor: word.factor });
|
|
10487
|
+
} else if (word.factor > member.partialFactor) {
|
|
10488
|
+
member.partialFactor = word.factor;
|
|
10274
10489
|
}
|
|
10275
10490
|
}
|
|
10276
|
-
|
|
10277
|
-
|
|
10278
|
-
|
|
10279
|
-
|
|
10280
|
-
|
|
10491
|
+
}
|
|
10492
|
+
function partialWordAnalysisOf(fields, language, analysis) {
|
|
10493
|
+
return {
|
|
10494
|
+
fields,
|
|
10495
|
+
language,
|
|
10496
|
+
options: {
|
|
10497
|
+
stopWordOverride: analysis?.stopWords,
|
|
10498
|
+
customTokenizer: analysis?.customTokenizer,
|
|
10499
|
+
collectSurfaces: true
|
|
10500
|
+
}
|
|
10501
|
+
};
|
|
10502
|
+
}
|
|
10503
|
+
function searchesAPartialField(params, partialFields) {
|
|
10504
|
+
if (params.fields === void 0) return true;
|
|
10505
|
+
for (const field of params.fields) {
|
|
10506
|
+
if (partialFields.includes(field)) return true;
|
|
10281
10507
|
}
|
|
10282
|
-
|
|
10283
|
-
|
|
10284
|
-
|
|
10508
|
+
return false;
|
|
10509
|
+
}
|
|
10510
|
+
function partialWordRequests(params, queryTokens, written, prefixToken) {
|
|
10511
|
+
const eligible = [];
|
|
10512
|
+
for (const queryToken of queryTokens) {
|
|
10513
|
+
const word = written.get(queryToken.position);
|
|
10514
|
+
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
10515
|
+
eligible.push({ token: queryToken.token, written: word });
|
|
10516
|
+
if (eligible.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
10517
|
+
}
|
|
10518
|
+
const count = eligible.length;
|
|
10519
|
+
if (count === 0) return [];
|
|
10520
|
+
const evenShare = Math.floor(PARTIAL_WORDS_ADDED_PER_QUERY / count);
|
|
10521
|
+
const remainder = PARTIAL_WORDS_ADDED_PER_QUERY % count;
|
|
10522
|
+
return eligible.map((entry, order) => ({
|
|
10523
|
+
token: entry.token,
|
|
10524
|
+
request: {
|
|
10525
|
+
written: entry.written,
|
|
10526
|
+
ownToken: entry.token,
|
|
10527
|
+
share: evenShare + (order < remainder ? 1 : 0),
|
|
10528
|
+
tolerance: entry.token === prefixToken ? 0 : params.tolerance ?? 0,
|
|
10529
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
10530
|
+
}
|
|
10531
|
+
}));
|
|
10532
|
+
}
|
|
10533
|
+
function resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis) {
|
|
10534
|
+
const term = params.term;
|
|
10535
|
+
if (term === void 0 || params.exact === true) return {};
|
|
10536
|
+
const usesPrefix = params.prefix === true;
|
|
10537
|
+
const partialFields = params.partialWords === true ? partialFieldsOf(schema) : [];
|
|
10538
|
+
const usesPartialWords = partialFields.length > 0 && searchesAPartialField(params, partialFields);
|
|
10539
|
+
if (!usesPrefix && !usesPartialWords) return {};
|
|
10540
|
+
const written = writtenWordsByPosition(term, language, analysis);
|
|
10541
|
+
const groups = /* @__PURE__ */ new Map();
|
|
10542
|
+
if (usesPrefix) addCompletions(groups, partition, lastToken, written.get(lastToken.position));
|
|
10543
|
+
const prefixToken = groups.has(lastToken.token) ? lastToken.token : void 0;
|
|
10544
|
+
if (usesPartialWords) {
|
|
10545
|
+
const requests = partialWordRequests(params, queryTokens, written, prefixToken);
|
|
10546
|
+
const added = partition.expandPartialWords(
|
|
10547
|
+
partialWordAnalysisOf(partialFields, language, analysis),
|
|
10548
|
+
requests.map((entry) => entry.request)
|
|
10549
|
+
);
|
|
10550
|
+
for (let at = 0; at < requests.length; at++) {
|
|
10551
|
+
if (added[at].length === 0) continue;
|
|
10552
|
+
const token = requests[at].token;
|
|
10553
|
+
addPartialWords(groupFor(groups, token, token !== prefixToken), added[at]);
|
|
10285
10554
|
}
|
|
10286
10555
|
}
|
|
10287
|
-
|
|
10556
|
+
const termGroups = [...groups.values()];
|
|
10557
|
+
if (termGroups.length === 0) return {};
|
|
10558
|
+
return usesPartialWords ? { termGroups, partialFields: [...partialFields] } : { termGroups };
|
|
10288
10559
|
}
|
|
10560
|
+
|
|
10561
|
+
// src/search/fulltext.ts
|
|
10289
10562
|
function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
10290
10563
|
if (!params.term || params.term.trim().length === 0) return null;
|
|
10291
10564
|
if (params.fields && params.fields.length === 0) return null;
|
|
@@ -10301,11 +10574,6 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
10301
10574
|
});
|
|
10302
10575
|
if (queryTokenResult.tokens.length === 0) return null;
|
|
10303
10576
|
const queryTokens = deduplicateTokens(queryTokenResult.tokens);
|
|
10304
|
-
let prefixExpansion;
|
|
10305
|
-
if (params.prefix === true && params.exact !== true) {
|
|
10306
|
-
const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
|
|
10307
|
-
prefixExpansion = resolvePrefixExpansion(partition, params, language, options, queryTokens, lastToken);
|
|
10308
|
-
}
|
|
10309
10577
|
let filterBitset;
|
|
10310
10578
|
if (params.filters) {
|
|
10311
10579
|
filterBitset = partition.applyFiltersBitset(
|
|
@@ -10315,14 +10583,17 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
10315
10583
|
);
|
|
10316
10584
|
if (bitsetIsEmpty(filterBitset)) return null;
|
|
10317
10585
|
}
|
|
10318
|
-
|
|
10586
|
+
const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
|
|
10587
|
+
const groups = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, options);
|
|
10588
|
+
return { queryTokens, ...groups, filterBitset };
|
|
10319
10589
|
}
|
|
10320
10590
|
function fulltextMatches(partition, params, language, schema, options) {
|
|
10321
10591
|
const prepared = prepareFulltextQuery(partition, params, language, schema, options);
|
|
10322
10592
|
if (prepared === null) return null;
|
|
10323
10593
|
return partition.searchFulltextMatches({
|
|
10324
10594
|
queryTokens: prepared.queryTokens,
|
|
10325
|
-
|
|
10595
|
+
termGroups: prepared.termGroups,
|
|
10596
|
+
partialFields: prepared.partialFields,
|
|
10326
10597
|
fields: params.fields,
|
|
10327
10598
|
tolerance: params.tolerance ?? 0,
|
|
10328
10599
|
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
@@ -10335,14 +10606,15 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
10335
10606
|
if (prepared === null) {
|
|
10336
10607
|
return { scored: [], totalMatched: 0 };
|
|
10337
10608
|
}
|
|
10338
|
-
const { queryTokens,
|
|
10609
|
+
const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
|
|
10339
10610
|
const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
10340
10611
|
const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
|
|
10341
10612
|
const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
|
|
10342
10613
|
const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
|
|
10343
10614
|
const rawResult = partition.searchFulltext({
|
|
10344
10615
|
queryTokens,
|
|
10345
|
-
|
|
10616
|
+
termGroups,
|
|
10617
|
+
partialFields,
|
|
10346
10618
|
fields: params.fields,
|
|
10347
10619
|
boost: params.boost,
|
|
10348
10620
|
tolerance: params.tolerance ?? 0,
|
|
@@ -10366,7 +10638,8 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
10366
10638
|
params.tolerance ?? 0,
|
|
10367
10639
|
params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
10368
10640
|
params.exact ?? false,
|
|
10369
|
-
|
|
10641
|
+
termGroups,
|
|
10642
|
+
partialFields ?? []
|
|
10370
10643
|
);
|
|
10371
10644
|
}
|
|
10372
10645
|
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
@@ -10408,45 +10681,48 @@ function deduplicateTokens(tokens) {
|
|
|
10408
10681
|
}
|
|
10409
10682
|
return result;
|
|
10410
10683
|
}
|
|
10411
|
-
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact,
|
|
10684
|
+
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
|
|
10412
10685
|
const requiredCount = policy === "all" ? queryTokens.length : policy;
|
|
10413
10686
|
if (requiredCount <= 0) return scored;
|
|
10414
10687
|
if (requiredCount > queryTokens.length) return [];
|
|
10415
|
-
const
|
|
10688
|
+
const groups = /* @__PURE__ */ new Map();
|
|
10689
|
+
for (const group of termGroups ?? []) {
|
|
10690
|
+
const partialOnly = /* @__PURE__ */ new Set();
|
|
10691
|
+
for (const member of group.members) {
|
|
10692
|
+
if (member.factor === 0) partialOnly.add(member.token);
|
|
10693
|
+
}
|
|
10694
|
+
groups.set(group.token, {
|
|
10695
|
+
members: new Set(group.members.map((member) => member.token)),
|
|
10696
|
+
partialOnly,
|
|
10697
|
+
typoMatches: group.typoMatches
|
|
10698
|
+
});
|
|
10699
|
+
}
|
|
10416
10700
|
return scored.filter((doc) => {
|
|
10417
|
-
const matched = countDocTermMatches(
|
|
10418
|
-
doc,
|
|
10419
|
-
queryTokens,
|
|
10420
|
-
tolerance,
|
|
10421
|
-
prefixLength,
|
|
10422
|
-
exact,
|
|
10423
|
-
prefixExpansion?.token,
|
|
10424
|
-
expansionTerms
|
|
10425
|
-
);
|
|
10701
|
+
const matched = countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields);
|
|
10426
10702
|
return matched >= requiredCount;
|
|
10427
10703
|
});
|
|
10428
10704
|
}
|
|
10429
|
-
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact,
|
|
10705
|
+
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
|
|
10430
10706
|
const indexTokens = Object.keys(doc.idf);
|
|
10431
10707
|
if (indexTokens.length === 0) return 0;
|
|
10432
10708
|
let count = 0;
|
|
10433
10709
|
for (const qt of queryTokens) {
|
|
10434
|
-
|
|
10435
|
-
|
|
10436
|
-
|
|
10437
|
-
}
|
|
10438
|
-
continue;
|
|
10439
|
-
}
|
|
10440
|
-
if (queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact)) {
|
|
10441
|
-
count++;
|
|
10442
|
-
}
|
|
10710
|
+
const group = groups.get(qt.token);
|
|
10711
|
+
const satisfied = group === void 0 ? queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact) : groupSatisfied(group, doc, indexTokens, partialFields) || group.typoMatches && queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact);
|
|
10712
|
+
if (satisfied) count++;
|
|
10443
10713
|
}
|
|
10444
10714
|
return count;
|
|
10445
10715
|
}
|
|
10446
|
-
function
|
|
10716
|
+
function matchedInPartialField(doc, token, partialFields) {
|
|
10717
|
+
for (const field of partialFields) {
|
|
10718
|
+
if (doc.termFrequencies[`${field}:${token}`] !== void 0) return true;
|
|
10719
|
+
}
|
|
10720
|
+
return false;
|
|
10721
|
+
}
|
|
10722
|
+
function groupSatisfied(group, doc, indexTokens, partialFields) {
|
|
10447
10723
|
for (const indexToken of indexTokens) {
|
|
10448
|
-
if (indexToken
|
|
10449
|
-
if (
|
|
10724
|
+
if (!group.members.has(indexToken)) continue;
|
|
10725
|
+
if (!group.partialOnly.has(indexToken) || matchedInPartialField(doc, indexToken, partialFields)) return true;
|
|
10450
10726
|
}
|
|
10451
10727
|
return false;
|
|
10452
10728
|
}
|
|
@@ -10598,7 +10874,7 @@ function frozenSegmentBytes(segment) {
|
|
|
10598
10874
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
10599
10875
|
}
|
|
10600
10876
|
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
10601
|
-
return bytes;
|
|
10877
|
+
return bytes + segment.partialWords.bytes();
|
|
10602
10878
|
}
|
|
10603
10879
|
|
|
10604
10880
|
// src/core/partition/composite/compaction.ts
|
|
@@ -10733,20 +11009,19 @@ function subMatchesView(matches, layout, subIndex) {
|
|
|
10733
11009
|
|
|
10734
11010
|
// src/core/partition/composite/stats.ts
|
|
10735
11011
|
function collectCandidateTerms(subs, params) {
|
|
10736
|
-
const { queryTokens,
|
|
11012
|
+
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
10737
11013
|
const terms = /* @__PURE__ */ new Set();
|
|
10738
11014
|
for (const qt of queryTokens) {
|
|
10739
|
-
|
|
10740
|
-
|
|
10741
|
-
for (const
|
|
10742
|
-
continue;
|
|
11015
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
11016
|
+
if (group !== void 0) {
|
|
11017
|
+
for (const member of group.members) terms.add(member.token);
|
|
10743
11018
|
}
|
|
10744
|
-
if (exact || tolerance === 0) {
|
|
11019
|
+
if (group === void 0 && (exact || tolerance === 0)) {
|
|
10745
11020
|
terms.add(qt.token);
|
|
10746
11021
|
continue;
|
|
10747
11022
|
}
|
|
10748
11023
|
for (const sub of subs) {
|
|
10749
|
-
for (const match of sub.invertedIdx
|
|
11024
|
+
for (const match of typoMatchesFor(sub.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
10750
11025
|
terms.add(match.token);
|
|
10751
11026
|
}
|
|
10752
11027
|
}
|
|
@@ -10756,11 +11031,7 @@ function collectCandidateTerms(subs, params) {
|
|
|
10756
11031
|
function resolveQueryTermFrequencies(subs, params) {
|
|
10757
11032
|
const frequencies = /* @__PURE__ */ Object.create(null);
|
|
10758
11033
|
for (const term of collectCandidateTerms(subs, params)) {
|
|
10759
|
-
|
|
10760
|
-
for (const sub of subs) {
|
|
10761
|
-
const list = sub.invertedIdx.lookup(term);
|
|
10762
|
-
if (list !== void 0) total += list.docIdSet.size;
|
|
10763
|
-
}
|
|
11034
|
+
const total = partitionDocumentFrequency(subs, term);
|
|
10764
11035
|
if (total > 0) frequencies[term] = total;
|
|
10765
11036
|
}
|
|
10766
11037
|
return frequencies;
|
|
@@ -10946,9 +11217,10 @@ function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
|
|
|
10946
11217
|
|
|
10947
11218
|
// src/core/partition/composite/suggest.ts
|
|
10948
11219
|
function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
11220
|
+
if (limit <= 0) return [];
|
|
10949
11221
|
const merged = /* @__PURE__ */ new Map();
|
|
10950
11222
|
for (const sub of subs) {
|
|
10951
|
-
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix,
|
|
11223
|
+
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, Number.MAX_SAFE_INTEGER)) {
|
|
10952
11224
|
let entry = merged.get(suggestion.token);
|
|
10953
11225
|
if (entry === void 0) {
|
|
10954
11226
|
entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
|
|
@@ -10973,19 +11245,6 @@ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
|
10973
11245
|
if (suggestions.length > limit) suggestions.length = limit;
|
|
10974
11246
|
return suggestions;
|
|
10975
11247
|
}
|
|
10976
|
-
function compositeExpandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
|
|
10977
|
-
const expansions = [];
|
|
10978
|
-
const seen = /* @__PURE__ */ new Set();
|
|
10979
|
-
for (const sub of subs) {
|
|
10980
|
-
for (const term of expandTermPrefix(sub, surfacePrefix, stemmedToken, maxExpansions)) {
|
|
10981
|
-
if (seen.has(term)) continue;
|
|
10982
|
-
seen.add(term);
|
|
10983
|
-
expansions.push(term);
|
|
10984
|
-
if (expansions.length === maxExpansions) return expansions;
|
|
10985
|
-
}
|
|
10986
|
-
}
|
|
10987
|
-
return expansions;
|
|
10988
|
-
}
|
|
10989
11248
|
|
|
10990
11249
|
// src/core/partition/composite/index.ts
|
|
10991
11250
|
function isCompositePartition(partition) {
|
|
@@ -11189,7 +11448,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11189
11448
|
return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
|
|
11190
11449
|
},
|
|
11191
11450
|
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
11192
|
-
return
|
|
11451
|
+
return expandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
|
|
11452
|
+
},
|
|
11453
|
+
expandPartialWords(analysis, requests) {
|
|
11454
|
+
return expandPartialWords(subs(), analysis, requests);
|
|
11455
|
+
},
|
|
11456
|
+
preparePartialWords(analysis) {
|
|
11457
|
+
expandPartialWords(subs(), analysis, []);
|
|
11193
11458
|
},
|
|
11194
11459
|
get(docId, projection) {
|
|
11195
11460
|
const owner = frozenOwner(docId);
|
|
@@ -11250,6 +11515,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11250
11515
|
}
|
|
11251
11516
|
|
|
11252
11517
|
// src/partitioning/insert-options.ts
|
|
11518
|
+
function resolvePartialWordAnalysis(config, language, analysis) {
|
|
11519
|
+
const fields = partialFieldsOf(config.schema);
|
|
11520
|
+
if (fields.length === 0) return null;
|
|
11521
|
+
return { fields, language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
11522
|
+
}
|
|
11253
11523
|
function resolvePartitionInsertOptions(config, analysis, options) {
|
|
11254
11524
|
const applyStrict = config.strict === true;
|
|
11255
11525
|
const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
|
|
@@ -11327,8 +11597,10 @@ function setNestedValue(obj, path, value) {
|
|
|
11327
11597
|
// src/partitioning/manager/index.ts
|
|
11328
11598
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
11329
11599
|
requirePatternSearchFor(config.schema);
|
|
11600
|
+
requirePartialWordSearchFor(config.schema);
|
|
11330
11601
|
const count = initialPartitionCount ?? 1;
|
|
11331
11602
|
const analysis = resolveIndexAnalysis(config);
|
|
11603
|
+
const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
11332
11604
|
const trackPositions = config.trackPositions ?? true;
|
|
11333
11605
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
11334
11606
|
let partitions = [];
|
|
@@ -11348,6 +11620,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11348
11620
|
function resolveInsertOptions(options) {
|
|
11349
11621
|
return resolvePartitionInsertOptions(config, analysis, options);
|
|
11350
11622
|
}
|
|
11623
|
+
function preparePartialWords(partitionId) {
|
|
11624
|
+
if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
|
|
11625
|
+
}
|
|
11351
11626
|
function locateDocument(docId) {
|
|
11352
11627
|
return docPartitionMap.get(docId);
|
|
11353
11628
|
}
|
|
@@ -11403,6 +11678,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11403
11678
|
get analysis() {
|
|
11404
11679
|
return analysis;
|
|
11405
11680
|
},
|
|
11681
|
+
get partialWordAnalysis() {
|
|
11682
|
+
return partialWordAnalysis;
|
|
11683
|
+
},
|
|
11406
11684
|
getPartition(partitionId) {
|
|
11407
11685
|
validatePartitionId(partitionId);
|
|
11408
11686
|
return partitions[partitionId];
|
|
@@ -11479,6 +11757,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11479
11757
|
rebuildTextIndex(partitionId) {
|
|
11480
11758
|
validatePartitionId(partitionId);
|
|
11481
11759
|
partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
|
|
11760
|
+
preparePartialWords(partitionId);
|
|
11482
11761
|
},
|
|
11483
11762
|
beginBatchRemove() {
|
|
11484
11763
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -11580,6 +11859,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11580
11859
|
deserializePartition(partitionId, data) {
|
|
11581
11860
|
validatePartitionId(partitionId);
|
|
11582
11861
|
partitions[partitionId].deserialize(data, config.schema);
|
|
11862
|
+
preparePartialWords(partitionId);
|
|
11583
11863
|
rebuildDocPartitionMapForPartition(partitionId);
|
|
11584
11864
|
},
|
|
11585
11865
|
getAggregateStats() {
|
|
@@ -18050,6 +18330,15 @@ function decodeStoredDocument(bytes, discardedFields) {
|
|
|
18050
18330
|
}
|
|
18051
18331
|
return document2;
|
|
18052
18332
|
}
|
|
18333
|
+
function* storedDocumentsOf(data) {
|
|
18334
|
+
const count = data.offsets.length - 1;
|
|
18335
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
18336
|
+
yield decodeStoredDocument(
|
|
18337
|
+
data.blob.subarray(data.offsets[ordinal], data.offsets[ordinal + 1]),
|
|
18338
|
+
data.discardedFields
|
|
18339
|
+
);
|
|
18340
|
+
}
|
|
18341
|
+
}
|
|
18053
18342
|
function decodeEveryStoredDocument(data) {
|
|
18054
18343
|
const count = data.offsets.length - 1;
|
|
18055
18344
|
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
@@ -18093,6 +18382,9 @@ function wrapEncodedDocumentTable(data) {
|
|
|
18093
18382
|
decodedBytes += documentValueBytes(document2);
|
|
18094
18383
|
}
|
|
18095
18384
|
return document2;
|
|
18385
|
+
},
|
|
18386
|
+
readAt(ordinal) {
|
|
18387
|
+
return decoded[ordinal] ?? decodeStoredDocument(blob.subarray(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
|
|
18096
18388
|
}
|
|
18097
18389
|
};
|
|
18098
18390
|
}
|
|
@@ -18106,6 +18398,9 @@ function wrapDocumentArray(documents) {
|
|
|
18106
18398
|
byteLength: residentBytes,
|
|
18107
18399
|
docAt(ordinal) {
|
|
18108
18400
|
return documents[ordinal];
|
|
18401
|
+
},
|
|
18402
|
+
readAt(ordinal) {
|
|
18403
|
+
return documents[ordinal];
|
|
18109
18404
|
}
|
|
18110
18405
|
};
|
|
18111
18406
|
}
|
|
@@ -18492,6 +18787,10 @@ function createFrozenInvertedReader(table, views) {
|
|
|
18492
18787
|
if (at < 0) return void 0;
|
|
18493
18788
|
return views.viewReadOnce(table.payloadSlot(at), table.documentFrequencyAt(at));
|
|
18494
18789
|
},
|
|
18790
|
+
documentFrequency(token) {
|
|
18791
|
+
const at = table.find(token);
|
|
18792
|
+
return at < 0 ? 0 : table.documentFrequencyAt(at);
|
|
18793
|
+
},
|
|
18495
18794
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
18496
18795
|
if (tolerance === 0) {
|
|
18497
18796
|
const at = table.find(token);
|
|
@@ -18814,6 +19113,35 @@ function sharedSurfaceTable(payload) {
|
|
|
18814
19113
|
if (table.counts.length === 0) return null;
|
|
18815
19114
|
return { blob: sharedUint8(table.blob), offsets: sharedUint32(table.offsets), counts: sharedUint32(table.counts) };
|
|
18816
19115
|
}
|
|
19116
|
+
function sharePartialWordTable(table) {
|
|
19117
|
+
if (typeof SharedArrayBuffer !== "function") return table;
|
|
19118
|
+
return {
|
|
19119
|
+
fields: [...table.fields],
|
|
19120
|
+
spellings: sharedUint8(table.spellings),
|
|
19121
|
+
spellingOffsets: sharedUint32(table.spellingOffsets),
|
|
19122
|
+
spellingTokens: sharedUint32(table.spellingTokens),
|
|
19123
|
+
spellingCounts: sharedUint32(table.spellingCounts),
|
|
19124
|
+
runs: sharedUint32(table.runs),
|
|
19125
|
+
runOffsets: sharedUint32(table.runOffsets),
|
|
19126
|
+
runEntries: sharedUint32(table.runEntries)
|
|
19127
|
+
};
|
|
19128
|
+
}
|
|
19129
|
+
function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
|
|
19130
|
+
let source = spellings;
|
|
19131
|
+
if (source === null && analysis !== null && analysis.fields.length > 0) {
|
|
19132
|
+
source = encodePartialSpellings(analysis.fields, spellingCountsOf(storedDocumentsOf(documentData), analysis));
|
|
19133
|
+
}
|
|
19134
|
+
if (source === null || source.fields.length === 0) return null;
|
|
19135
|
+
const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
|
|
19136
|
+
return table === null ? null : sharePartialWordTable(table);
|
|
19137
|
+
}
|
|
19138
|
+
function shareSegmentArrays(arrays, analysis, segmentId) {
|
|
19139
|
+
return {
|
|
19140
|
+
...arrays,
|
|
19141
|
+
segmentId: segmentId ?? generateId(),
|
|
19142
|
+
partialWords: sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis)
|
|
19143
|
+
};
|
|
19144
|
+
}
|
|
18817
19145
|
function segmentArraysOf(payload, documentData) {
|
|
18818
19146
|
const tokenTable = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
18819
19147
|
const idTable = encodeExternalIdTableData(payload.docIds);
|
|
@@ -18842,11 +19170,11 @@ function segmentArraysOf(payload, documentData) {
|
|
|
18842
19170
|
surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable
|
|
18843
19171
|
};
|
|
18844
19172
|
}
|
|
18845
|
-
function freezeSegmentShared(payload, documents, segmentId) {
|
|
19173
|
+
function freezeSegmentShared(payload, documents, segmentId, analysis = null) {
|
|
18846
19174
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
18847
|
-
return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId);
|
|
19175
|
+
return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId, analysis);
|
|
18848
19176
|
}
|
|
18849
|
-
function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
19177
|
+
function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null) {
|
|
18850
19178
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
18851
19179
|
const tokenData = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
18852
19180
|
const idData = encodeExternalIdTableData(payload.docIds);
|
|
@@ -18913,7 +19241,8 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
|
18913
19241
|
offsets: sharedUint32(entry.offsets),
|
|
18914
19242
|
postings: sharedUint32(entry.postings)
|
|
18915
19243
|
})),
|
|
18916
|
-
surfaceTable: sharedSurfaceTable(payload)
|
|
19244
|
+
surfaceTable: sharedSurfaceTable(payload),
|
|
19245
|
+
partialWords: sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis)
|
|
18917
19246
|
};
|
|
18918
19247
|
}
|
|
18919
19248
|
|
|
@@ -18949,7 +19278,12 @@ function buildGeoReaders(entries) {
|
|
|
18949
19278
|
}
|
|
18950
19279
|
return readers;
|
|
18951
19280
|
}
|
|
18952
|
-
function
|
|
19281
|
+
function* survivingDocuments(documentSource, tombstones) {
|
|
19282
|
+
for (let ordinal = 0; ordinal < documentSource.count; ordinal++) {
|
|
19283
|
+
if (!tombstones.has(ordinal)) yield documentSource.readAt(ordinal);
|
|
19284
|
+
}
|
|
19285
|
+
}
|
|
19286
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
|
|
18953
19287
|
const tombstones = createFrozenTombstones();
|
|
18954
19288
|
const postingViews = createFrozenPostingViews(source, tombstones);
|
|
18955
19289
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
@@ -18957,6 +19291,10 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
18957
19291
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
18958
19292
|
}
|
|
18959
19293
|
const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
|
|
19294
|
+
const partialWords = createFrozenPartialWords(partialWordTable, source.tokenTable, {
|
|
19295
|
+
documentAt: (ordinal) => documentSource.readAt(ordinal),
|
|
19296
|
+
survivors: () => survivingDocuments(documentSource, tombstones)
|
|
19297
|
+
});
|
|
18960
19298
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
18961
19299
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
18962
19300
|
const sizeIndexes = /* @__PURE__ */ new Map();
|
|
@@ -18989,6 +19327,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
18989
19327
|
patternIndexes,
|
|
18990
19328
|
sizeIndexes,
|
|
18991
19329
|
fieldNameTable,
|
|
19330
|
+
partialWords,
|
|
18992
19331
|
trackPositions: source.positionOffsets !== null,
|
|
18993
19332
|
flatSchemaCache: null,
|
|
18994
19333
|
lastSchemaRef: null,
|
|
@@ -19008,6 +19347,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
19008
19347
|
if (ordinal === void 0) return false;
|
|
19009
19348
|
const added = tombstones.add(ordinal);
|
|
19010
19349
|
if (added) {
|
|
19350
|
+
partialWords.afterRemoval(ordinal);
|
|
19011
19351
|
docStore.releaseSortedDocIds();
|
|
19012
19352
|
segment.sortColumns = null;
|
|
19013
19353
|
segment.scoreBuffer = null;
|
|
@@ -19023,18 +19363,20 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
19023
19363
|
return segment;
|
|
19024
19364
|
}
|
|
19025
19365
|
function createFrozenSegment(payload, documents, segmentId) {
|
|
19366
|
+
const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
|
|
19026
19367
|
return assembleFrozenSegment(
|
|
19027
19368
|
segmentId ?? generateId(),
|
|
19028
19369
|
{
|
|
19029
19370
|
...payload,
|
|
19030
|
-
tokenTable
|
|
19371
|
+
tokenTable,
|
|
19031
19372
|
idTable: buildExternalIdTable(payload.docIds),
|
|
19032
19373
|
documentTable: null,
|
|
19033
19374
|
readSurfaceForms: () => payload.surfaceForms,
|
|
19034
19375
|
docFrequencies: () => payload.docFrequencies
|
|
19035
19376
|
},
|
|
19036
19377
|
wrapDocumentArray(documents),
|
|
19037
|
-
null
|
|
19378
|
+
null,
|
|
19379
|
+
payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, tokenTable)
|
|
19038
19380
|
);
|
|
19039
19381
|
}
|
|
19040
19382
|
function createSharedFrozenSegment(snapshot) {
|
|
@@ -19056,7 +19398,8 @@ function createSharedFrozenSegment(snapshot) {
|
|
|
19056
19398
|
}
|
|
19057
19399
|
},
|
|
19058
19400
|
wrapEncodedDocumentTable(snapshot.documentTable),
|
|
19059
|
-
snapshot
|
|
19401
|
+
snapshot,
|
|
19402
|
+
snapshot.partialWords
|
|
19060
19403
|
);
|
|
19061
19404
|
}
|
|
19062
19405
|
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
@@ -19638,10 +19981,10 @@ function mergeDocuments2(inputs, documentCount) {
|
|
|
19638
19981
|
}
|
|
19639
19982
|
return { blob, offsets };
|
|
19640
19983
|
}
|
|
19641
|
-
function mergeFrozenSegments(segments) {
|
|
19984
|
+
function mergeFrozenSegments(segments, analysis) {
|
|
19642
19985
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
19643
19986
|
const merged = mergeSegmentContents(segments.map((segment) => ({ segment })));
|
|
19644
|
-
return freezeEncodedSegmentShared(merged.payload, merged.documents);
|
|
19987
|
+
return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis);
|
|
19645
19988
|
}
|
|
19646
19989
|
function mergeSegmentContents(selections) {
|
|
19647
19990
|
const docIds = [];
|
|
@@ -19670,6 +20013,7 @@ function mergeSegmentContents(selections) {
|
|
|
19670
20013
|
totalFieldLengths: lengths.totalFieldLengths,
|
|
19671
20014
|
docFrequencies: postings.docFrequencies,
|
|
19672
20015
|
surfaceForms: mergeSurfaceForms2(inputs),
|
|
20016
|
+
partialSpellings: null,
|
|
19673
20017
|
...mergeFieldIndexes2(inputs)
|
|
19674
20018
|
};
|
|
19675
20019
|
return { payload, documents: mergeDocuments2(inputs, documentCount) };
|
|
@@ -19970,14 +20314,18 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
19970
20314
|
patternIndexes: /* @__PURE__ */ new Map(),
|
|
19971
20315
|
sizeIndexes: /* @__PURE__ */ new Map()
|
|
19972
20316
|
};
|
|
20317
|
+
const partialFields = partialFieldsOf(schema);
|
|
20318
|
+
const spellings = /* @__PURE__ */ new Map();
|
|
20319
|
+
let fieldSpellings = null;
|
|
19973
20320
|
let positionOffset = 0;
|
|
19974
20321
|
const addToken = (token, position, surface) => {
|
|
19975
20322
|
postings.addToken(token, positionOffset + position);
|
|
19976
20323
|
surfaces.count(surface, token);
|
|
20324
|
+
if (fieldSpellings !== null) countSpelling(fieldSpellings, token, surface ?? token, 1);
|
|
19977
20325
|
};
|
|
19978
20326
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
19979
20327
|
const document2 = documents[ordinal].document;
|
|
19980
|
-
for (const { path: fieldPath, base: fieldType, pattern } of fields) {
|
|
20328
|
+
for (const { path: fieldPath, base: fieldType, pattern, partial } of fields) {
|
|
19981
20329
|
const value = getNestedValue(document2, fieldPath);
|
|
19982
20330
|
if (value === void 0 || value === null) continue;
|
|
19983
20331
|
recordValueSize(filters.sizeIndexes, ordinal, fieldPath, fieldType, value);
|
|
@@ -19986,6 +20334,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
19986
20334
|
indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
|
|
19987
20335
|
continue;
|
|
19988
20336
|
}
|
|
20337
|
+
fieldSpellings = partial ? spellings : null;
|
|
19989
20338
|
postings.beginField();
|
|
19990
20339
|
positionOffset = 0;
|
|
19991
20340
|
for (const text of fieldType === "string" ? [value] : value) {
|
|
@@ -20006,6 +20355,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
20006
20355
|
fieldLengthColumns: lengths.columns,
|
|
20007
20356
|
totalFieldLengths: lengths.totalsByName(),
|
|
20008
20357
|
surfaceForms: surfaces.encode(),
|
|
20358
|
+
partialSpellings: partialFields.length === 0 ? null : encodePartialSpellings(partialFields, spellings),
|
|
20009
20359
|
...encodeFieldIndexes(filters, identity)
|
|
20010
20360
|
};
|
|
20011
20361
|
}
|
|
@@ -20072,7 +20422,7 @@ function runSegmentAction(entry, action) {
|
|
|
20072
20422
|
}
|
|
20073
20423
|
case "compactSegments": {
|
|
20074
20424
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
20075
|
-
return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds));
|
|
20425
|
+
return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds), entry.manager.partialWordAnalysis);
|
|
20076
20426
|
}
|
|
20077
20427
|
case "swapSegments": {
|
|
20078
20428
|
requireComposite(entry, action.indexName, action.partitionId, "swap");
|
|
@@ -23465,7 +23815,7 @@ function selectionOfMask(segment, keep) {
|
|
|
23465
23815
|
}
|
|
23466
23816
|
function frozenLiveTail(live, shared2) {
|
|
23467
23817
|
if (live.count() === 0) return null;
|
|
23468
|
-
const payload = live.encodeSegment();
|
|
23818
|
+
const payload = { ...live.encodeSegment(), partialSpellings: null };
|
|
23469
23819
|
const documents = survivorDocumentsOf(partitionStateOf(live), payload);
|
|
23470
23820
|
if (shared2) {
|
|
23471
23821
|
const snapshot = freezeSegmentShared(payload, documents);
|
|
@@ -24981,6 +25331,7 @@ function emptyPayload() {
|
|
|
24981
25331
|
totalFieldLengths: {},
|
|
24982
25332
|
docFrequencies: {},
|
|
24983
25333
|
surfaceForms: null,
|
|
25334
|
+
partialSpellings: null,
|
|
24984
25335
|
numeric: [],
|
|
24985
25336
|
boolean: [],
|
|
24986
25337
|
enums: [],
|
|
@@ -25086,7 +25437,7 @@ function chooseWindow(segments, wantedLength) {
|
|
|
25086
25437
|
return null;
|
|
25087
25438
|
}
|
|
25088
25439
|
function frozenOf(contents) {
|
|
25089
|
-
return createSharedFrozenSegment(
|
|
25440
|
+
return createSharedFrozenSegment(shareSegmentArrays(contents.arrays, null));
|
|
25090
25441
|
}
|
|
25091
25442
|
function applyTimeOrder(frozen, contents) {
|
|
25092
25443
|
for (let newer = 1; newer < frozen.length; newer++) {
|
|
@@ -25354,7 +25705,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
25354
25705
|
const contents = await readSegmentFile(store, ref);
|
|
25355
25706
|
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
25356
25707
|
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
25357
|
-
const segment = createSharedFrozenSegment(
|
|
25708
|
+
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
25358
25709
|
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
25359
25710
|
if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
|
|
25360
25711
|
await yieldToEventLoop6();
|
|
@@ -25787,7 +26138,7 @@ function validateHnswMetric2(value) {
|
|
|
25787
26138
|
}
|
|
25788
26139
|
return void 0;
|
|
25789
26140
|
}
|
|
25790
|
-
function
|
|
26141
|
+
function storedDocumentsOf2(raw) {
|
|
25791
26142
|
requireValidFieldTypes(raw.schema ?? {});
|
|
25792
26143
|
const documents = {};
|
|
25793
26144
|
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
@@ -25801,11 +26152,11 @@ function storedDocumentsOf(raw) {
|
|
|
25801
26152
|
}
|
|
25802
26153
|
function documentsOfPayloadV2(data) {
|
|
25803
26154
|
const raw = decode(data);
|
|
25804
|
-
return { documents:
|
|
26155
|
+
return { documents: storedDocumentsOf2(raw), schema: raw.schema ?? {} };
|
|
25805
26156
|
}
|
|
25806
26157
|
function deserializePayloadV2(data) {
|
|
25807
26158
|
const raw = decode(data);
|
|
25808
|
-
const documents =
|
|
26159
|
+
const documents = storedDocumentsOf2(raw);
|
|
25809
26160
|
const fieldNames = raw.inverted_index?.field_names ?? [];
|
|
25810
26161
|
const invertedIndex = {};
|
|
25811
26162
|
for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
|
|
@@ -27873,6 +28224,16 @@ function validatePartitionConfig(partitions) {
|
|
|
27873
28224
|
});
|
|
27874
28225
|
}
|
|
27875
28226
|
}
|
|
28227
|
+
function requireSurfaceFormsForPartialFields(schema, surfaceForms) {
|
|
28228
|
+
if (surfaceForms !== false) return;
|
|
28229
|
+
const partialFields = partialFieldsOf(schema);
|
|
28230
|
+
if (partialFields.length === 0) return;
|
|
28231
|
+
throw new NarsilError(
|
|
28232
|
+
ErrorCodes.CONFIG_INVALID,
|
|
28233
|
+
`Field "${partialFields[0]}" declares "partial", which compares words as the documents wrote them, so its index needs surfaceForms left on`,
|
|
28234
|
+
{ fields: [...partialFields], surfaceForms: false }
|
|
28235
|
+
);
|
|
28236
|
+
}
|
|
27876
28237
|
function validatePatternValueLimit(limit) {
|
|
27877
28238
|
if (limit === void 0) return;
|
|
27878
28239
|
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < MIN_PATTERN_VALUE_LIMIT || limit > MAX_PATTERN_VALUE_LIMIT) {
|
|
@@ -27987,6 +28348,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
27987
28348
|
try {
|
|
27988
28349
|
indexConfig.schema = validateSchema(indexConfig.schema);
|
|
27989
28350
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
28351
|
+
requireSurfaceFormsForPartialFields(indexConfig.schema, indexConfig.surfaceForms);
|
|
27990
28352
|
} catch (err) {
|
|
27991
28353
|
throw recoveryFailure(metadata, err);
|
|
27992
28354
|
}
|
|
@@ -28700,7 +29062,7 @@ function checkHeapAfterRecovery(core) {
|
|
|
28700
29062
|
}
|
|
28701
29063
|
|
|
28702
29064
|
// src/core/partition/frozen/share.ts
|
|
28703
|
-
function shareFrozenSegment(segment) {
|
|
29065
|
+
function shareFrozenSegment(segment, analysis) {
|
|
28704
29066
|
if (segment.sharedSnapshot !== null) {
|
|
28705
29067
|
return { snapshot: segment.sharedSnapshot, tombstonedDocIds: segment.tombstonedDocIds() };
|
|
28706
29068
|
}
|
|
@@ -28714,7 +29076,7 @@ function shareFrozenSegment(segment) {
|
|
|
28714
29076
|
}
|
|
28715
29077
|
return stored.fields;
|
|
28716
29078
|
});
|
|
28717
|
-
const snapshot = freezeSegmentShared(payload, documents, segment.segmentId);
|
|
29079
|
+
const snapshot = freezeSegmentShared(payload, documents, segment.segmentId, analysis);
|
|
28718
29080
|
return snapshot === null ? null : { snapshot, tombstonedDocIds: [] };
|
|
28719
29081
|
}
|
|
28720
29082
|
|
|
@@ -28725,7 +29087,7 @@ function captureComposite(manager, partitionId) {
|
|
|
28725
29087
|
const segmentIds = partition.frozenSegmentSizes().map((size) => size.segmentId);
|
|
28726
29088
|
const frozen = [];
|
|
28727
29089
|
for (const segment of partition.frozenSegmentsById(segmentIds)) {
|
|
28728
|
-
const shared2 = shareFrozenSegment(segment);
|
|
29090
|
+
const shared2 = shareFrozenSegment(segment, manager.partialWordAnalysis);
|
|
28729
29091
|
if (shared2 === null) return null;
|
|
28730
29092
|
frozen.push(shared2);
|
|
28731
29093
|
}
|
|
@@ -29130,7 +29492,7 @@ function liveTailCount(manager, partitionId) {
|
|
|
29130
29492
|
}
|
|
29131
29493
|
function freezeIntoSharedSegment(manager, partitionId) {
|
|
29132
29494
|
return manager.freezeLiveTail(partitionId, (payload, documents) => {
|
|
29133
|
-
const snapshot = freezeSegmentShared(payload, documents);
|
|
29495
|
+
const snapshot = freezeSegmentShared(payload, documents, void 0, manager.partialWordAnalysis);
|
|
29134
29496
|
return snapshot === null ? null : createSharedFrozenSegment(snapshot);
|
|
29135
29497
|
});
|
|
29136
29498
|
}
|
|
@@ -29253,7 +29615,7 @@ async function compactPartitionSegments2(state, indexName, manager, partitionId,
|
|
|
29253
29615
|
const mainCanMerge = composite !== null && mainHoldsAll(composite, segmentIds);
|
|
29254
29616
|
let snapshot = await compactOnWorker(state, indexName, partitionId, segmentIds);
|
|
29255
29617
|
if (snapshot === null && mainCanMerge && composite !== null) {
|
|
29256
|
-
snapshot = mergeFrozenSegments(composite.frozenSegmentsById(segmentIds));
|
|
29618
|
+
snapshot = mergeFrozenSegments(composite.frozenSegmentsById(segmentIds), manager.partialWordAnalysis);
|
|
29257
29619
|
}
|
|
29258
29620
|
if (snapshot === null) {
|
|
29259
29621
|
if (!mainCanMerge) forgetLedgerSegments(state, indexName, partitionId, segmentIds);
|
|
@@ -30333,57 +30695,61 @@ var POSITIONS_PER_WORD = 32;
|
|
|
30333
30695
|
function positionWords(positions) {
|
|
30334
30696
|
return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
|
|
30335
30697
|
}
|
|
30336
|
-
function automatonBytes(positions) {
|
|
30337
|
-
return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES;
|
|
30338
|
-
}
|
|
30339
30698
|
function smallestMatcherBytes(positions) {
|
|
30340
|
-
return
|
|
30699
|
+
return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
|
|
30341
30700
|
}
|
|
30342
|
-
function widestPatternSplit(
|
|
30343
|
-
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES /
|
|
30701
|
+
function widestPatternSplit(footprintBytes) {
|
|
30702
|
+
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / Math.max(1, footprintBytes)));
|
|
30344
30703
|
}
|
|
30345
30704
|
|
|
30346
30705
|
// src/core/pattern-index/query-positions.ts
|
|
30347
30706
|
function isRecord6(value) {
|
|
30348
30707
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
30349
30708
|
}
|
|
30350
|
-
function
|
|
30351
|
-
if (typeof text !== "string") return
|
|
30352
|
-
return caseFold ? foldedCodePointCount(text) : codePointCount(text);
|
|
30709
|
+
function literalFootprint(text, caseFold) {
|
|
30710
|
+
if (typeof text !== "string") return null;
|
|
30711
|
+
return smallestMatcherBytes(caseFold ? foldedCodePointCount(text) : codePointCount(text));
|
|
30353
30712
|
}
|
|
30354
30713
|
function larger(left, right) {
|
|
30355
30714
|
if (left === null) return right;
|
|
30356
30715
|
if (right === null) return left;
|
|
30357
30716
|
return Math.max(left, right);
|
|
30358
30717
|
}
|
|
30359
|
-
function largestTestOf(filter) {
|
|
30718
|
+
function largestTestOf(filter, module) {
|
|
30360
30719
|
const caseFold = filter.caseFold === true;
|
|
30361
30720
|
let largest = null;
|
|
30362
30721
|
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
30363
30722
|
const operand = filter[operator];
|
|
30364
30723
|
if (operand === void 0) continue;
|
|
30365
30724
|
const texts = Array.isArray(operand) ? operand : [operand];
|
|
30366
|
-
for (const text of texts) largest = larger(largest,
|
|
30725
|
+
for (const text of texts) largest = larger(largest, literalFootprint(text, caseFold));
|
|
30726
|
+
}
|
|
30727
|
+
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
30728
|
+
const pattern = filter[operator];
|
|
30729
|
+
if (typeof pattern !== "string") continue;
|
|
30730
|
+
largest = larger(largest, module.expressionFootprint(operator, pattern, caseFold, PATTERN_HOST));
|
|
30367
30731
|
}
|
|
30368
30732
|
return largest;
|
|
30369
30733
|
}
|
|
30370
|
-
function largestIn(expression, patternFields) {
|
|
30734
|
+
function largestIn(expression, patternFields, module) {
|
|
30371
30735
|
let largest = null;
|
|
30372
30736
|
if (isRecord6(expression.fields)) {
|
|
30373
30737
|
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
30374
|
-
if (patternFields.has(fieldPath) && isRecord6(filter)) largest = larger(largest, largestTestOf(filter));
|
|
30738
|
+
if (patternFields.has(fieldPath) && isRecord6(filter)) largest = larger(largest, largestTestOf(filter, module));
|
|
30375
30739
|
}
|
|
30376
30740
|
}
|
|
30377
30741
|
const nested = [...expression.and ?? [], ...expression.or ?? []];
|
|
30378
30742
|
if (expression.not !== void 0) nested.push(expression.not);
|
|
30379
|
-
for (const child of nested) largest = larger(largest, largestIn(child, patternFields));
|
|
30743
|
+
for (const child of nested) largest = larger(largest, largestIn(child, patternFields, module));
|
|
30380
30744
|
return largest;
|
|
30381
30745
|
}
|
|
30382
|
-
function
|
|
30746
|
+
function largestPatternFootprint(expression, schema) {
|
|
30383
30747
|
if (expression === void 0) return null;
|
|
30748
|
+
const module = availablePatternSearch();
|
|
30749
|
+
if (module === null) return null;
|
|
30384
30750
|
const patternFields = /* @__PURE__ */ new Set();
|
|
30385
30751
|
for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
|
|
30386
|
-
return patternFields.size === 0 ? null : largestIn(expression, patternFields);
|
|
30752
|
+
return patternFields.size === 0 ? null : largestIn(expression, patternFields, module);
|
|
30387
30753
|
}
|
|
30388
30754
|
|
|
30389
30755
|
// src/engine/orchestration/search.ts
|
|
@@ -30450,7 +30816,7 @@ function isWorkCapError(err) {
|
|
|
30450
30816
|
function patternSplitWidth(state, indexName, params) {
|
|
30451
30817
|
const schema = state.indexRegistry.get(indexName)?.config.schema;
|
|
30452
30818
|
if (schema === void 0) return Number.POSITIVE_INFINITY;
|
|
30453
|
-
const largest =
|
|
30819
|
+
const largest = largestPatternFootprint(params.filters, schema);
|
|
30454
30820
|
if (largest === null) return Number.POSITIVE_INFINITY;
|
|
30455
30821
|
if (typeof SharedArrayBuffer !== "function") return 1;
|
|
30456
30822
|
return widestPatternSplit(largest);
|
|
@@ -31226,7 +31592,9 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
31226
31592
|
if (appliesCreationLimits) requireCreatableSchema(schema);
|
|
31227
31593
|
const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
31228
31594
|
requirePatternSearchFor(schema);
|
|
31595
|
+
requirePartialWordSearchFor(schema);
|
|
31229
31596
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
31597
|
+
requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
|
|
31230
31598
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
31231
31599
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
31232
31600
|
validateBM25Params(indexConfig.bm25);
|
|
@@ -31256,10 +31624,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
31256
31624
|
const fields = indexConfig.embedding.fields;
|
|
31257
31625
|
if (typeof configuredAdapter === "string") {
|
|
31258
31626
|
embeddingAdapterName = configuredAdapter;
|
|
31259
|
-
const
|
|
31260
|
-
if (
|
|
31627
|
+
const registered3 = core.embeddingAdapters.get(configuredAdapter);
|
|
31628
|
+
if (registered3 !== void 0) {
|
|
31261
31629
|
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
31262
|
-
{ fields, adapter:
|
|
31630
|
+
{ fields, adapter: registered3 },
|
|
31263
31631
|
indexConfig.schema,
|
|
31264
31632
|
config?.embedding
|
|
31265
31633
|
);
|
|
@@ -34552,6 +34920,7 @@ function prepareRestoredIndex(configuration, deps, reject2) {
|
|
|
34552
34920
|
const language = getLanguage(languageName);
|
|
34553
34921
|
const schema = validateSchema(configuration.schema);
|
|
34554
34922
|
validatePatternValueLimit(configFields.patternValueLimit);
|
|
34923
|
+
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
34555
34924
|
validateVectorPromotion(configFields.vectorPromotion);
|
|
34556
34925
|
validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
|
|
34557
34926
|
if (configFields.required !== void 0 && configFields.required.length > 0) {
|
|
@@ -35109,7 +35478,7 @@ function boundVectorOf(vector) {
|
|
|
35109
35478
|
return bound;
|
|
35110
35479
|
}
|
|
35111
35480
|
var FILTERS_SLOT = 2;
|
|
35112
|
-
var BOUND_SLOT_COUNT =
|
|
35481
|
+
var BOUND_SLOT_COUNT = 16;
|
|
35113
35482
|
function bindingOf(values) {
|
|
35114
35483
|
const stream = { bytes: new Uint8Array(256), length: 0 };
|
|
35115
35484
|
for (const value of values) {
|
|
@@ -35128,6 +35497,7 @@ function queryBindingOf(params) {
|
|
|
35128
35497
|
params.tolerance,
|
|
35129
35498
|
params.prefixLength,
|
|
35130
35499
|
params.prefix,
|
|
35500
|
+
params.partialWords,
|
|
35131
35501
|
params.exact,
|
|
35132
35502
|
params.pinned,
|
|
35133
35503
|
params.mode,
|
|
@@ -35296,9 +35666,18 @@ function requireKnownMode(params) {
|
|
|
35296
35666
|
{ mode: String(params.mode), modes: [...SEARCH_MODES] }
|
|
35297
35667
|
);
|
|
35298
35668
|
}
|
|
35669
|
+
function requireBooleanPartialWords(params) {
|
|
35670
|
+
const setting = params.partialWords;
|
|
35671
|
+
if (setting === void 0 || typeof setting === "boolean") return;
|
|
35672
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The "partialWords" query option takes true or false', {
|
|
35673
|
+
option: "partialWords",
|
|
35674
|
+
received: setting === null ? "null" : typeof setting
|
|
35675
|
+
});
|
|
35676
|
+
}
|
|
35299
35677
|
function requireValidQueryOptions(params, config) {
|
|
35300
35678
|
const strict = config.strict === true;
|
|
35301
35679
|
requireValidTypoTolerance(params);
|
|
35680
|
+
requireBooleanPartialWords(params);
|
|
35302
35681
|
requireSortableFields(params.sort, config.schema, strict);
|
|
35303
35682
|
requireValidQueryFilter(params.filters, config.schema);
|
|
35304
35683
|
requireUsableQueryFields(params, config.schema, strict);
|
|
@@ -35650,6 +36029,31 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
|
|
|
35650
36029
|
if (needsTrailingEllipsis) snippet = `${snippet}...`;
|
|
35651
36030
|
return { snippet, visiblePositions };
|
|
35652
36031
|
}
|
|
36032
|
+
function containsPartialWord(spelling, words) {
|
|
36033
|
+
for (const word of words) {
|
|
36034
|
+
if (spelling.length > word.length && spelling.includes(word)) return true;
|
|
36035
|
+
}
|
|
36036
|
+
return false;
|
|
36037
|
+
}
|
|
36038
|
+
function partialWordMatcher(text, language, partial) {
|
|
36039
|
+
const indexed = tokenize(text, language, {
|
|
36040
|
+
stem: true,
|
|
36041
|
+
removeStopWords: true,
|
|
36042
|
+
collectSurfaces: true,
|
|
36043
|
+
customTokenizer: partial.customTokenizer,
|
|
36044
|
+
stopWordOverride: partial.stopWordOverride
|
|
36045
|
+
});
|
|
36046
|
+
const positions = /* @__PURE__ */ new Set();
|
|
36047
|
+
const spellings = /* @__PURE__ */ new Set();
|
|
36048
|
+
for (let at = 0; at < indexed.tokens.length; at++) {
|
|
36049
|
+
const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
|
|
36050
|
+
if (!containsPartialWord(spelling, partial.words)) continue;
|
|
36051
|
+
positions.add(indexed.tokens[at].position);
|
|
36052
|
+
spellings.add(spelling);
|
|
36053
|
+
}
|
|
36054
|
+
if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
|
|
36055
|
+
return (position) => positions.has(position);
|
|
36056
|
+
}
|
|
35653
36057
|
function highlightField(text, queryTokens, language, options) {
|
|
35654
36058
|
const preTag = options?.preTag ?? "<mark>";
|
|
35655
36059
|
const postTag = options?.postTag ?? "</mark>";
|
|
@@ -35686,11 +36090,13 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
35686
36090
|
matchByFieldToken.set(fieldToken, matches);
|
|
35687
36091
|
return matches;
|
|
35688
36092
|
};
|
|
36093
|
+
const matchesInside = options?.partialWords === void 0 ? null : partialWordMatcher(text, language, options.partialWords);
|
|
35689
36094
|
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
35690
36095
|
if (i >= charOffsets.length) break;
|
|
35691
36096
|
const fieldToken = fieldResult.tokens[i].token;
|
|
35692
36097
|
const prefixMatch = prefixToken !== void 0 && fieldToken.startsWith(prefixToken);
|
|
35693
|
-
|
|
36098
|
+
const insideMatch = matchesInside?.(fieldResult.tokens[i].position, fieldToken) === true;
|
|
36099
|
+
if (prefixMatch || insideMatch || matchesQuery(fieldToken)) {
|
|
35694
36100
|
matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
|
|
35695
36101
|
}
|
|
35696
36102
|
}
|
|
@@ -35722,7 +36128,39 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
35722
36128
|
}
|
|
35723
36129
|
|
|
35724
36130
|
// src/engine/query/highlight.ts
|
|
35725
|
-
function
|
|
36131
|
+
function writtenWordsOf(params, language, analysis) {
|
|
36132
|
+
return tokenize(params.term ?? "", language, {
|
|
36133
|
+
stem: false,
|
|
36134
|
+
removeStopWords: true,
|
|
36135
|
+
customTokenizer: analysis.customTokenizer,
|
|
36136
|
+
stopWordOverride: analysis.stopWords
|
|
36137
|
+
}).tokens;
|
|
36138
|
+
}
|
|
36139
|
+
function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
36140
|
+
if (params.partialWords !== true || params.exact === true) return null;
|
|
36141
|
+
const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
|
|
36142
|
+
if (fields.length === 0) return null;
|
|
36143
|
+
const writtenAt = /* @__PURE__ */ new Map();
|
|
36144
|
+
for (const word of written) {
|
|
36145
|
+
if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
|
|
36146
|
+
}
|
|
36147
|
+
const seenTokens = /* @__PURE__ */ new Set();
|
|
36148
|
+
const words = [];
|
|
36149
|
+
for (const queryToken of queryTokens) {
|
|
36150
|
+
if (seenTokens.has(queryToken.token)) continue;
|
|
36151
|
+
seenTokens.add(queryToken.token);
|
|
36152
|
+
const word = writtenAt.get(queryToken.position);
|
|
36153
|
+
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
36154
|
+
words.push(word);
|
|
36155
|
+
if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
36156
|
+
}
|
|
36157
|
+
if (words.length === 0) return null;
|
|
36158
|
+
return {
|
|
36159
|
+
fields: new Set(fields),
|
|
36160
|
+
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
36161
|
+
};
|
|
36162
|
+
}
|
|
36163
|
+
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument) {
|
|
35726
36164
|
if (!params.highlight) return;
|
|
35727
36165
|
const queryTokenResult = tokenize(params.term ?? "", language, {
|
|
35728
36166
|
stem: true,
|
|
@@ -35730,17 +36168,15 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
35730
36168
|
customTokenizer: analysis.customTokenizer,
|
|
35731
36169
|
stopWordOverride: analysis.stopWords
|
|
35732
36170
|
});
|
|
36171
|
+
const queryTokens = queryTokenResult.tokens;
|
|
36172
|
+
const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
|
|
36173
|
+
const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
|
|
36174
|
+
const written = usesPrefix || usesPartialWords ? writtenWordsOf(params, language, analysis) : [];
|
|
35733
36175
|
let prefixToken;
|
|
35734
36176
|
let completedToken;
|
|
35735
|
-
if (
|
|
35736
|
-
const lastToken =
|
|
35737
|
-
|
|
35738
|
-
stem: false,
|
|
35739
|
-
removeStopWords: true,
|
|
35740
|
-
customTokenizer: analysis.customTokenizer,
|
|
35741
|
-
stopWordOverride: analysis.stopWords
|
|
35742
|
-
});
|
|
35743
|
-
for (const t of unstemmed.tokens) {
|
|
36177
|
+
if (usesPrefix) {
|
|
36178
|
+
const lastToken = queryTokens[queryTokens.length - 1];
|
|
36179
|
+
for (const t of written) {
|
|
35744
36180
|
if (t.position === lastToken.position) {
|
|
35745
36181
|
prefixToken = t.token;
|
|
35746
36182
|
completedToken = lastToken.token;
|
|
@@ -35748,6 +36184,7 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
35748
36184
|
}
|
|
35749
36185
|
}
|
|
35750
36186
|
}
|
|
36187
|
+
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
35751
36188
|
for (const hit of hits) {
|
|
35752
36189
|
const highlights = {};
|
|
35753
36190
|
const projected = hit.document;
|
|
@@ -35759,14 +36196,15 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
35759
36196
|
fieldValue = stored?.[field];
|
|
35760
36197
|
}
|
|
35761
36198
|
if (typeof fieldValue === "string") {
|
|
35762
|
-
highlights[field] = highlightField(fieldValue,
|
|
36199
|
+
highlights[field] = highlightField(fieldValue, queryTokens, language, {
|
|
35763
36200
|
preTag: params.highlight.preTag,
|
|
35764
36201
|
postTag: params.highlight.postTag,
|
|
35765
36202
|
maxSnippetLength: params.highlight.maxSnippetLength,
|
|
35766
36203
|
prefixToken,
|
|
35767
36204
|
completedToken,
|
|
35768
36205
|
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
35769
|
-
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
36206
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
36207
|
+
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0
|
|
35770
36208
|
});
|
|
35771
36209
|
}
|
|
35772
36210
|
}
|
|
@@ -36276,7 +36714,7 @@ async function executeQuery(params, callerContext) {
|
|
|
36276
36714
|
if (params.highlight) {
|
|
36277
36715
|
const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
|
|
36278
36716
|
const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
|
|
36279
|
-
applyHighlights(paginated, params, language, manager.analysis, readStoredDocument);
|
|
36717
|
+
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument);
|
|
36280
36718
|
}
|
|
36281
36719
|
const elapsed = now() - startTime;
|
|
36282
36720
|
return {
|
|
@@ -36868,4 +37306,4 @@ async function createNarsil(config) {
|
|
|
36868
37306
|
// src/index.ts
|
|
36869
37307
|
var VERSION = "0.1.0";
|
|
36870
37308
|
|
|
36871
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerPatternSearch, registerStopWords, registerTokenizer, resetNormalizationCache };
|
|
37309
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerTokenizer, resetNormalizationCache };
|