@delali/narsil 0.3.1-next.73 → 0.3.1-next.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{capabilities-CYeAWYxQ.d.ts → capabilities-CAMzmkAg.d.ts} +2 -2
- package/dist/{chunk-PU43WDY4.mjs → chunk-2E35UM3G.mjs} +1 -13
- package/dist/chunk-3DDUNI6N.mjs +48 -0
- package/dist/{chunk-THMSFOJO.mjs → chunk-BVBIQQKG.mjs} +576 -547
- package/dist/{chunk-ZULSEZCE.mjs → chunk-CTOSGRM7.mjs} +100 -19
- package/dist/{chunk-CVFAFZSN.mjs → chunk-CUZOX6CN.mjs} +23 -15
- package/dist/chunk-I23G3IZC.mjs +593 -0
- package/dist/chunk-KEHK2O6P.mjs +13 -0
- package/dist/{chunk-AFKH4ZK3.mjs → chunk-LX4JERXI.mjs} +1 -1
- package/dist/{chunk-LPBWCHBE.mjs → chunk-NOZUQM7J.mjs} +2 -2
- package/dist/{chunk-WVDFG45M.mjs → chunk-PO53FV4Q.mjs} +2 -1
- package/dist/chunk-T5ZWEWVD.mjs +658 -0
- package/dist/{chunk-2WLD2NFW.mjs → chunk-TVFLHM7M.mjs} +2 -2
- package/dist/{chunk-3YKWSO54.mjs → chunk-VKX2BPXD.mjs} +147 -647
- package/dist/{chunk-AK3BLGCI.mjs → chunk-Z3DQPOJQ.mjs} +15 -48
- package/dist/{client-D698E5ge.d.ts → client-C-I70ArH.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/{config-Bv8etwUZ.d.ts → config-C6Iw5rCX.d.ts} +2 -2
- package/dist/{config-QXU3HMpq.d.ts → config-DJPH3IQB.d.ts} +2 -2
- package/dist/direct-executor-KFAGILJC.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +19 -9
- package/dist/{engine-D_OzKZ-6.d.ts → engine-B8eRPO7m.d.ts} +2 -2
- package/dist/index.browser.mjs +1200 -808
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +12 -8
- package/dist/{memory-B4jMmygw.d.ts → memory-DJEwWoCi.d.ts} +1205 -1180
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +11 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +33 -13
- package/dist/persistence/durability/checkpoint-worker.mjs +12 -8
- package/dist/react.d.ts +7 -7
- package/dist/{schema-BBOCdhND.d.ts → schema-DgyDijhf.d.ts} +6 -2
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -6
- package/dist/{tasks-DpgCpW1x.d.ts → tasks-BVmAKYBh.d.ts} +1 -1
- package/dist/{thread-3LNVA43V.mjs → thread-W6DPKEF6.mjs} +8 -5
- package/dist/{types-BiUmaiWT.d.ts → types-ChrVy4lH.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +3 -3
- package/package.json +15 -7
- package/dist/direct-executor-5SRDFZAS.mjs +0 -13
package/dist/index.browser.mjs
CHANGED
|
@@ -209,10 +209,302 @@ function resolveIndexAnalysis(config) {
|
|
|
209
209
|
};
|
|
210
210
|
}
|
|
211
211
|
|
|
212
|
-
// src/core/
|
|
213
|
-
function
|
|
212
|
+
// src/core/partition/partial-words/platform.browser.ts
|
|
213
|
+
function platformPartialWordSearch() {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// src/core/ordering/code-points.ts
|
|
218
|
+
function exceedsCodePoints(value, maximum) {
|
|
219
|
+
if (value.length <= maximum) return false;
|
|
220
|
+
let counted = 0;
|
|
221
|
+
let i = 0;
|
|
222
|
+
while (i < value.length) {
|
|
223
|
+
const unit = value.charCodeAt(i);
|
|
224
|
+
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
225
|
+
const low = value.charCodeAt(i + 1);
|
|
226
|
+
if (low >= 56320 && low < 57344) i++;
|
|
227
|
+
}
|
|
228
|
+
i++;
|
|
229
|
+
counted++;
|
|
230
|
+
if (counted > maximum) return true;
|
|
231
|
+
}
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
var SURROGATE_UNIT = /[\ud800-\udfff]/;
|
|
235
|
+
function holdsUnpairedSurrogate(value) {
|
|
236
|
+
if (!SURROGATE_UNIT.test(value)) return false;
|
|
237
|
+
for (let i = 0; i < value.length; i++) {
|
|
238
|
+
const unit = value.charCodeAt(i);
|
|
239
|
+
if (unit < 55296 || unit >= 57344) continue;
|
|
240
|
+
if (unit >= 56320) return true;
|
|
241
|
+
const low = value.charCodeAt(i + 1);
|
|
242
|
+
if (!(low >= 56320 && low < 57344)) return true;
|
|
243
|
+
i++;
|
|
244
|
+
}
|
|
245
|
+
return false;
|
|
246
|
+
}
|
|
247
|
+
function codePointCount(value) {
|
|
248
|
+
let counted = 0;
|
|
249
|
+
for (let i = 0; i < value.length; i++) {
|
|
250
|
+
const unit = value.charCodeAt(i);
|
|
251
|
+
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
252
|
+
const low = value.charCodeAt(i + 1);
|
|
253
|
+
if (low >= 56320 && low < 57344) i++;
|
|
254
|
+
}
|
|
255
|
+
counted++;
|
|
256
|
+
}
|
|
257
|
+
return counted;
|
|
258
|
+
}
|
|
259
|
+
function adjustUtf16Unit(unit) {
|
|
260
|
+
if (unit < 55296) return unit;
|
|
261
|
+
if (unit < 57344) return unit + 8192;
|
|
262
|
+
return unit - 2048;
|
|
263
|
+
}
|
|
264
|
+
function compareCodePoints(a, b) {
|
|
265
|
+
const shorter = a.length < b.length ? a.length : b.length;
|
|
266
|
+
let i = 0;
|
|
267
|
+
while (i < shorter && a.charCodeAt(i) === b.charCodeAt(i)) i++;
|
|
268
|
+
if (i === shorter) return a.length - b.length;
|
|
269
|
+
return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// src/core/ordering/fold-table.ts
|
|
273
|
+
var SINGLE_FOLDS = "1t,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2j,lj;b,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;y,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3d;1,1;2,1;2,1;2,-7g;2,5u;1,1;2,1;2,5q;1,1;2,5p;1,5p;1,1;3,27;1,5m;1,5n;1,1;2,5p;1,5r;2,5v;1,5t;1,1;4,5v;1,5x;2,5y;1,1;2,1;2,1;2,62;1,1;2,62;3,1;2,62;1,1;2,61;1,61;1,1;2,1;2,63;1,1;4,1;8,2;1,1;2,2;1,1;2,2;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,2;1,1;2,1;2,-2p;1,-1k;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;8,8bv;1,1;2,-4j;1,8bs;3,1;2,-5f;1,1x;1,1z;1,1;2,1;2,1;2,1;2,1;6v,38;17,1;2,1;4,1;9,38;7,12;2,11;1,11;1,11;2,1s;2,1r;1,1r;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;n,1;d,8;1,-u;1,-p;4,-f;1,-m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1i;1,-1c;3,-1o;1,-1s;2,1;2,-7;1,1;3,-3m;1,-3m;1,-3m;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1d,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,f;1,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;28a,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;2,5ls;6,5ls;mj,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1oj,-4su;1,-4st;1,-4sk;1,-4si;1,-4si;1,-4sj;1,-4sc;1,-4ro;1,r7n;1,1;7,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;3,-2bk;1,-2bk;1,-2bk;8x,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;7,-1m;5,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;b,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;c,-8;2,-8;2,-8;2,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;21,-8;1,-8;1,-22;1,-22;3,-5j9;a,-2e;1,-2e;1,-2e;1,-2e;d,-8;1,-8;1,-2s;1,-2s;d,-8;1,-8;1,-34;1,-34;1,-7;c,-3k;1,-3k;1,-3i;1,-3i;8b,-5st;4,-6gv;1,-6di;7,s;1a,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;k,1;mr,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1f5,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1d,1;2,-8af;1,-2xy;1,-89z;3,1;2,1;2,1;2,-8bg;1,-8al;1,-8bj;1,-8bi;2,1;3,1;9,-8cf;1,-8cf;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;9,1;2,1;5,1;nym,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;k,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3s,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;b,1;2,1;2,-r9g;1,1;2,1;2,1;2,1;2,1;5,1;2,-wmg;3,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wn8;1,-wnj;1,-wnf;1,-wn5;1,-wn8;2,-wlu;1,-wmi;1,-wlx;1,ps;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1c;1,-wn7;1,-raw;1,1;2,1;2,-wo7;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wu9;p,1;or,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;ggy,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;xy,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;3t,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;4d,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1d7,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;4e,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;27v,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;gw1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1t,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;o5k,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;";
|
|
274
|
+
var MULTI_FOLDS = "67,37,37;29,2x,lj;p,jg,32;4n,2y,lo;bk,qh,lk,ld;w,qt,lk,ld;d3,12d,136;4y7,2w,mp;1,38,lk;1,3b,lm;1,3d,lm;1,2p,ji;4,37,37;4y,qt,lv;2,qt,lv,lc;2,qt,lv,ld;2,qt,lv,n6;16,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;3,67k,qh;1,q9,qh;1,q4,qh;2,q9,n6;1,q9,n6,qh;5,q9,qh;6,67o,qh;1,qf,qh;1,q6,qh;2,qf,n6;1,qf,n6,qh;5,qf,qh;6,qh,lk,lc;1,qh,lk,ld;3,qh,n6;1,qh,lk,n6;b,qt,lk,lc;1,qt,lk,ld;1,qp,lv;2,qt,n6;1,qt,lk,n6;b,67w,qh;1,qx,qh;1,r2,qh;2,qx,n6;1,qx,n6,qh;5,qx,qh;179g,2u,2u;1,2u,2x;1,2u,30;1,2u,2u,2x;1,2u,2u,30;1,37,38;1,37,38;d,12s,12u;1,12s,12d;1,12s,12j;1,132,12u;1,12s,12l;";
|
|
275
|
+
var singleTable = null;
|
|
276
|
+
var multiTable = null;
|
|
277
|
+
function singleFoldTable() {
|
|
278
|
+
if (singleTable === null) {
|
|
279
|
+
singleTable = /* @__PURE__ */ new Map();
|
|
280
|
+
let source = 0;
|
|
281
|
+
for (const entry of SINGLE_FOLDS.split(";")) {
|
|
282
|
+
if (entry.length === 0) continue;
|
|
283
|
+
const [delta, offset] = entry.split(",");
|
|
284
|
+
source += Number.parseInt(delta, 36);
|
|
285
|
+
singleTable.set(source, source + Number.parseInt(offset, 36));
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
return singleTable;
|
|
289
|
+
}
|
|
290
|
+
function multiFoldTable() {
|
|
291
|
+
if (multiTable === null) {
|
|
292
|
+
multiTable = /* @__PURE__ */ new Map();
|
|
293
|
+
let source = 0;
|
|
294
|
+
for (const entry of MULTI_FOLDS.split(";")) {
|
|
295
|
+
if (entry.length === 0) continue;
|
|
296
|
+
const parts = entry.split(",");
|
|
297
|
+
source += Number.parseInt(parts[0], 36);
|
|
298
|
+
multiTable.set(
|
|
299
|
+
source,
|
|
300
|
+
parts.slice(1).map((cp) => Number.parseInt(cp, 36))
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return multiTable;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// src/core/ordering/fold-compare.ts
|
|
308
|
+
var EXHAUSTED = -1;
|
|
309
|
+
function foldOf(codePoint, single, multi) {
|
|
310
|
+
if (codePoint < 128) return codePoint >= 65 && codePoint <= 90 ? codePoint + 32 : codePoint;
|
|
311
|
+
return single.get(codePoint) ?? multi.get(codePoint) ?? codePoint;
|
|
312
|
+
}
|
|
313
|
+
function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
314
|
+
if (cursor.pending !== null) {
|
|
315
|
+
const cp2 = cursor.pending[cursor.pendingIndex];
|
|
316
|
+
cursor.pendingIndex++;
|
|
317
|
+
if (cursor.pendingIndex === cursor.pending.length) {
|
|
318
|
+
cursor.pending = null;
|
|
319
|
+
cursor.pendingIndex = 0;
|
|
320
|
+
}
|
|
321
|
+
return cp2;
|
|
322
|
+
}
|
|
323
|
+
if (cursor.index >= text.length) return EXHAUSTED;
|
|
324
|
+
let cp = text.charCodeAt(cursor.index);
|
|
325
|
+
cursor.index++;
|
|
326
|
+
if (cp >= 55296 && cp < 56320 && cursor.index < text.length) {
|
|
327
|
+
const low = text.charCodeAt(cursor.index);
|
|
328
|
+
if (low >= 56320 && low < 57344) {
|
|
329
|
+
cp = 65536 + (cp - 55296 << 10) + (low - 56320);
|
|
330
|
+
cursor.index++;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const folded = foldOf(cp, single, multi);
|
|
334
|
+
if (typeof folded === "number") return folded;
|
|
335
|
+
if (folded.length > 1) {
|
|
336
|
+
cursor.pending = folded;
|
|
337
|
+
cursor.pendingIndex = 1;
|
|
338
|
+
}
|
|
339
|
+
return folded[0];
|
|
340
|
+
}
|
|
341
|
+
function compareCaseFolded(a, b) {
|
|
342
|
+
const shorter = a.length < b.length ? a.length : b.length;
|
|
343
|
+
let start = 0;
|
|
344
|
+
while (start < shorter && a.charCodeAt(start) === b.charCodeAt(start)) start++;
|
|
345
|
+
if (start === a.length && start === b.length) return 0;
|
|
346
|
+
if (start > 0) {
|
|
347
|
+
const previous = a.charCodeAt(start - 1);
|
|
348
|
+
if (previous >= 55296 && previous < 56320) start--;
|
|
349
|
+
}
|
|
350
|
+
const single = singleFoldTable();
|
|
351
|
+
const multi = multiFoldTable();
|
|
352
|
+
const aCursor = { index: start, pending: null, pendingIndex: 0 };
|
|
353
|
+
const bCursor = { index: start, pending: null, pendingIndex: 0 };
|
|
354
|
+
for (; ; ) {
|
|
355
|
+
const aCp = nextFoldedCodePoint(a, aCursor, single, multi);
|
|
356
|
+
const bCp = nextFoldedCodePoint(b, bCursor, single, multi);
|
|
357
|
+
if (aCp !== bCp) return aCp - bCp;
|
|
358
|
+
if (aCp === EXHAUSTED) return 0;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
function compareSortStrings(a, b) {
|
|
362
|
+
const folded = compareCaseFolded(a, b);
|
|
363
|
+
if (folded !== 0) return folded;
|
|
364
|
+
return compareCodePoints(a, b);
|
|
365
|
+
}
|
|
366
|
+
function appendFoldedCodePoint(codePoint, out) {
|
|
367
|
+
const folded = foldOf(codePoint, singleFoldTable(), multiFoldTable());
|
|
368
|
+
if (typeof folded === "number") {
|
|
369
|
+
out.push(folded);
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
for (const foldedCodePoint of folded) out.push(foldedCodePoint);
|
|
373
|
+
}
|
|
374
|
+
function appendFoldedCodePoints(text, out) {
|
|
375
|
+
const single = singleFoldTable();
|
|
376
|
+
const multi = multiFoldTable();
|
|
377
|
+
const cursor = { index: 0, pending: null, pendingIndex: 0 };
|
|
378
|
+
for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
|
|
379
|
+
out.push(codePoint);
|
|
380
|
+
codePoint = nextFoldedCodePoint(text, cursor, single, multi);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
function foldedCodePointCount(text) {
|
|
384
|
+
const folded = [];
|
|
385
|
+
appendFoldedCodePoints(text, folded);
|
|
386
|
+
return folded.length;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// src/core/ordering/constants.ts
|
|
390
|
+
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
391
|
+
|
|
392
|
+
// src/core/ordering/sort-values.ts
|
|
393
|
+
var SORT_MODES = ["min", "max", "avg", "median"];
|
|
394
|
+
function isSortMode(value) {
|
|
395
|
+
return value === "min" || value === "max" || value === "avg" || value === "median";
|
|
396
|
+
}
|
|
397
|
+
function defaultSortMode(direction) {
|
|
398
|
+
return direction === "desc" ? "max" : "min";
|
|
399
|
+
}
|
|
400
|
+
function sortModeOf(entry) {
|
|
401
|
+
return entry.mode ?? defaultSortMode(entry.direction);
|
|
402
|
+
}
|
|
403
|
+
function truncateSortString(value) {
|
|
404
|
+
if (value.length <= SORT_VALUE_MAX_CODE_POINTS) return value;
|
|
405
|
+
let index = 0;
|
|
406
|
+
let counted = 0;
|
|
407
|
+
while (index < value.length && counted < SORT_VALUE_MAX_CODE_POINTS) {
|
|
408
|
+
const unit = value.charCodeAt(index);
|
|
409
|
+
if (unit >= 55296 && unit < 56320 && index + 1 < value.length) {
|
|
410
|
+
const low = value.charCodeAt(index + 1);
|
|
411
|
+
if (low >= 56320 && low < 57344) index++;
|
|
412
|
+
}
|
|
413
|
+
index++;
|
|
414
|
+
counted++;
|
|
415
|
+
}
|
|
416
|
+
return value.slice(0, index);
|
|
417
|
+
}
|
|
418
|
+
function toComparableSortValue(value) {
|
|
419
|
+
if (typeof value === "string") return truncateSortString(value);
|
|
420
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
421
|
+
if (typeof value === "boolean") return value;
|
|
214
422
|
return null;
|
|
215
423
|
}
|
|
424
|
+
function numericReduction(values, mode) {
|
|
425
|
+
const numbers = [];
|
|
426
|
+
for (const element of values) {
|
|
427
|
+
if (typeof element === "number" && Number.isFinite(element)) numbers.push(element);
|
|
428
|
+
}
|
|
429
|
+
if (numbers.length === 0) return null;
|
|
430
|
+
if (mode === "avg") {
|
|
431
|
+
let mean = 0;
|
|
432
|
+
for (let index = 0; index < numbers.length; index++) {
|
|
433
|
+
mean += (numbers[index] - mean) / (index + 1);
|
|
434
|
+
}
|
|
435
|
+
return mean;
|
|
436
|
+
}
|
|
437
|
+
numbers.sort((a, b) => a - b);
|
|
438
|
+
const middle = numbers.length >> 1;
|
|
439
|
+
if (numbers.length % 2 === 1) return numbers[middle];
|
|
440
|
+
return numbers[middle - 1] / 2 + numbers[middle] / 2;
|
|
441
|
+
}
|
|
442
|
+
function toReducedSortValue(value, mode) {
|
|
443
|
+
if (!Array.isArray(value)) return toComparableSortValue(value);
|
|
444
|
+
if (mode === "avg" || mode === "median") return numericReduction(value, mode);
|
|
445
|
+
let chosen = null;
|
|
446
|
+
for (const element of value) {
|
|
447
|
+
const comparable = toComparableSortValue(element);
|
|
448
|
+
if (comparable === null) continue;
|
|
449
|
+
if (chosen === null) {
|
|
450
|
+
chosen = comparable;
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
const comparison = comparePresentValues(comparable, chosen);
|
|
454
|
+
if (mode === "min" ? comparison < 0 : comparison > 0) chosen = comparable;
|
|
455
|
+
}
|
|
456
|
+
return chosen;
|
|
457
|
+
}
|
|
458
|
+
function readSortField(document2, path) {
|
|
459
|
+
const segments = path.split(".");
|
|
460
|
+
let current = document2;
|
|
461
|
+
for (const segment of segments) {
|
|
462
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
463
|
+
if (!Object.hasOwn(current, segment)) return void 0;
|
|
464
|
+
current = current[segment];
|
|
465
|
+
}
|
|
466
|
+
return current;
|
|
467
|
+
}
|
|
468
|
+
function typeRank(value) {
|
|
469
|
+
if (typeof value === "number") return 0;
|
|
470
|
+
if (typeof value === "string") return 1;
|
|
471
|
+
return 2;
|
|
472
|
+
}
|
|
473
|
+
function comparePresentValues(a, b) {
|
|
474
|
+
const rankDifference = typeRank(a) - typeRank(b);
|
|
475
|
+
if (rankDifference !== 0) return rankDifference;
|
|
476
|
+
if (typeof a === "number" && typeof b === "number") return a - b;
|
|
477
|
+
if (typeof a === "string" && typeof b === "string") return compareSortStrings(a, b);
|
|
478
|
+
if (a === b) return 0;
|
|
479
|
+
return a ? 1 : -1;
|
|
480
|
+
}
|
|
481
|
+
function compareComparableValues(a, b, direction) {
|
|
482
|
+
if (a === null && b === null) return 0;
|
|
483
|
+
if (a === null) return 1;
|
|
484
|
+
if (b === null) return -1;
|
|
485
|
+
const comparison = comparePresentValues(a, b);
|
|
486
|
+
if (comparison === 0) return 0;
|
|
487
|
+
return direction === "desc" ? -comparison : comparison;
|
|
488
|
+
}
|
|
489
|
+
function compareComparableKeys(aKey, bKey, directions) {
|
|
490
|
+
for (let i = 0; i < directions.length; i++) {
|
|
491
|
+
const comparison = compareComparableValues(aKey[i] ?? null, bKey[i] ?? null, directions[i]);
|
|
492
|
+
if (comparison !== 0) return comparison;
|
|
493
|
+
}
|
|
494
|
+
return 0;
|
|
495
|
+
}
|
|
496
|
+
function compareSortValues(aValues, bValues, directions) {
|
|
497
|
+
for (let i = 0; i < directions.length; i++) {
|
|
498
|
+
const a = toComparableSortValue(aValues[i]);
|
|
499
|
+
const b = toComparableSortValue(bValues[i]);
|
|
500
|
+
if (a === null && b === null) continue;
|
|
501
|
+
if (a === null) return 1;
|
|
502
|
+
if (b === null) return -1;
|
|
503
|
+
const comparison = comparePresentValues(a, b);
|
|
504
|
+
if (comparison !== 0) return directions[i] === "desc" ? -comparison : comparison;
|
|
505
|
+
}
|
|
506
|
+
return 0;
|
|
507
|
+
}
|
|
216
508
|
|
|
217
509
|
// src/schema/constants.ts
|
|
218
510
|
var MAX_NESTING_DEPTH = 4;
|
|
@@ -477,6 +769,18 @@ function schemaFieldsOf(schema) {
|
|
|
477
769
|
parsedSchemas.set(schema, frozen);
|
|
478
770
|
return frozen;
|
|
479
771
|
}
|
|
772
|
+
var partialFieldsBySchema = /* @__PURE__ */ new WeakMap();
|
|
773
|
+
function partialFieldsOf(schema) {
|
|
774
|
+
const cached = partialFieldsBySchema.get(schema);
|
|
775
|
+
if (cached !== void 0) return cached;
|
|
776
|
+
const paths = [];
|
|
777
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
778
|
+
if (field.partial) paths.push(field.path);
|
|
779
|
+
}
|
|
780
|
+
const frozen = Object.freeze(paths.sort(compareCodePoints));
|
|
781
|
+
partialFieldsBySchema.set(schema, frozen);
|
|
782
|
+
return frozen;
|
|
783
|
+
}
|
|
480
784
|
function extractVectorFieldsFromSchema(schema) {
|
|
481
785
|
const result = /* @__PURE__ */ new Map();
|
|
482
786
|
for (const field of schemaFieldsOf(schema)) {
|
|
@@ -485,8 +789,68 @@ function extractVectorFieldsFromSchema(schema) {
|
|
|
485
789
|
return result;
|
|
486
790
|
}
|
|
487
791
|
|
|
488
|
-
// src/core/
|
|
792
|
+
// src/core/partition/partial-words/registry.ts
|
|
489
793
|
var registered = null;
|
|
794
|
+
function isPartialWordSearchModule(value) {
|
|
795
|
+
return value !== null && typeof value === "object" && value.name === "partialWords" && "createGrowingSlot" in value && typeof value.createGrowingSlot === "function" && "createFrozenSlot" in value && typeof value.createFrozenSlot === "function" && "buildTable" in value && typeof value.buildTable === "function" && "expand" in value && typeof value.expand === "function";
|
|
796
|
+
}
|
|
797
|
+
function registerPartialWordSearch(module) {
|
|
798
|
+
if (!isPartialWordSearchModule(module)) {
|
|
799
|
+
throw new NarsilError(
|
|
800
|
+
ErrorCodes.CONFIG_INVALID,
|
|
801
|
+
'Pass registerPartialWordSearch the partialWordSearch object that "@delali/narsil/partial-words" exports'
|
|
802
|
+
);
|
|
803
|
+
}
|
|
804
|
+
registered = module;
|
|
805
|
+
}
|
|
806
|
+
function availablePartialWordSearch() {
|
|
807
|
+
return registered ?? platformPartialWordSearch();
|
|
808
|
+
}
|
|
809
|
+
function requirePartialWordSearchFor(schema) {
|
|
810
|
+
const fields = partialFieldsOf(schema);
|
|
811
|
+
if (fields.length === 0 || availablePartialWordSearch() !== null) return;
|
|
812
|
+
throw new NarsilError(
|
|
813
|
+
ErrorCodes.CONFIG_INVALID,
|
|
814
|
+
`The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load the index`,
|
|
815
|
+
{ field: fields[0] }
|
|
816
|
+
);
|
|
817
|
+
}
|
|
818
|
+
var EMPTY_GROWING_SLOT = {
|
|
819
|
+
readerFor: () => null,
|
|
820
|
+
spellings: () => null,
|
|
821
|
+
bytes: () => 0,
|
|
822
|
+
afterDocument: () => void 0,
|
|
823
|
+
afterMerge: () => void 0,
|
|
824
|
+
forgetSpellings: () => void 0,
|
|
825
|
+
clear: () => void 0
|
|
826
|
+
};
|
|
827
|
+
var EMPTY_FROZEN_SLOT = {
|
|
828
|
+
readerFor: () => null,
|
|
829
|
+
spellings: () => null,
|
|
830
|
+
bytes: () => 0,
|
|
831
|
+
afterRemoval: () => void 0
|
|
832
|
+
};
|
|
833
|
+
function createGrowingPartialWords(view) {
|
|
834
|
+
return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
|
|
835
|
+
}
|
|
836
|
+
function createFrozenPartialWords(initial, tokenTable, documents) {
|
|
837
|
+
return availablePartialWordSearch()?.createFrozenSlot(initial, tokenTable, documents) ?? EMPTY_FROZEN_SLOT;
|
|
838
|
+
}
|
|
839
|
+
function buildPartialWordTableFrom(source, tokenTable) {
|
|
840
|
+
return availablePartialWordSearch()?.buildTable(source, tokenTable) ?? null;
|
|
841
|
+
}
|
|
842
|
+
function expandPartialWords(subs, analysis, requests) {
|
|
843
|
+
const module = availablePartialWordSearch();
|
|
844
|
+
return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
// src/core/pattern-index/platform.browser.ts
|
|
848
|
+
function platformPatternSearch() {
|
|
849
|
+
return null;
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
// src/core/pattern-index/registry.ts
|
|
853
|
+
var registered2 = null;
|
|
490
854
|
function isPatternSearchModule(value) {
|
|
491
855
|
return value !== null && typeof value === "object" && value.name === "pattern" && "createIndex" in value && typeof value.createIndex === "function" && "readArrays" in value && typeof value.readArrays === "function" && "mergeArrays" in value && typeof value.mergeArrays === "function" && "matchInto" in value && typeof value.matchInto === "function";
|
|
492
856
|
}
|
|
@@ -497,10 +861,10 @@ function registerPatternSearch(module) {
|
|
|
497
861
|
'registerPatternSearch takes the patternSearch object that "@delali/narsil/pattern" exports'
|
|
498
862
|
);
|
|
499
863
|
}
|
|
500
|
-
|
|
864
|
+
registered2 = module;
|
|
501
865
|
}
|
|
502
866
|
function availablePatternSearch() {
|
|
503
|
-
return
|
|
867
|
+
return registered2 ?? platformPatternSearch();
|
|
504
868
|
}
|
|
505
869
|
function requirePatternSearchFor(schema) {
|
|
506
870
|
for (const { path, pattern } of schemaFieldsOf(schema)) {
|
|
@@ -2269,61 +2633,6 @@ var PATTERN_WORK_FLUSH_UNITS = 65536;
|
|
|
2269
2633
|
var PATTERN_RUN_CODE_POINTS = 3;
|
|
2270
2634
|
var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "contains"];
|
|
2271
2635
|
|
|
2272
|
-
// src/core/ordering/code-points.ts
|
|
2273
|
-
function exceedsCodePoints(value, maximum) {
|
|
2274
|
-
if (value.length <= maximum) return false;
|
|
2275
|
-
let counted = 0;
|
|
2276
|
-
let i = 0;
|
|
2277
|
-
while (i < value.length) {
|
|
2278
|
-
const unit = value.charCodeAt(i);
|
|
2279
|
-
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
2280
|
-
const low = value.charCodeAt(i + 1);
|
|
2281
|
-
if (low >= 56320 && low < 57344) i++;
|
|
2282
|
-
}
|
|
2283
|
-
i++;
|
|
2284
|
-
counted++;
|
|
2285
|
-
if (counted > maximum) return true;
|
|
2286
|
-
}
|
|
2287
|
-
return false;
|
|
2288
|
-
}
|
|
2289
|
-
var SURROGATE_UNIT = /[\ud800-\udfff]/;
|
|
2290
|
-
function holdsUnpairedSurrogate(value) {
|
|
2291
|
-
if (!SURROGATE_UNIT.test(value)) return false;
|
|
2292
|
-
for (let i = 0; i < value.length; i++) {
|
|
2293
|
-
const unit = value.charCodeAt(i);
|
|
2294
|
-
if (unit < 55296 || unit >= 57344) continue;
|
|
2295
|
-
if (unit >= 56320) return true;
|
|
2296
|
-
const low = value.charCodeAt(i + 1);
|
|
2297
|
-
if (!(low >= 56320 && low < 57344)) return true;
|
|
2298
|
-
i++;
|
|
2299
|
-
}
|
|
2300
|
-
return false;
|
|
2301
|
-
}
|
|
2302
|
-
function codePointCount(value) {
|
|
2303
|
-
let counted = 0;
|
|
2304
|
-
for (let i = 0; i < value.length; i++) {
|
|
2305
|
-
const unit = value.charCodeAt(i);
|
|
2306
|
-
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
2307
|
-
const low = value.charCodeAt(i + 1);
|
|
2308
|
-
if (low >= 56320 && low < 57344) i++;
|
|
2309
|
-
}
|
|
2310
|
-
counted++;
|
|
2311
|
-
}
|
|
2312
|
-
return counted;
|
|
2313
|
-
}
|
|
2314
|
-
function adjustUtf16Unit(unit) {
|
|
2315
|
-
if (unit < 55296) return unit;
|
|
2316
|
-
if (unit < 57344) return unit + 8192;
|
|
2317
|
-
return unit - 2048;
|
|
2318
|
-
}
|
|
2319
|
-
function compareCodePoints(a, b) {
|
|
2320
|
-
const shorter = a.length < b.length ? a.length : b.length;
|
|
2321
|
-
let i = 0;
|
|
2322
|
-
while (i < shorter && a.charCodeAt(i) === b.charCodeAt(i)) i++;
|
|
2323
|
-
if (i === shorter) return a.length - b.length;
|
|
2324
|
-
return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
|
|
2325
|
-
}
|
|
2326
|
-
|
|
2327
2636
|
// src/geo/constants.ts
|
|
2328
2637
|
var DEG_TO_RAD = Math.PI / 180;
|
|
2329
2638
|
var MAX_LATITUDE_DEGREES = 90;
|
|
@@ -2845,243 +3154,6 @@ function validateVectorPromotion(config) {
|
|
|
2845
3154
|
}
|
|
2846
3155
|
}
|
|
2847
3156
|
|
|
2848
|
-
// src/core/ordering/fold-table.ts
|
|
2849
|
-
var SINGLE_FOLDS = "1t,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2j,lj;b,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;y,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3d;1,1;2,1;2,1;2,-7g;2,5u;1,1;2,1;2,5q;1,1;2,5p;1,5p;1,1;3,27;1,5m;1,5n;1,1;2,5p;1,5r;2,5v;1,5t;1,1;4,5v;1,5x;2,5y;1,1;2,1;2,1;2,62;1,1;2,62;3,1;2,62;1,1;2,61;1,61;1,1;2,1;2,63;1,1;4,1;8,2;1,1;2,2;1,1;2,2;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,2;1,1;2,1;2,-2p;1,-1k;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;8,8bv;1,1;2,-4j;1,8bs;3,1;2,-5f;1,1x;1,1z;1,1;2,1;2,1;2,1;2,1;6v,38;17,1;2,1;4,1;9,38;7,12;2,11;1,11;1,11;2,1s;2,1r;1,1r;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;n,1;d,8;1,-u;1,-p;4,-f;1,-m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1i;1,-1c;3,-1o;1,-1s;2,1;2,-7;1,1;3,-3m;1,-3m;1,-3m;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1d,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,f;1,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;28a,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;2,5ls;6,5ls;mj,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1oj,-4su;1,-4st;1,-4sk;1,-4si;1,-4si;1,-4sj;1,-4sc;1,-4ro;1,r7n;1,1;7,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;3,-2bk;1,-2bk;1,-2bk;8x,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;7,-1m;5,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;b,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;c,-8;2,-8;2,-8;2,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;21,-8;1,-8;1,-22;1,-22;3,-5j9;a,-2e;1,-2e;1,-2e;1,-2e;d,-8;1,-8;1,-2s;1,-2s;d,-8;1,-8;1,-34;1,-34;1,-7;c,-3k;1,-3k;1,-3i;1,-3i;8b,-5st;4,-6gv;1,-6di;7,s;1a,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;k,1;mr,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1f5,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1d,1;2,-8af;1,-2xy;1,-89z;3,1;2,1;2,1;2,-8bg;1,-8al;1,-8bj;1,-8bi;2,1;3,1;9,-8cf;1,-8cf;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;9,1;2,1;5,1;nym,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;k,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3s,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;b,1;2,1;2,-r9g;1,1;2,1;2,1;2,1;2,1;5,1;2,-wmg;3,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wn8;1,-wnj;1,-wnf;1,-wn5;1,-wn8;2,-wlu;1,-wmi;1,-wlx;1,ps;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1c;1,-wn7;1,-raw;1,1;2,1;2,-wo7;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wu9;p,1;or,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;ggy,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;xy,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;3t,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;4d,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1d7,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;4e,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;27v,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;gw1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1t,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;o5k,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;";
|
|
2850
|
-
var MULTI_FOLDS = "67,37,37;29,2x,lj;p,jg,32;4n,2y,lo;bk,qh,lk,ld;w,qt,lk,ld;d3,12d,136;4y7,2w,mp;1,38,lk;1,3b,lm;1,3d,lm;1,2p,ji;4,37,37;4y,qt,lv;2,qt,lv,lc;2,qt,lv,ld;2,qt,lv,n6;16,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;3,67k,qh;1,q9,qh;1,q4,qh;2,q9,n6;1,q9,n6,qh;5,q9,qh;6,67o,qh;1,qf,qh;1,q6,qh;2,qf,n6;1,qf,n6,qh;5,qf,qh;6,qh,lk,lc;1,qh,lk,ld;3,qh,n6;1,qh,lk,n6;b,qt,lk,lc;1,qt,lk,ld;1,qp,lv;2,qt,n6;1,qt,lk,n6;b,67w,qh;1,qx,qh;1,r2,qh;2,qx,n6;1,qx,n6,qh;5,qx,qh;179g,2u,2u;1,2u,2x;1,2u,30;1,2u,2u,2x;1,2u,2u,30;1,37,38;1,37,38;d,12s,12u;1,12s,12d;1,12s,12j;1,132,12u;1,12s,12l;";
|
|
2851
|
-
var singleTable = null;
|
|
2852
|
-
var multiTable = null;
|
|
2853
|
-
function singleFoldTable() {
|
|
2854
|
-
if (singleTable === null) {
|
|
2855
|
-
singleTable = /* @__PURE__ */ new Map();
|
|
2856
|
-
let source = 0;
|
|
2857
|
-
for (const entry of SINGLE_FOLDS.split(";")) {
|
|
2858
|
-
if (entry.length === 0) continue;
|
|
2859
|
-
const [delta, offset] = entry.split(",");
|
|
2860
|
-
source += Number.parseInt(delta, 36);
|
|
2861
|
-
singleTable.set(source, source + Number.parseInt(offset, 36));
|
|
2862
|
-
}
|
|
2863
|
-
}
|
|
2864
|
-
return singleTable;
|
|
2865
|
-
}
|
|
2866
|
-
function multiFoldTable() {
|
|
2867
|
-
if (multiTable === null) {
|
|
2868
|
-
multiTable = /* @__PURE__ */ new Map();
|
|
2869
|
-
let source = 0;
|
|
2870
|
-
for (const entry of MULTI_FOLDS.split(";")) {
|
|
2871
|
-
if (entry.length === 0) continue;
|
|
2872
|
-
const parts = entry.split(",");
|
|
2873
|
-
source += Number.parseInt(parts[0], 36);
|
|
2874
|
-
multiTable.set(
|
|
2875
|
-
source,
|
|
2876
|
-
parts.slice(1).map((cp) => Number.parseInt(cp, 36))
|
|
2877
|
-
);
|
|
2878
|
-
}
|
|
2879
|
-
}
|
|
2880
|
-
return multiTable;
|
|
2881
|
-
}
|
|
2882
|
-
|
|
2883
|
-
// src/core/ordering/fold-compare.ts
|
|
2884
|
-
var EXHAUSTED = -1;
|
|
2885
|
-
function foldOf(codePoint, single, multi) {
|
|
2886
|
-
if (codePoint < 128) return codePoint >= 65 && codePoint <= 90 ? codePoint + 32 : codePoint;
|
|
2887
|
-
return single.get(codePoint) ?? multi.get(codePoint) ?? codePoint;
|
|
2888
|
-
}
|
|
2889
|
-
function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
2890
|
-
if (cursor.pending !== null) {
|
|
2891
|
-
const cp2 = cursor.pending[cursor.pendingIndex];
|
|
2892
|
-
cursor.pendingIndex++;
|
|
2893
|
-
if (cursor.pendingIndex === cursor.pending.length) {
|
|
2894
|
-
cursor.pending = null;
|
|
2895
|
-
cursor.pendingIndex = 0;
|
|
2896
|
-
}
|
|
2897
|
-
return cp2;
|
|
2898
|
-
}
|
|
2899
|
-
if (cursor.index >= text.length) return EXHAUSTED;
|
|
2900
|
-
let cp = text.charCodeAt(cursor.index);
|
|
2901
|
-
cursor.index++;
|
|
2902
|
-
if (cp >= 55296 && cp < 56320 && cursor.index < text.length) {
|
|
2903
|
-
const low = text.charCodeAt(cursor.index);
|
|
2904
|
-
if (low >= 56320 && low < 57344) {
|
|
2905
|
-
cp = 65536 + (cp - 55296 << 10) + (low - 56320);
|
|
2906
|
-
cursor.index++;
|
|
2907
|
-
}
|
|
2908
|
-
}
|
|
2909
|
-
const folded = foldOf(cp, single, multi);
|
|
2910
|
-
if (typeof folded === "number") return folded;
|
|
2911
|
-
if (folded.length > 1) {
|
|
2912
|
-
cursor.pending = folded;
|
|
2913
|
-
cursor.pendingIndex = 1;
|
|
2914
|
-
}
|
|
2915
|
-
return folded[0];
|
|
2916
|
-
}
|
|
2917
|
-
function compareCaseFolded(a, b) {
|
|
2918
|
-
const shorter = a.length < b.length ? a.length : b.length;
|
|
2919
|
-
let start = 0;
|
|
2920
|
-
while (start < shorter && a.charCodeAt(start) === b.charCodeAt(start)) start++;
|
|
2921
|
-
if (start === a.length && start === b.length) return 0;
|
|
2922
|
-
if (start > 0) {
|
|
2923
|
-
const previous = a.charCodeAt(start - 1);
|
|
2924
|
-
if (previous >= 55296 && previous < 56320) start--;
|
|
2925
|
-
}
|
|
2926
|
-
const single = singleFoldTable();
|
|
2927
|
-
const multi = multiFoldTable();
|
|
2928
|
-
const aCursor = { index: start, pending: null, pendingIndex: 0 };
|
|
2929
|
-
const bCursor = { index: start, pending: null, pendingIndex: 0 };
|
|
2930
|
-
for (; ; ) {
|
|
2931
|
-
const aCp = nextFoldedCodePoint(a, aCursor, single, multi);
|
|
2932
|
-
const bCp = nextFoldedCodePoint(b, bCursor, single, multi);
|
|
2933
|
-
if (aCp !== bCp) return aCp - bCp;
|
|
2934
|
-
if (aCp === EXHAUSTED) return 0;
|
|
2935
|
-
}
|
|
2936
|
-
}
|
|
2937
|
-
function compareSortStrings(a, b) {
|
|
2938
|
-
const folded = compareCaseFolded(a, b);
|
|
2939
|
-
if (folded !== 0) return folded;
|
|
2940
|
-
return compareCodePoints(a, b);
|
|
2941
|
-
}
|
|
2942
|
-
function appendFoldedCodePoint(codePoint, out) {
|
|
2943
|
-
const folded = foldOf(codePoint, singleFoldTable(), multiFoldTable());
|
|
2944
|
-
if (typeof folded === "number") {
|
|
2945
|
-
out.push(folded);
|
|
2946
|
-
return;
|
|
2947
|
-
}
|
|
2948
|
-
for (const foldedCodePoint of folded) out.push(foldedCodePoint);
|
|
2949
|
-
}
|
|
2950
|
-
function appendFoldedCodePoints(text, out) {
|
|
2951
|
-
const single = singleFoldTable();
|
|
2952
|
-
const multi = multiFoldTable();
|
|
2953
|
-
const cursor = { index: 0, pending: null, pendingIndex: 0 };
|
|
2954
|
-
for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
|
|
2955
|
-
out.push(codePoint);
|
|
2956
|
-
codePoint = nextFoldedCodePoint(text, cursor, single, multi);
|
|
2957
|
-
}
|
|
2958
|
-
}
|
|
2959
|
-
function foldedCodePointCount(text) {
|
|
2960
|
-
const folded = [];
|
|
2961
|
-
appendFoldedCodePoints(text, folded);
|
|
2962
|
-
return folded.length;
|
|
2963
|
-
}
|
|
2964
|
-
|
|
2965
|
-
// src/core/ordering/constants.ts
|
|
2966
|
-
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
2967
|
-
|
|
2968
|
-
// src/core/ordering/sort-values.ts
|
|
2969
|
-
var SORT_MODES = ["min", "max", "avg", "median"];
|
|
2970
|
-
function isSortMode(value) {
|
|
2971
|
-
return value === "min" || value === "max" || value === "avg" || value === "median";
|
|
2972
|
-
}
|
|
2973
|
-
function defaultSortMode(direction) {
|
|
2974
|
-
return direction === "desc" ? "max" : "min";
|
|
2975
|
-
}
|
|
2976
|
-
function sortModeOf(entry) {
|
|
2977
|
-
return entry.mode ?? defaultSortMode(entry.direction);
|
|
2978
|
-
}
|
|
2979
|
-
function truncateSortString(value) {
|
|
2980
|
-
if (value.length <= SORT_VALUE_MAX_CODE_POINTS) return value;
|
|
2981
|
-
let index = 0;
|
|
2982
|
-
let counted = 0;
|
|
2983
|
-
while (index < value.length && counted < SORT_VALUE_MAX_CODE_POINTS) {
|
|
2984
|
-
const unit = value.charCodeAt(index);
|
|
2985
|
-
if (unit >= 55296 && unit < 56320 && index + 1 < value.length) {
|
|
2986
|
-
const low = value.charCodeAt(index + 1);
|
|
2987
|
-
if (low >= 56320 && low < 57344) index++;
|
|
2988
|
-
}
|
|
2989
|
-
index++;
|
|
2990
|
-
counted++;
|
|
2991
|
-
}
|
|
2992
|
-
return value.slice(0, index);
|
|
2993
|
-
}
|
|
2994
|
-
function toComparableSortValue(value) {
|
|
2995
|
-
if (typeof value === "string") return truncateSortString(value);
|
|
2996
|
-
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
2997
|
-
if (typeof value === "boolean") return value;
|
|
2998
|
-
return null;
|
|
2999
|
-
}
|
|
3000
|
-
function numericReduction(values, mode) {
|
|
3001
|
-
const numbers = [];
|
|
3002
|
-
for (const element of values) {
|
|
3003
|
-
if (typeof element === "number" && Number.isFinite(element)) numbers.push(element);
|
|
3004
|
-
}
|
|
3005
|
-
if (numbers.length === 0) return null;
|
|
3006
|
-
if (mode === "avg") {
|
|
3007
|
-
let mean = 0;
|
|
3008
|
-
for (let index = 0; index < numbers.length; index++) {
|
|
3009
|
-
mean += (numbers[index] - mean) / (index + 1);
|
|
3010
|
-
}
|
|
3011
|
-
return mean;
|
|
3012
|
-
}
|
|
3013
|
-
numbers.sort((a, b) => a - b);
|
|
3014
|
-
const middle = numbers.length >> 1;
|
|
3015
|
-
if (numbers.length % 2 === 1) return numbers[middle];
|
|
3016
|
-
return numbers[middle - 1] / 2 + numbers[middle] / 2;
|
|
3017
|
-
}
|
|
3018
|
-
function toReducedSortValue(value, mode) {
|
|
3019
|
-
if (!Array.isArray(value)) return toComparableSortValue(value);
|
|
3020
|
-
if (mode === "avg" || mode === "median") return numericReduction(value, mode);
|
|
3021
|
-
let chosen = null;
|
|
3022
|
-
for (const element of value) {
|
|
3023
|
-
const comparable = toComparableSortValue(element);
|
|
3024
|
-
if (comparable === null) continue;
|
|
3025
|
-
if (chosen === null) {
|
|
3026
|
-
chosen = comparable;
|
|
3027
|
-
continue;
|
|
3028
|
-
}
|
|
3029
|
-
const comparison = comparePresentValues(comparable, chosen);
|
|
3030
|
-
if (mode === "min" ? comparison < 0 : comparison > 0) chosen = comparable;
|
|
3031
|
-
}
|
|
3032
|
-
return chosen;
|
|
3033
|
-
}
|
|
3034
|
-
function readSortField(document2, path) {
|
|
3035
|
-
const segments = path.split(".");
|
|
3036
|
-
let current = document2;
|
|
3037
|
-
for (const segment of segments) {
|
|
3038
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
3039
|
-
if (!Object.hasOwn(current, segment)) return void 0;
|
|
3040
|
-
current = current[segment];
|
|
3041
|
-
}
|
|
3042
|
-
return current;
|
|
3043
|
-
}
|
|
3044
|
-
function typeRank(value) {
|
|
3045
|
-
if (typeof value === "number") return 0;
|
|
3046
|
-
if (typeof value === "string") return 1;
|
|
3047
|
-
return 2;
|
|
3048
|
-
}
|
|
3049
|
-
function comparePresentValues(a, b) {
|
|
3050
|
-
const rankDifference = typeRank(a) - typeRank(b);
|
|
3051
|
-
if (rankDifference !== 0) return rankDifference;
|
|
3052
|
-
if (typeof a === "number" && typeof b === "number") return a - b;
|
|
3053
|
-
if (typeof a === "string" && typeof b === "string") return compareSortStrings(a, b);
|
|
3054
|
-
if (a === b) return 0;
|
|
3055
|
-
return a ? 1 : -1;
|
|
3056
|
-
}
|
|
3057
|
-
function compareComparableValues(a, b, direction) {
|
|
3058
|
-
if (a === null && b === null) return 0;
|
|
3059
|
-
if (a === null) return 1;
|
|
3060
|
-
if (b === null) return -1;
|
|
3061
|
-
const comparison = comparePresentValues(a, b);
|
|
3062
|
-
if (comparison === 0) return 0;
|
|
3063
|
-
return direction === "desc" ? -comparison : comparison;
|
|
3064
|
-
}
|
|
3065
|
-
function compareComparableKeys(aKey, bKey, directions) {
|
|
3066
|
-
for (let i = 0; i < directions.length; i++) {
|
|
3067
|
-
const comparison = compareComparableValues(aKey[i] ?? null, bKey[i] ?? null, directions[i]);
|
|
3068
|
-
if (comparison !== 0) return comparison;
|
|
3069
|
-
}
|
|
3070
|
-
return 0;
|
|
3071
|
-
}
|
|
3072
|
-
function compareSortValues(aValues, bValues, directions) {
|
|
3073
|
-
for (let i = 0; i < directions.length; i++) {
|
|
3074
|
-
const a = toComparableSortValue(aValues[i]);
|
|
3075
|
-
const b = toComparableSortValue(bValues[i]);
|
|
3076
|
-
if (a === null && b === null) continue;
|
|
3077
|
-
if (a === null) return 1;
|
|
3078
|
-
if (b === null) return -1;
|
|
3079
|
-
const comparison = comparePresentValues(a, b);
|
|
3080
|
-
if (comparison !== 0) return directions[i] === "desc" ? -comparison : comparison;
|
|
3081
|
-
}
|
|
3082
|
-
return 0;
|
|
3083
|
-
}
|
|
3084
|
-
|
|
3085
3157
|
// src/core/document-store.ts
|
|
3086
3158
|
function detachedView(view) {
|
|
3087
3159
|
if (view.byteOffset === 0 && view.byteLength === view.buffer.byteLength) return view;
|
|
@@ -3436,8 +3508,6 @@ function compactList(list) {
|
|
|
3436
3508
|
}
|
|
3437
3509
|
}
|
|
3438
3510
|
writeIdx++;
|
|
3439
|
-
} else {
|
|
3440
|
-
list.totalTermFrequency -= list.termFrequencies[i];
|
|
3441
3511
|
}
|
|
3442
3512
|
}
|
|
3443
3513
|
list.docIds.length = writeIdx;
|
|
@@ -3462,8 +3532,6 @@ function compactDocEntries(list, internalId) {
|
|
|
3462
3532
|
}
|
|
3463
3533
|
}
|
|
3464
3534
|
writeIdx++;
|
|
3465
|
-
} else {
|
|
3466
|
-
list.totalTermFrequency -= list.termFrequencies[i];
|
|
3467
3535
|
}
|
|
3468
3536
|
}
|
|
3469
3537
|
list.docIds.length = writeIdx;
|
|
@@ -3549,12 +3617,13 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
3549
3617
|
}
|
|
3550
3618
|
list.length++;
|
|
3551
3619
|
},
|
|
3552
|
-
remove(token, internalId) {
|
|
3620
|
+
remove(token, internalId, occurrences) {
|
|
3553
3621
|
const list = index.get(token);
|
|
3554
3622
|
if (!list || !list.docIdSet.has(internalId)) return;
|
|
3555
3623
|
list.structureRevision++;
|
|
3556
3624
|
list.docIdSet.delete(internalId);
|
|
3557
3625
|
list.deletedDocs.add(internalId);
|
|
3626
|
+
list.totalTermFrequency = Math.max(0, list.totalTermFrequency - occurrences);
|
|
3558
3627
|
if (list.docIdSet.size === 0) {
|
|
3559
3628
|
index.delete(token);
|
|
3560
3629
|
untrackToken(token);
|
|
@@ -3587,6 +3656,9 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
3587
3656
|
lookup(token) {
|
|
3588
3657
|
return index.get(token);
|
|
3589
3658
|
},
|
|
3659
|
+
documentFrequency(token) {
|
|
3660
|
+
return index.get(token)?.docIdSet.size ?? 0;
|
|
3661
|
+
},
|
|
3590
3662
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
3591
3663
|
if (tolerance === 0) {
|
|
3592
3664
|
const exact = index.get(token);
|
|
@@ -5177,6 +5249,53 @@ function createNumericIndex() {
|
|
|
5177
5249
|
};
|
|
5178
5250
|
}
|
|
5179
5251
|
|
|
5252
|
+
// src/core/partition/partial-words/spellings.ts
|
|
5253
|
+
function countSpelling(counts, token, spelling, change) {
|
|
5254
|
+
let byToken = counts.get(token);
|
|
5255
|
+
if (byToken === void 0) {
|
|
5256
|
+
byToken = /* @__PURE__ */ new Map();
|
|
5257
|
+
counts.set(token, byToken);
|
|
5258
|
+
}
|
|
5259
|
+
const next = (byToken.get(spelling) ?? 0) + change;
|
|
5260
|
+
if (next > 0) {
|
|
5261
|
+
byToken.set(spelling, next);
|
|
5262
|
+
return;
|
|
5263
|
+
}
|
|
5264
|
+
byToken.delete(spelling);
|
|
5265
|
+
if (byToken.size === 0) counts.delete(token);
|
|
5266
|
+
}
|
|
5267
|
+
function textsOf(value) {
|
|
5268
|
+
if (typeof value === "string") return [value];
|
|
5269
|
+
if (!Array.isArray(value)) return [];
|
|
5270
|
+
return value.filter((element) => typeof element === "string");
|
|
5271
|
+
}
|
|
5272
|
+
function countDocumentSpellings(counts, document2, analysis, direction) {
|
|
5273
|
+
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
5274
|
+
for (const field of analysis.fields) {
|
|
5275
|
+
for (const text of textsOf(getNestedValue(document2, field))) {
|
|
5276
|
+
for (const occurrence of tokenizeIterator(text, analysis.language, tokenizing)) {
|
|
5277
|
+
countSpelling(counts, occurrence.token, occurrence.surface ?? occurrence.token, direction);
|
|
5278
|
+
}
|
|
5279
|
+
}
|
|
5280
|
+
}
|
|
5281
|
+
}
|
|
5282
|
+
function spellingCountsOf(documents, analysis) {
|
|
5283
|
+
const counts = /* @__PURE__ */ new Map();
|
|
5284
|
+
for (const document2 of documents) countDocumentSpellings(counts, document2, analysis, 1);
|
|
5285
|
+
return counts;
|
|
5286
|
+
}
|
|
5287
|
+
function encodePartialSpellings(fields, counts) {
|
|
5288
|
+
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
5289
|
+
for (const [token, byToken] of counts) {
|
|
5290
|
+
for (const [spelling, count] of byToken) {
|
|
5291
|
+
encoded.tokens.push(token);
|
|
5292
|
+
encoded.spellings.push(spelling);
|
|
5293
|
+
encoded.counts.push(count);
|
|
5294
|
+
}
|
|
5295
|
+
}
|
|
5296
|
+
return encoded;
|
|
5297
|
+
}
|
|
5298
|
+
|
|
5180
5299
|
// src/core/partition/value-sizes.ts
|
|
5181
5300
|
function keepsValueSize(base) {
|
|
5182
5301
|
return base === "string" || base === "verbatim" || base.endsWith("[]");
|
|
@@ -5250,6 +5369,13 @@ function resolveInternalId(state, docId) {
|
|
|
5250
5369
|
}
|
|
5251
5370
|
return internalId;
|
|
5252
5371
|
}
|
|
5372
|
+
function emptyIndexedText(fields) {
|
|
5373
|
+
const holdsPartialField = fields.some((field) => field.partial);
|
|
5374
|
+
return { fieldLengths: {}, tokensByField: {}, partialSpellings: holdsPartialField ? /* @__PURE__ */ new Map() : null };
|
|
5375
|
+
}
|
|
5376
|
+
function partialSpellingsFor(field, indexed) {
|
|
5377
|
+
return field.partial ? indexed.partialSpellings : null;
|
|
5378
|
+
}
|
|
5253
5379
|
function countSurface(counts, surface, token) {
|
|
5254
5380
|
if (surface === void 0 || surface.length === 0) return;
|
|
5255
5381
|
const existing = counts.get(surface);
|
|
@@ -5259,6 +5385,18 @@ function countSurface(counts, surface, token) {
|
|
|
5259
5385
|
counts.set(surface, { token, count: 1 });
|
|
5260
5386
|
}
|
|
5261
5387
|
}
|
|
5388
|
+
function countOccurrence(counts, spellings, surface, token) {
|
|
5389
|
+
countSurface(counts, surface, token);
|
|
5390
|
+
if (spellings !== null) countSpelling(spellings, token, surface ?? token, 1);
|
|
5391
|
+
}
|
|
5392
|
+
function addFieldOccurrences(occurrencesByToken, fieldTokens) {
|
|
5393
|
+
const fieldCounts = /* @__PURE__ */ new Map();
|
|
5394
|
+
for (const token of fieldTokens) fieldCounts.set(token, (fieldCounts.get(token) ?? 0) + 1);
|
|
5395
|
+
for (const [token, count] of fieldCounts) {
|
|
5396
|
+
const stored = count > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : count;
|
|
5397
|
+
occurrencesByToken.set(token, (occurrencesByToken.get(token) ?? 0) + stored);
|
|
5398
|
+
}
|
|
5399
|
+
}
|
|
5262
5400
|
function applySurfaceCounts(state, counts, direction) {
|
|
5263
5401
|
for (const [surface, entry] of counts) {
|
|
5264
5402
|
if (direction === 1) {
|
|
@@ -5268,7 +5406,7 @@ function applySurfaceCounts(state, counts, direction) {
|
|
|
5268
5406
|
}
|
|
5269
5407
|
}
|
|
5270
5408
|
}
|
|
5271
|
-
function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField) {
|
|
5409
|
+
function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField, spellings = null) {
|
|
5272
5410
|
const fieldTokenList = [];
|
|
5273
5411
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
|
|
5274
5412
|
const opts = tokenizeOptions(options);
|
|
@@ -5284,7 +5422,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
5284
5422
|
tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
|
|
5285
5423
|
}
|
|
5286
5424
|
fieldTokenList.push(t.token);
|
|
5287
|
-
|
|
5425
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
5288
5426
|
}
|
|
5289
5427
|
for (const [token, freq] of tokenFreqs) {
|
|
5290
5428
|
state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex2, freq.positions);
|
|
@@ -5294,7 +5432,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
5294
5432
|
for (const t of tokenizeIterator(text, language, opts)) {
|
|
5295
5433
|
tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
|
|
5296
5434
|
fieldTokenList.push(t.token);
|
|
5297
|
-
|
|
5435
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
5298
5436
|
}
|
|
5299
5437
|
for (const [token, count] of tokenCounts) {
|
|
5300
5438
|
state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
|
|
@@ -5304,7 +5442,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
5304
5442
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
5305
5443
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5306
5444
|
}
|
|
5307
|
-
function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
|
|
5445
|
+
function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField, spellings = null) {
|
|
5308
5446
|
const fieldTokenList = [];
|
|
5309
5447
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
|
|
5310
5448
|
const opts = tokenizeOptions(options);
|
|
@@ -5323,7 +5461,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5323
5461
|
tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
|
|
5324
5462
|
}
|
|
5325
5463
|
fieldTokenList.push(t.token);
|
|
5326
|
-
|
|
5464
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
5327
5465
|
itemTokenCount++;
|
|
5328
5466
|
}
|
|
5329
5467
|
positionOffset += itemTokenCount;
|
|
@@ -5338,7 +5476,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5338
5476
|
for (const t of tokenizeIterator(item, language, opts)) {
|
|
5339
5477
|
tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
|
|
5340
5478
|
fieldTokenList.push(t.token);
|
|
5341
|
-
|
|
5479
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
5342
5480
|
}
|
|
5343
5481
|
}
|
|
5344
5482
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
@@ -5350,17 +5488,28 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5350
5488
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5351
5489
|
}
|
|
5352
5490
|
function indexDocument(state, docId, document2, fields, language, options) {
|
|
5353
|
-
const
|
|
5354
|
-
const tokensByField =
|
|
5491
|
+
const indexed = emptyIndexedText(fields);
|
|
5492
|
+
const { fieldLengths, tokensByField } = indexed;
|
|
5355
5493
|
const internalId = resolveInternalId(state, docId);
|
|
5356
|
-
for (const
|
|
5494
|
+
for (const field of fields) {
|
|
5495
|
+
const { path: fieldPath, base, pattern } = field;
|
|
5357
5496
|
const value = getNestedValue(document2, fieldPath);
|
|
5358
5497
|
if (value === void 0 || value === null) continue;
|
|
5359
5498
|
recordValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
5360
5499
|
ensureFieldIndex(state, fieldPath, base);
|
|
5361
5500
|
if (pattern) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
|
|
5362
5501
|
if (base === "string") {
|
|
5363
|
-
indexStringField(
|
|
5502
|
+
indexStringField(
|
|
5503
|
+
state,
|
|
5504
|
+
internalId,
|
|
5505
|
+
fieldPath,
|
|
5506
|
+
value,
|
|
5507
|
+
language,
|
|
5508
|
+
options,
|
|
5509
|
+
fieldLengths,
|
|
5510
|
+
tokensByField,
|
|
5511
|
+
partialSpellingsFor(field, indexed)
|
|
5512
|
+
);
|
|
5364
5513
|
} else if (base === "number") {
|
|
5365
5514
|
getOrCreateNumericIndex(state, fieldPath).insert(internalId, value);
|
|
5366
5515
|
} else if (base === "boolean") {
|
|
@@ -5379,7 +5528,8 @@ function indexDocument(state, docId, document2, fields, language, options) {
|
|
|
5379
5528
|
language,
|
|
5380
5529
|
options,
|
|
5381
5530
|
fieldLengths,
|
|
5382
|
-
tokensByField
|
|
5531
|
+
tokensByField,
|
|
5532
|
+
partialSpellingsFor(field, indexed)
|
|
5383
5533
|
);
|
|
5384
5534
|
} else if (base === "number[]") {
|
|
5385
5535
|
const numIdx = getOrCreateNumericIndex(state, fieldPath);
|
|
@@ -5394,15 +5544,17 @@ function indexDocument(state, docId, document2, fields, language, options) {
|
|
|
5394
5544
|
}
|
|
5395
5545
|
}
|
|
5396
5546
|
}
|
|
5397
|
-
return
|
|
5547
|
+
return indexed;
|
|
5398
5548
|
}
|
|
5399
5549
|
function removeFromIndexes(state, docId, storedDoc, schemaFields, language, options) {
|
|
5400
|
-
const
|
|
5401
|
-
const tokensByField =
|
|
5550
|
+
const removed = emptyIndexedText(schemaFields);
|
|
5551
|
+
const { fieldLengths, tokensByField } = removed;
|
|
5402
5552
|
const fields = storedDoc.fields;
|
|
5403
5553
|
const internalId = resolveInternalId(state, docId);
|
|
5404
5554
|
const opts = tokenizeOptions(options);
|
|
5405
|
-
|
|
5555
|
+
const occurrencesByToken = /* @__PURE__ */ new Map();
|
|
5556
|
+
for (const field of schemaFields) {
|
|
5557
|
+
const { path: fieldPath, base, pattern } = field;
|
|
5406
5558
|
const value = getNestedValue(fields, fieldPath);
|
|
5407
5559
|
if (value === void 0 || value === null) continue;
|
|
5408
5560
|
forgetValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
@@ -5410,16 +5562,15 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
5410
5562
|
if (base === "string") {
|
|
5411
5563
|
const result = tokenize(value, language, opts);
|
|
5412
5564
|
fieldLengths[fieldPath] = result.tokens.length;
|
|
5413
|
-
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
5414
5565
|
const fieldTokenList = [];
|
|
5415
5566
|
const surfaceCounts = /* @__PURE__ */ new Map();
|
|
5567
|
+
const spellings = partialSpellingsFor(field, removed);
|
|
5416
5568
|
for (let i = 0; i < result.tokens.length; i++) {
|
|
5417
5569
|
const t = result.tokens[i];
|
|
5418
|
-
uniqueTokens.add(t.token);
|
|
5419
5570
|
fieldTokenList.push(t.token);
|
|
5420
|
-
|
|
5571
|
+
countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
|
|
5421
5572
|
}
|
|
5422
|
-
|
|
5573
|
+
addFieldOccurrences(occurrencesByToken, fieldTokenList);
|
|
5423
5574
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
5424
5575
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5425
5576
|
} else if (base === "number") {
|
|
@@ -5432,20 +5583,19 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
5432
5583
|
state.geoIndexes.get(fieldPath)?.remove(internalId);
|
|
5433
5584
|
} else if (base === "string[]") {
|
|
5434
5585
|
const arr = value;
|
|
5435
|
-
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
5436
5586
|
const fieldTokenList = [];
|
|
5437
5587
|
const surfaceCounts = /* @__PURE__ */ new Map();
|
|
5588
|
+
const spellings = partialSpellingsFor(field, removed);
|
|
5438
5589
|
for (const item of arr) {
|
|
5439
5590
|
const result = tokenize(item, language, opts);
|
|
5440
5591
|
for (let i = 0; i < result.tokens.length; i++) {
|
|
5441
5592
|
const t = result.tokens[i];
|
|
5442
|
-
uniqueTokens.add(t.token);
|
|
5443
5593
|
fieldTokenList.push(t.token);
|
|
5444
|
-
|
|
5594
|
+
countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
|
|
5445
5595
|
}
|
|
5446
5596
|
}
|
|
5447
5597
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
5448
|
-
|
|
5598
|
+
addFieldOccurrences(occurrencesByToken, fieldTokenList);
|
|
5449
5599
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
5450
5600
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5451
5601
|
} else if (base === "number[]") {
|
|
@@ -5465,7 +5615,8 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
5465
5615
|
}
|
|
5466
5616
|
}
|
|
5467
5617
|
}
|
|
5468
|
-
|
|
5618
|
+
for (const [token, occurrences] of occurrencesByToken) state.invertedIdx.remove(token, internalId, occurrences);
|
|
5619
|
+
return removed;
|
|
5469
5620
|
}
|
|
5470
5621
|
|
|
5471
5622
|
// src/core/partition/field-updates.ts
|
|
@@ -5568,7 +5719,7 @@ function excluding(context, fieldPath, excluded) {
|
|
|
5568
5719
|
const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
|
|
5569
5720
|
return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
|
|
5570
5721
|
}
|
|
5571
|
-
function
|
|
5722
|
+
function textsOf2(operand) {
|
|
5572
5723
|
const texts = /* @__PURE__ */ new Set();
|
|
5573
5724
|
if (!Array.isArray(operand)) return texts;
|
|
5574
5725
|
for (const text of operand) if (typeof text === "string") texts.add(text);
|
|
@@ -5583,8 +5734,8 @@ function evaluatePatternTests(fieldPath, filter, context, bitsets) {
|
|
|
5583
5734
|
};
|
|
5584
5735
|
if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq]));
|
|
5585
5736
|
if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
|
|
5586
|
-
if (filter.in !== void 0) bitsets.push(matching("eq",
|
|
5587
|
-
if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq",
|
|
5737
|
+
if (filter.in !== void 0) bitsets.push(matching("eq", textsOf2(filter.in)));
|
|
5738
|
+
if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf2(filter.nin))));
|
|
5588
5739
|
if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
|
|
5589
5740
|
if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
|
|
5590
5741
|
if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
|
|
@@ -7016,6 +7167,67 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
7016
7167
|
};
|
|
7017
7168
|
}
|
|
7018
7169
|
|
|
7170
|
+
// src/core/scorer.ts
|
|
7171
|
+
var DEFAULT_K1 = 1.2;
|
|
7172
|
+
var DEFAULT_B = 0.75;
|
|
7173
|
+
function computeIDF(docFrequency, totalDocs) {
|
|
7174
|
+
if (totalDocs === 0) return 0;
|
|
7175
|
+
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
7176
|
+
}
|
|
7177
|
+
function bm25PruningSound(params) {
|
|
7178
|
+
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7179
|
+
const b = params?.b ?? DEFAULT_B;
|
|
7180
|
+
return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
|
|
7181
|
+
}
|
|
7182
|
+
function resolveBM25Params(params) {
|
|
7183
|
+
return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
|
|
7184
|
+
}
|
|
7185
|
+
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
7186
|
+
if (avgFieldLength === 0) return 0;
|
|
7187
|
+
const numerator = termFrequency * (k1 + 1);
|
|
7188
|
+
const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
|
|
7189
|
+
if (denominator === 0) return 0;
|
|
7190
|
+
return idf * (numerator / denominator);
|
|
7191
|
+
}
|
|
7192
|
+
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
7193
|
+
if (totalDocs === 0) return 0;
|
|
7194
|
+
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7195
|
+
const b = params?.b ?? DEFAULT_B;
|
|
7196
|
+
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
7197
|
+
}
|
|
7198
|
+
|
|
7199
|
+
// src/core/partition/field-scoring.ts
|
|
7200
|
+
function loadFieldScoring(state, fields, boost, averageFieldLengths) {
|
|
7201
|
+
const fieldNames = state.fieldNameTable.names;
|
|
7202
|
+
const scoring = {
|
|
7203
|
+
searchable: new Uint8Array(fieldNames.length),
|
|
7204
|
+
boosts: new Float64Array(fieldNames.length),
|
|
7205
|
+
averageLengths: new Float64Array(fieldNames.length),
|
|
7206
|
+
lengthColumns: new Array(fieldNames.length).fill(null)
|
|
7207
|
+
};
|
|
7208
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
7209
|
+
const fieldName = fieldNames[fieldIndex];
|
|
7210
|
+
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
7211
|
+
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
7212
|
+
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
7213
|
+
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
7214
|
+
}
|
|
7215
|
+
return scoring;
|
|
7216
|
+
}
|
|
7217
|
+
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
7218
|
+
const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
|
|
7219
|
+
if (column === null || internalId >= column.length) return averageLength;
|
|
7220
|
+
const stored = column[internalId];
|
|
7221
|
+
return stored > 0 ? stored : averageLength;
|
|
7222
|
+
}
|
|
7223
|
+
function bestSearchable(searchable, values) {
|
|
7224
|
+
let best = 0;
|
|
7225
|
+
for (let index = 0; index < searchable.length; index++) {
|
|
7226
|
+
if (searchable[index] === 1 && values[index] > best) best = values[index];
|
|
7227
|
+
}
|
|
7228
|
+
return best;
|
|
7229
|
+
}
|
|
7230
|
+
|
|
7019
7231
|
// src/core/partition/posting-columns.ts
|
|
7020
7232
|
function postingColumns(list) {
|
|
7021
7233
|
const deletedDocs = list.deletedDocs;
|
|
@@ -7029,39 +7241,174 @@ function postingColumns(list) {
|
|
|
7029
7241
|
};
|
|
7030
7242
|
}
|
|
7031
7243
|
|
|
7244
|
+
// src/core/partition/scoring.ts
|
|
7245
|
+
var EMPTY_COMPONENTS = Object.freeze({});
|
|
7246
|
+
function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
|
|
7247
|
+
const existing = components.get(internalId);
|
|
7248
|
+
if (existing) {
|
|
7249
|
+
existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
|
|
7250
|
+
existing.fieldLengths[fieldName] = fieldLength;
|
|
7251
|
+
existing.idf[token] = idf;
|
|
7252
|
+
return;
|
|
7253
|
+
}
|
|
7254
|
+
components.set(internalId, {
|
|
7255
|
+
termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
|
|
7256
|
+
fieldLengths: { [fieldName]: fieldLength },
|
|
7257
|
+
idf: { [token]: idf }
|
|
7258
|
+
});
|
|
7259
|
+
}
|
|
7260
|
+
function mergeGroupComponents(components, internalId, contribution) {
|
|
7261
|
+
const existing = components.get(internalId);
|
|
7262
|
+
if (existing) {
|
|
7263
|
+
Object.assign(existing.termFrequencies, contribution.termFrequencies);
|
|
7264
|
+
Object.assign(existing.fieldLengths, contribution.fieldLengths);
|
|
7265
|
+
existing.idf[contribution.token] = contribution.idf;
|
|
7266
|
+
return;
|
|
7267
|
+
}
|
|
7268
|
+
components.set(internalId, {
|
|
7269
|
+
termFrequencies: contribution.termFrequencies,
|
|
7270
|
+
fieldLengths: contribution.fieldLengths,
|
|
7271
|
+
idf: { [contribution.token]: contribution.idf }
|
|
7272
|
+
});
|
|
7273
|
+
}
|
|
7274
|
+
|
|
7275
|
+
// src/core/partition/group-scoring.ts
|
|
7276
|
+
function partialFieldMarks(fieldNames, partialFields) {
|
|
7277
|
+
const marks = new Uint8Array(fieldNames.length);
|
|
7278
|
+
if (partialFields === void 0 || partialFields.length === 0) return marks;
|
|
7279
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
7280
|
+
if (partialFields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
|
|
7281
|
+
}
|
|
7282
|
+
return marks;
|
|
7283
|
+
}
|
|
7284
|
+
function termGroupOf(groups, token) {
|
|
7285
|
+
if (groups === void 0) return void 0;
|
|
7286
|
+
for (const group of groups) {
|
|
7287
|
+
if (group.token === token) return group;
|
|
7288
|
+
}
|
|
7289
|
+
return void 0;
|
|
7290
|
+
}
|
|
7291
|
+
function typoMatchesFor(index, token, group, exact, tolerance, prefixLength) {
|
|
7292
|
+
if (group !== void 0) {
|
|
7293
|
+
if (!group.typoMatches || exact || tolerance === 0) return [];
|
|
7294
|
+
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token);
|
|
7295
|
+
}
|
|
7296
|
+
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
7297
|
+
const postingList = index.lookup(token);
|
|
7298
|
+
return postingList ? [{ token, postingList }] : [];
|
|
7299
|
+
}
|
|
7300
|
+
function resolveGroupMatches(context, group) {
|
|
7301
|
+
let groupDocumentFrequency = 0;
|
|
7302
|
+
const found = [];
|
|
7303
|
+
for (const member of group.members) {
|
|
7304
|
+
const documents = context.globalFrequency(member.token) ?? context.index.documentFrequency(member.token);
|
|
7305
|
+
if (documents > groupDocumentFrequency) groupDocumentFrequency = documents;
|
|
7306
|
+
const postingList = context.index.lookup(member.token);
|
|
7307
|
+
if (postingList === void 0) continue;
|
|
7308
|
+
found.push({ token: member.token, factor: member.factor, partialFactor: member.partialFactor, postingList });
|
|
7309
|
+
}
|
|
7310
|
+
if (found.length === 0) return [];
|
|
7311
|
+
const idf = computeIDF(groupDocumentFrequency, context.totalDocs);
|
|
7312
|
+
return found.map((entry) => ({ ...entry, idf }));
|
|
7313
|
+
}
|
|
7314
|
+
function groupPostingCount(matches) {
|
|
7315
|
+
let total = 0;
|
|
7316
|
+
for (const match of matches) total += match.postingList.length;
|
|
7317
|
+
return total;
|
|
7318
|
+
}
|
|
7319
|
+
function computeGroupContributions(context, matches, collect) {
|
|
7320
|
+
const { fields, fieldNames, filterBitset, partialMarks, k1, b } = context;
|
|
7321
|
+
const scoresAreZero = context.totalDocs === 0;
|
|
7322
|
+
const best = /* @__PURE__ */ new Map();
|
|
7323
|
+
for (const match of matches) {
|
|
7324
|
+
const perTerm = /* @__PURE__ */ new Map();
|
|
7325
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
7326
|
+
match.postingList
|
|
7327
|
+
);
|
|
7328
|
+
for (let pi = 0; pi < count; pi++) {
|
|
7329
|
+
const internalId = docIds[pi];
|
|
7330
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7331
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
7332
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
7333
|
+
if (fields.searchable[fieldIndex] === 0) continue;
|
|
7334
|
+
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
7335
|
+
if (factor === 0) continue;
|
|
7336
|
+
const fieldName = fieldNames[fieldIndex];
|
|
7337
|
+
const termFrequency = termFrequencies[pi];
|
|
7338
|
+
const fieldBoost = fields.boosts[fieldIndex];
|
|
7339
|
+
const avgLen = fields.averageLengths[fieldIndex];
|
|
7340
|
+
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
7341
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * factor;
|
|
7342
|
+
const existing = perTerm.get(internalId);
|
|
7343
|
+
if (existing) {
|
|
7344
|
+
existing.score += termScore;
|
|
7345
|
+
if (collect) {
|
|
7346
|
+
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
7347
|
+
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
7348
|
+
}
|
|
7349
|
+
} else if (collect) {
|
|
7350
|
+
perTerm.set(internalId, {
|
|
7351
|
+
score: termScore,
|
|
7352
|
+
token: match.token,
|
|
7353
|
+
idf: match.idf,
|
|
7354
|
+
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
7355
|
+
fieldLengths: { [fieldName]: actualFieldLength }
|
|
7356
|
+
});
|
|
7357
|
+
} else {
|
|
7358
|
+
perTerm.set(internalId, {
|
|
7359
|
+
score: termScore,
|
|
7360
|
+
token: match.token,
|
|
7361
|
+
idf: match.idf,
|
|
7362
|
+
termFrequencies: EMPTY_COMPONENTS,
|
|
7363
|
+
fieldLengths: EMPTY_COMPONENTS
|
|
7364
|
+
});
|
|
7365
|
+
}
|
|
7366
|
+
}
|
|
7367
|
+
for (const [internalId, contribution] of perTerm) {
|
|
7368
|
+
const current = best.get(internalId);
|
|
7369
|
+
if (!current || contribution.score > current.score) {
|
|
7370
|
+
best.set(internalId, contribution);
|
|
7371
|
+
}
|
|
7372
|
+
}
|
|
7373
|
+
}
|
|
7374
|
+
return best;
|
|
7375
|
+
}
|
|
7376
|
+
|
|
7032
7377
|
// src/core/partition/matches.ts
|
|
7033
|
-
function markPostingList(matched, list,
|
|
7378
|
+
function markPostingList(matched, list, markedFields, filterBitset) {
|
|
7034
7379
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(list);
|
|
7035
7380
|
for (let pi = 0; pi < count; pi++) {
|
|
7036
7381
|
const internalId = docIds[pi];
|
|
7037
7382
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7038
7383
|
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
7039
|
-
if (
|
|
7384
|
+
if (markedFields[fieldNameIndices[pi]] === 0) continue;
|
|
7040
7385
|
bitsetSet(matched, internalId);
|
|
7041
7386
|
}
|
|
7042
7387
|
}
|
|
7388
|
+
function searchedFieldMarks(fieldNames, fields) {
|
|
7389
|
+
const marks = new Uint8Array(fieldNames.length);
|
|
7390
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
7391
|
+
if (fields === void 0 || fields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
|
|
7392
|
+
}
|
|
7393
|
+
return marks;
|
|
7394
|
+
}
|
|
7043
7395
|
function searchFulltextMatches(state, params) {
|
|
7044
|
-
const { queryTokens,
|
|
7396
|
+
const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
7045
7397
|
const fieldNames = state.fieldNameTable.names;
|
|
7046
7398
|
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
7399
|
+
const searched = searchedFieldMarks(fieldNames, fields);
|
|
7400
|
+
const partial = partialFieldMarks(fieldNames, params.partialFields);
|
|
7401
|
+
const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
|
|
7047
7402
|
for (const qt of queryTokens) {
|
|
7048
|
-
|
|
7049
|
-
|
|
7050
|
-
|
|
7051
|
-
if (seen.has(term)) continue;
|
|
7052
|
-
seen.add(term);
|
|
7053
|
-
const postingList = state.invertedIdx.lookup(term);
|
|
7054
|
-
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
7055
|
-
}
|
|
7056
|
-
continue;
|
|
7403
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
7404
|
+
for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
7405
|
+
markPostingList(matched, match.postingList, searched, filterBitset);
|
|
7057
7406
|
}
|
|
7058
|
-
if (
|
|
7059
|
-
|
|
7060
|
-
|
|
7061
|
-
continue;
|
|
7062
|
-
|
|
7063
|
-
for (const match of state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength)) {
|
|
7064
|
-
markPostingList(matched, match.postingList, fieldNames, fields, filterBitset);
|
|
7407
|
+
if (group === void 0) continue;
|
|
7408
|
+
for (const member of group.members) {
|
|
7409
|
+
const postingList = state.invertedIdx.lookup(member.token);
|
|
7410
|
+
if (postingList === void 0) continue;
|
|
7411
|
+
markPostingList(matched, postingList, member.factor > 0 ? searched : searchedPartial, filterBitset);
|
|
7065
7412
|
}
|
|
7066
7413
|
}
|
|
7067
7414
|
const docStore = state.docStore;
|
|
@@ -7149,6 +7496,7 @@ function estimatePartitionBytes(state) {
|
|
|
7149
7496
|
bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7150
7497
|
bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7151
7498
|
for (const index of state.patternIndexes.values()) bytes += index.bytes();
|
|
7499
|
+
bytes += state.partialWords.bytes();
|
|
7152
7500
|
return bytes + sortColumnBytes(state);
|
|
7153
7501
|
}
|
|
7154
7502
|
|
|
@@ -7280,6 +7628,7 @@ function mergeSegmentState(target, source) {
|
|
|
7280
7628
|
mergeFieldIndexes(target, source, idMapping);
|
|
7281
7629
|
mergeStatistics(target, source);
|
|
7282
7630
|
mergeSurfaceForms(target, source);
|
|
7631
|
+
target.partialWords.afterMerge(source.partialWords.spellings(), [...idMapping.values()]);
|
|
7283
7632
|
target.sortColumns = null;
|
|
7284
7633
|
target.scoreBuffer = null;
|
|
7285
7634
|
}
|
|
@@ -7339,70 +7688,10 @@ function rebuildTextIndex(state, schema, language, options) {
|
|
|
7339
7688
|
}
|
|
7340
7689
|
} finally {
|
|
7341
7690
|
state.invertedIdx.endBatch();
|
|
7691
|
+
state.partialWords.forgetSpellings();
|
|
7342
7692
|
}
|
|
7343
7693
|
}
|
|
7344
7694
|
|
|
7345
|
-
// src/core/scorer.ts
|
|
7346
|
-
var DEFAULT_K1 = 1.2;
|
|
7347
|
-
var DEFAULT_B = 0.75;
|
|
7348
|
-
function computeIDF(docFrequency, totalDocs) {
|
|
7349
|
-
if (totalDocs === 0) return 0;
|
|
7350
|
-
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
7351
|
-
}
|
|
7352
|
-
function bm25PruningSound(params) {
|
|
7353
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7354
|
-
const b = params?.b ?? DEFAULT_B;
|
|
7355
|
-
return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
|
|
7356
|
-
}
|
|
7357
|
-
function resolveBM25Params(params) {
|
|
7358
|
-
return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
|
|
7359
|
-
}
|
|
7360
|
-
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
7361
|
-
if (avgFieldLength === 0) return 0;
|
|
7362
|
-
const numerator = termFrequency * (k1 + 1);
|
|
7363
|
-
const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
|
|
7364
|
-
if (denominator === 0) return 0;
|
|
7365
|
-
return idf * (numerator / denominator);
|
|
7366
|
-
}
|
|
7367
|
-
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
7368
|
-
if (totalDocs === 0) return 0;
|
|
7369
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7370
|
-
const b = params?.b ?? DEFAULT_B;
|
|
7371
|
-
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
7372
|
-
}
|
|
7373
|
-
|
|
7374
|
-
// src/core/partition/field-scoring.ts
|
|
7375
|
-
function loadFieldScoring(state, fields, boost, averageFieldLengths) {
|
|
7376
|
-
const fieldNames = state.fieldNameTable.names;
|
|
7377
|
-
const scoring = {
|
|
7378
|
-
searchable: new Uint8Array(fieldNames.length),
|
|
7379
|
-
boosts: new Float64Array(fieldNames.length),
|
|
7380
|
-
averageLengths: new Float64Array(fieldNames.length),
|
|
7381
|
-
lengthColumns: new Array(fieldNames.length).fill(null)
|
|
7382
|
-
};
|
|
7383
|
-
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
7384
|
-
const fieldName = fieldNames[fieldIndex];
|
|
7385
|
-
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
7386
|
-
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
7387
|
-
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
7388
|
-
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
7389
|
-
}
|
|
7390
|
-
return scoring;
|
|
7391
|
-
}
|
|
7392
|
-
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
7393
|
-
const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
|
|
7394
|
-
if (column === null || internalId >= column.length) return averageLength;
|
|
7395
|
-
const stored = column[internalId];
|
|
7396
|
-
return stored > 0 ? stored : averageLength;
|
|
7397
|
-
}
|
|
7398
|
-
function bestSearchable(searchable, values) {
|
|
7399
|
-
let best = 0;
|
|
7400
|
-
for (let index = 0; index < searchable.length; index++) {
|
|
7401
|
-
if (searchable[index] === 1 && values[index] > best) best = values[index];
|
|
7402
|
-
}
|
|
7403
|
-
return best;
|
|
7404
|
-
}
|
|
7405
|
-
|
|
7406
7695
|
// src/core/partition/block-bounds.ts
|
|
7407
7696
|
var ABSENT_FIELD_LENGTH_BOUND = 0;
|
|
7408
7697
|
var boundsByList = /* @__PURE__ */ new WeakMap();
|
|
@@ -7498,37 +7787,6 @@ function blockBoundsFor(list, columns) {
|
|
|
7498
7787
|
return built;
|
|
7499
7788
|
}
|
|
7500
7789
|
|
|
7501
|
-
// src/core/partition/scoring.ts
|
|
7502
|
-
var EMPTY_COMPONENTS = Object.freeze({});
|
|
7503
|
-
function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
|
|
7504
|
-
const existing = components.get(internalId);
|
|
7505
|
-
if (existing) {
|
|
7506
|
-
existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
|
|
7507
|
-
existing.fieldLengths[fieldName] = fieldLength;
|
|
7508
|
-
existing.idf[token] = idf;
|
|
7509
|
-
return;
|
|
7510
|
-
}
|
|
7511
|
-
components.set(internalId, {
|
|
7512
|
-
termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
|
|
7513
|
-
fieldLengths: { [fieldName]: fieldLength },
|
|
7514
|
-
idf: { [token]: idf }
|
|
7515
|
-
});
|
|
7516
|
-
}
|
|
7517
|
-
function mergePrefixComponents(components, internalId, contribution) {
|
|
7518
|
-
const existing = components.get(internalId);
|
|
7519
|
-
if (existing) {
|
|
7520
|
-
Object.assign(existing.termFrequencies, contribution.termFrequencies);
|
|
7521
|
-
Object.assign(existing.fieldLengths, contribution.fieldLengths);
|
|
7522
|
-
existing.idf[contribution.token] = contribution.idf;
|
|
7523
|
-
return;
|
|
7524
|
-
}
|
|
7525
|
-
components.set(internalId, {
|
|
7526
|
-
termFrequencies: contribution.termFrequencies,
|
|
7527
|
-
fieldLengths: contribution.fieldLengths,
|
|
7528
|
-
idf: { [contribution.token]: contribution.idf }
|
|
7529
|
-
});
|
|
7530
|
-
}
|
|
7531
|
-
|
|
7532
7790
|
// src/core/partition/top-k-heap.ts
|
|
7533
7791
|
function candidateWorse(a, b) {
|
|
7534
7792
|
if (a.score !== b.score) return a.score < b.score;
|
|
@@ -7692,7 +7950,7 @@ function topKFromBuffer(buffer, k, resolver, components) {
|
|
|
7692
7950
|
// src/core/partition/multi-term-topk.ts
|
|
7693
7951
|
function prunableMultiTermLists(params, index) {
|
|
7694
7952
|
if (params.queryTokens.length < 2) return null;
|
|
7695
|
-
if (params.
|
|
7953
|
+
if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
|
|
7696
7954
|
if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
|
|
7697
7955
|
if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
|
|
7698
7956
|
if (params.collectComponents !== false) return null;
|
|
@@ -7864,87 +8122,6 @@ function multiTermTopK(request) {
|
|
|
7864
8122
|
return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
|
|
7865
8123
|
}
|
|
7866
8124
|
|
|
7867
|
-
// src/core/partition/prefix-scoring.ts
|
|
7868
|
-
function resolvePrefixMatches(context, token, expansionTerms) {
|
|
7869
|
-
const found = [];
|
|
7870
|
-
const seen = /* @__PURE__ */ new Set();
|
|
7871
|
-
for (const term of [token, ...expansionTerms]) {
|
|
7872
|
-
if (seen.has(term)) continue;
|
|
7873
|
-
seen.add(term);
|
|
7874
|
-
const postingList = context.index.lookup(term);
|
|
7875
|
-
if (!postingList) continue;
|
|
7876
|
-
found.push({ token: term, postingList, docFreq: context.documentFrequency(term, postingList) });
|
|
7877
|
-
}
|
|
7878
|
-
if (found.length === 0) return [];
|
|
7879
|
-
let blendedDf = 0;
|
|
7880
|
-
for (const f of found) {
|
|
7881
|
-
if (f.docFreq > blendedDf) blendedDf = f.docFreq;
|
|
7882
|
-
}
|
|
7883
|
-
const blendedIdf = computeIDF(blendedDf, context.totalDocs);
|
|
7884
|
-
return found.map((f) => ({
|
|
7885
|
-
token: f.token,
|
|
7886
|
-
factor: Math.min(1, token.length / f.token.length),
|
|
7887
|
-
postingList: f.postingList,
|
|
7888
|
-
docFreq: blendedDf,
|
|
7889
|
-
idf: blendedIdf
|
|
7890
|
-
}));
|
|
7891
|
-
}
|
|
7892
|
-
function computePrefixContributions(context, matches, collect) {
|
|
7893
|
-
const { fields, fieldNames, filterBitset, k1, b } = context;
|
|
7894
|
-
const scoresAreZero = context.totalDocs === 0;
|
|
7895
|
-
const best = /* @__PURE__ */ new Map();
|
|
7896
|
-
for (const match of matches) {
|
|
7897
|
-
const perTerm = /* @__PURE__ */ new Map();
|
|
7898
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
7899
|
-
match.postingList
|
|
7900
|
-
);
|
|
7901
|
-
for (let pi = 0; pi < count; pi++) {
|
|
7902
|
-
const internalId = docIds[pi];
|
|
7903
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7904
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
7905
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
7906
|
-
if (fields.searchable[fieldIndex] === 0) continue;
|
|
7907
|
-
const fieldName = fieldNames[fieldIndex];
|
|
7908
|
-
const termFrequency = termFrequencies[pi];
|
|
7909
|
-
const fieldBoost = fields.boosts[fieldIndex];
|
|
7910
|
-
const avgLen = fields.averageLengths[fieldIndex];
|
|
7911
|
-
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
7912
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * match.factor;
|
|
7913
|
-
const existing = perTerm.get(internalId);
|
|
7914
|
-
if (existing) {
|
|
7915
|
-
existing.score += termScore;
|
|
7916
|
-
if (collect) {
|
|
7917
|
-
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
7918
|
-
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
7919
|
-
}
|
|
7920
|
-
} else if (collect) {
|
|
7921
|
-
perTerm.set(internalId, {
|
|
7922
|
-
score: termScore,
|
|
7923
|
-
token: match.token,
|
|
7924
|
-
idf: match.idf,
|
|
7925
|
-
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
7926
|
-
fieldLengths: { [fieldName]: actualFieldLength }
|
|
7927
|
-
});
|
|
7928
|
-
} else {
|
|
7929
|
-
perTerm.set(internalId, {
|
|
7930
|
-
score: termScore,
|
|
7931
|
-
token: match.token,
|
|
7932
|
-
idf: match.idf,
|
|
7933
|
-
termFrequencies: EMPTY_COMPONENTS,
|
|
7934
|
-
fieldLengths: EMPTY_COMPONENTS
|
|
7935
|
-
});
|
|
7936
|
-
}
|
|
7937
|
-
}
|
|
7938
|
-
for (const [internalId, contribution] of perTerm) {
|
|
7939
|
-
const current = best.get(internalId);
|
|
7940
|
-
if (!current || contribution.score > current.score) {
|
|
7941
|
-
best.set(internalId, contribution);
|
|
7942
|
-
}
|
|
7943
|
-
}
|
|
7944
|
-
}
|
|
7945
|
-
return best;
|
|
7946
|
-
}
|
|
7947
|
-
|
|
7948
8125
|
// src/core/partition/single-term-topk.ts
|
|
7949
8126
|
function fieldsCoverEvery(fields, fieldNames) {
|
|
7950
8127
|
if (fields === void 0) return true;
|
|
@@ -7955,7 +8132,7 @@ function fieldsCoverEvery(fields, fieldNames) {
|
|
|
7955
8132
|
}
|
|
7956
8133
|
function prunableSingleTermList(params, index, fieldNames) {
|
|
7957
8134
|
if (params.queryTokens.length !== 1) return null;
|
|
7958
|
-
if (params.
|
|
8135
|
+
if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
|
|
7959
8136
|
if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
|
|
7960
8137
|
if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
|
|
7961
8138
|
if (params.collectComponents !== false) return null;
|
|
@@ -8065,15 +8242,11 @@ function singleTermTopK(request) {
|
|
|
8065
8242
|
function globalDocFreqFor(docFreqs, term, fallback) {
|
|
8066
8243
|
return Object.hasOwn(docFreqs, term) ? docFreqs[term] : fallback;
|
|
8067
8244
|
}
|
|
8068
|
-
function matchesFor(index, token, exact, tolerance, prefixLength) {
|
|
8069
|
-
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
8070
|
-
const postingList = index.lookup(token);
|
|
8071
|
-
return postingList ? [{ token, postingList }] : [];
|
|
8072
|
-
}
|
|
8073
8245
|
function searchFulltext(state, params) {
|
|
8074
8246
|
const {
|
|
8075
8247
|
queryTokens,
|
|
8076
|
-
|
|
8248
|
+
termGroups,
|
|
8249
|
+
partialFields,
|
|
8077
8250
|
fields,
|
|
8078
8251
|
boost,
|
|
8079
8252
|
tolerance = 0,
|
|
@@ -8105,19 +8278,75 @@ function searchFulltext(state, params) {
|
|
|
8105
8278
|
function documentFrequency(token, list) {
|
|
8106
8279
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
8107
8280
|
}
|
|
8108
|
-
const
|
|
8281
|
+
const groupContext = {
|
|
8109
8282
|
index: state.invertedIdx,
|
|
8110
8283
|
fields: fieldScoring,
|
|
8284
|
+
partialMarks: partialFieldMarks(fieldNames, partialFields),
|
|
8111
8285
|
fieldNames,
|
|
8112
8286
|
totalDocs,
|
|
8113
8287
|
k1,
|
|
8114
8288
|
b,
|
|
8115
8289
|
filterBitset,
|
|
8116
|
-
|
|
8290
|
+
globalFrequency(token) {
|
|
8291
|
+
if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
|
|
8292
|
+
return globalStats.docFrequencies[token];
|
|
8293
|
+
}
|
|
8117
8294
|
};
|
|
8118
|
-
function
|
|
8295
|
+
function mergeGroupContribution(internalId, contribution) {
|
|
8119
8296
|
addScore(scoreBuffer, internalId, contribution.score);
|
|
8120
|
-
if (components !== null)
|
|
8297
|
+
if (components !== null) mergeGroupComponents(components, internalId, contribution);
|
|
8298
|
+
}
|
|
8299
|
+
function resolveMatch(token, postingList) {
|
|
8300
|
+
const docFreq = documentFrequency(token, postingList);
|
|
8301
|
+
return { token, docFreq, idf: computeIDF(docFreq, totalDocs), postingList };
|
|
8302
|
+
}
|
|
8303
|
+
function scorePostings(match, onlyScored) {
|
|
8304
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
8305
|
+
match.postingList
|
|
8306
|
+
);
|
|
8307
|
+
if (!onlyScored && !hasDeleted && filterBitset === void 0 && components === null) {
|
|
8308
|
+
for (let pi = 0; pi < count; pi++) {
|
|
8309
|
+
const internalId = docIds[pi];
|
|
8310
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
8311
|
+
if (searchable[fieldIndex] === 0) continue;
|
|
8312
|
+
const avgLen = averageLengths[fieldIndex];
|
|
8313
|
+
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8314
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], match.idf, actualFieldLength, avgLen, k1, b);
|
|
8315
|
+
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8316
|
+
}
|
|
8317
|
+
return;
|
|
8318
|
+
}
|
|
8319
|
+
for (let pi = 0; pi < count; pi++) {
|
|
8320
|
+
const internalId = docIds[pi];
|
|
8321
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8322
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
8323
|
+
if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
|
|
8324
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
8325
|
+
if (searchable[fieldIndex] === 0) continue;
|
|
8326
|
+
const termFrequency = termFrequencies[pi];
|
|
8327
|
+
const avgLen = averageLengths[fieldIndex];
|
|
8328
|
+
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8329
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
|
|
8330
|
+
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8331
|
+
if (components !== null) {
|
|
8332
|
+
recordComponents(
|
|
8333
|
+
components,
|
|
8334
|
+
internalId,
|
|
8335
|
+
fieldNames[fieldIndex],
|
|
8336
|
+
match.token,
|
|
8337
|
+
termFrequency,
|
|
8338
|
+
actualFieldLength,
|
|
8339
|
+
match.idf
|
|
8340
|
+
);
|
|
8341
|
+
}
|
|
8342
|
+
}
|
|
8343
|
+
}
|
|
8344
|
+
function scoreGroup(group, onlyScored) {
|
|
8345
|
+
if (group === null) return;
|
|
8346
|
+
for (const [internalId, contribution] of computeGroupContributions(groupContext, group, collectComponents)) {
|
|
8347
|
+
if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
|
|
8348
|
+
mergeGroupContribution(internalId, contribution);
|
|
8349
|
+
}
|
|
8121
8350
|
}
|
|
8122
8351
|
const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
|
|
8123
8352
|
if (prunableList !== null && maxResults !== void 0) {
|
|
@@ -8147,127 +8376,30 @@ function searchFulltext(state, params) {
|
|
|
8147
8376
|
}
|
|
8148
8377
|
if (useIntersection) {
|
|
8149
8378
|
const resolved = [];
|
|
8150
|
-
let prefixMatches = [];
|
|
8151
8379
|
for (const qt of queryTokens) {
|
|
8152
|
-
|
|
8153
|
-
|
|
8154
|
-
|
|
8155
|
-
for (const m of prefixMatches) {
|
|
8156
|
-
totalPostings2 += m.postingList.length;
|
|
8157
|
-
}
|
|
8158
|
-
resolved.push({ token: qt.token, matches: [], totalPostings: totalPostings2, isPrefix: true });
|
|
8159
|
-
continue;
|
|
8160
|
-
}
|
|
8161
|
-
const rawMatches = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
|
|
8162
|
-
let totalPostings = 0;
|
|
8380
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
8381
|
+
const groupMatches = group === void 0 ? null : resolveGroupMatches(groupContext, group);
|
|
8382
|
+
let totalPostings = groupMatches === null ? 0 : groupPostingCount(groupMatches);
|
|
8163
8383
|
const matches = [];
|
|
8164
|
-
for (const m of
|
|
8165
|
-
const docFreq = documentFrequency(m.token, m.postingList);
|
|
8166
|
-
const idf = computeIDF(docFreq, totalDocs);
|
|
8384
|
+
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
8167
8385
|
totalPostings += m.postingList.length;
|
|
8168
|
-
matches.push(
|
|
8386
|
+
matches.push(resolveMatch(m.token, m.postingList));
|
|
8169
8387
|
}
|
|
8170
|
-
resolved.push({ token: qt.token, matches, totalPostings });
|
|
8388
|
+
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
8171
8389
|
}
|
|
8172
8390
|
resolved.sort((a, b2) => a.totalPostings - b2.totalPostings);
|
|
8173
8391
|
for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
|
|
8174
|
-
|
|
8175
|
-
|
|
8176
|
-
|
|
8177
|
-
if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
|
|
8178
|
-
mergePrefixContribution(internalId, contribution);
|
|
8179
|
-
}
|
|
8180
|
-
continue;
|
|
8181
|
-
}
|
|
8182
|
-
for (const match of resolved[tokenIndex].matches) {
|
|
8183
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
8184
|
-
match.postingList
|
|
8185
|
-
);
|
|
8186
|
-
for (let pi = 0; pi < count; pi++) {
|
|
8187
|
-
const internalId = docIds[pi];
|
|
8188
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8189
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
8190
|
-
if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
|
|
8191
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
8192
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
8193
|
-
const termFrequency = termFrequencies[pi];
|
|
8194
|
-
const fieldBoost = boosts[fieldIndex];
|
|
8195
|
-
const avgLen = averageLengths[fieldIndex];
|
|
8196
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8197
|
-
let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
|
|
8198
|
-
termScore *= fieldBoost;
|
|
8199
|
-
addScore(scoreBuffer, internalId, termScore);
|
|
8200
|
-
if (components !== null) {
|
|
8201
|
-
recordComponents(
|
|
8202
|
-
components,
|
|
8203
|
-
internalId,
|
|
8204
|
-
fieldNames[fieldIndex],
|
|
8205
|
-
match.token,
|
|
8206
|
-
termFrequency,
|
|
8207
|
-
actualFieldLength,
|
|
8208
|
-
match.idf
|
|
8209
|
-
);
|
|
8210
|
-
}
|
|
8211
|
-
}
|
|
8212
|
-
}
|
|
8392
|
+
const onlyScored = tokenIndex > 0;
|
|
8393
|
+
for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
|
|
8394
|
+
scoreGroup(resolved[tokenIndex].group, onlyScored);
|
|
8213
8395
|
}
|
|
8214
8396
|
} else {
|
|
8215
8397
|
for (const qt of queryTokens) {
|
|
8216
|
-
|
|
8217
|
-
|
|
8218
|
-
|
|
8219
|
-
resolvePrefixMatches(prefixContext, qt.token, prefixExpansion.terms),
|
|
8220
|
-
collectComponents
|
|
8221
|
-
);
|
|
8222
|
-
for (const [internalId, contribution] of contributions) {
|
|
8223
|
-
mergePrefixContribution(internalId, contribution);
|
|
8224
|
-
}
|
|
8225
|
-
continue;
|
|
8226
|
-
}
|
|
8227
|
-
const matchingPostings = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
|
|
8228
|
-
for (const match of matchingPostings) {
|
|
8229
|
-
const idf = computeIDF(documentFrequency(match.token, match.postingList), totalDocs);
|
|
8230
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
8231
|
-
match.postingList
|
|
8232
|
-
);
|
|
8233
|
-
if (!hasDeleted && filterBitset === void 0 && components === null) {
|
|
8234
|
-
for (let pi = 0; pi < count; pi++) {
|
|
8235
|
-
const internalId = docIds[pi];
|
|
8236
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
8237
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
8238
|
-
const avgLen = averageLengths[fieldIndex];
|
|
8239
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8240
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], idf, actualFieldLength, avgLen, k1, b);
|
|
8241
|
-
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8242
|
-
}
|
|
8243
|
-
continue;
|
|
8244
|
-
}
|
|
8245
|
-
for (let pi = 0; pi < count; pi++) {
|
|
8246
|
-
const internalId = docIds[pi];
|
|
8247
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8248
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
8249
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
8250
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
8251
|
-
const termFrequency = termFrequencies[pi];
|
|
8252
|
-
const fieldBoost = boosts[fieldIndex];
|
|
8253
|
-
const avgLen = averageLengths[fieldIndex];
|
|
8254
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8255
|
-
let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, idf, actualFieldLength, avgLen, k1, b);
|
|
8256
|
-
termScore *= fieldBoost;
|
|
8257
|
-
addScore(scoreBuffer, internalId, termScore);
|
|
8258
|
-
if (components !== null) {
|
|
8259
|
-
recordComponents(
|
|
8260
|
-
components,
|
|
8261
|
-
internalId,
|
|
8262
|
-
fieldNames[fieldIndex],
|
|
8263
|
-
match.token,
|
|
8264
|
-
termFrequency,
|
|
8265
|
-
actualFieldLength,
|
|
8266
|
-
idf
|
|
8267
|
-
);
|
|
8268
|
-
}
|
|
8269
|
-
}
|
|
8398
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
8399
|
+
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
8400
|
+
scorePostings(resolveMatch(m.token, m.postingList), false);
|
|
8270
8401
|
}
|
|
8402
|
+
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
|
|
8271
8403
|
}
|
|
8272
8404
|
}
|
|
8273
8405
|
const totalMatched = scoreBuffer.touchedCount;
|
|
@@ -8476,6 +8608,7 @@ function encodeSegmentState(state) {
|
|
|
8476
8608
|
totalFieldLengths: fieldLengths.totals,
|
|
8477
8609
|
docFrequencies,
|
|
8478
8610
|
surfaceForms: state.surfaceRegistry.size() === 0 ? null : state.surfaceRegistry.serialize(),
|
|
8611
|
+
partialSpellings: state.partialWords.spellings(),
|
|
8479
8612
|
...encodeFieldIndexes(state, remap)
|
|
8480
8613
|
};
|
|
8481
8614
|
}
|
|
@@ -8594,6 +8727,8 @@ function mergeSegmentPayload(target, payload, documents) {
|
|
|
8594
8727
|
target.surfaceRegistry.add(surface, value[1], value[0]);
|
|
8595
8728
|
}
|
|
8596
8729
|
}
|
|
8730
|
+
const mergedIds = Array.from({ length: payload.docIds.length }, (_, at) => ordinalBase + at);
|
|
8731
|
+
target.partialWords.afterMerge(payload.partialSpellings, mergedIds);
|
|
8597
8732
|
target.sortColumns = null;
|
|
8598
8733
|
target.scoreBuffer = null;
|
|
8599
8734
|
}
|
|
@@ -8883,6 +9018,7 @@ function deserializePartition(state, data, clearFn, schema) {
|
|
|
8883
9018
|
}
|
|
8884
9019
|
loadPatternIndexes(state, data.fieldIndexes.pattern, schema);
|
|
8885
9020
|
rebuildSizeIndexes(state, schema);
|
|
9021
|
+
state.partialWords.forgetSpellings();
|
|
8886
9022
|
}
|
|
8887
9023
|
|
|
8888
9024
|
// src/core/partition/suggestions.ts
|
|
@@ -8928,27 +9064,33 @@ function suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit) {
|
|
|
8928
9064
|
if (results.length > limit) results.length = limit;
|
|
8929
9065
|
return results;
|
|
8930
9066
|
}
|
|
8931
|
-
function
|
|
9067
|
+
function partitionDocumentFrequency(subs, token) {
|
|
9068
|
+
let documents = 0;
|
|
9069
|
+
for (const sub of subs) documents += sub.invertedIdx.documentFrequency(token);
|
|
9070
|
+
return documents;
|
|
9071
|
+
}
|
|
9072
|
+
function expandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
|
|
8932
9073
|
if (maxExpansions <= 0 || surfacePrefix.length === 0) return [];
|
|
8933
|
-
const
|
|
8934
|
-
|
|
8935
|
-
|
|
8936
|
-
|
|
8937
|
-
|
|
8938
|
-
|
|
8939
|
-
|
|
8940
|
-
|
|
8941
|
-
|
|
8942
|
-
|
|
8943
|
-
if (dfByToken.has(suggestion.term)) continue;
|
|
8944
|
-
if (verbatimOccurrences(state, suggestion.term) === 0) continue;
|
|
8945
|
-
dfByToken.set(suggestion.term, suggestion.documentFrequency);
|
|
9074
|
+
const candidates = /* @__PURE__ */ new Set();
|
|
9075
|
+
const holdsSurfaceForms = subs.some((sub) => sub.surfaceRegistry.size() > 0);
|
|
9076
|
+
const termPrefixes = !holdsSurfaceForms && stemmedToken !== surfacePrefix && stemmedToken.length > 0 ? [surfacePrefix, stemmedToken] : [surfacePrefix];
|
|
9077
|
+
for (const sub of subs) {
|
|
9078
|
+
for (const candidate of sub.surfaceRegistry.candidatesForPrefix(surfacePrefix)) candidates.add(candidate.token);
|
|
9079
|
+
for (const prefix of termPrefixes) {
|
|
9080
|
+
for (const suggestion of sub.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
9081
|
+
if (candidates.has(suggestion.term)) continue;
|
|
9082
|
+
if (verbatimOccurrences(sub, suggestion.term) > 0) candidates.add(suggestion.term);
|
|
9083
|
+
}
|
|
8946
9084
|
}
|
|
8947
9085
|
}
|
|
8948
|
-
const
|
|
8949
|
-
|
|
8950
|
-
|
|
8951
|
-
|
|
9086
|
+
const ranked = [];
|
|
9087
|
+
for (const token of candidates) {
|
|
9088
|
+
const documents = partitionDocumentFrequency(subs, token);
|
|
9089
|
+
if (documents > 0) ranked.push({ token, documents });
|
|
9090
|
+
}
|
|
9091
|
+
ranked.sort((a, b) => b.documents - a.documents || compareCodePoints(a.token, b.token));
|
|
9092
|
+
if (ranked.length > maxExpansions) ranked.length = maxExpansions;
|
|
9093
|
+
return ranked.map((entry) => entry.token);
|
|
8952
9094
|
}
|
|
8953
9095
|
|
|
8954
9096
|
// src/core/partition/index.ts
|
|
@@ -8965,9 +9107,10 @@ function partitionStateOf(partition) {
|
|
|
8965
9107
|
}
|
|
8966
9108
|
function createPartitionIndex(partitionId, trackPositions = true) {
|
|
8967
9109
|
const fieldNameTable = { names: [], indexMap: /* @__PURE__ */ new Map() };
|
|
9110
|
+
const docStore = createDocumentStore();
|
|
8968
9111
|
const state = {
|
|
8969
9112
|
invertedIdx: createInvertedIndex(fieldNameTable),
|
|
8970
|
-
docStore
|
|
9113
|
+
docStore,
|
|
8971
9114
|
stats: createPartitionStats(),
|
|
8972
9115
|
surfaceRegistry: createSurfaceRegistry(),
|
|
8973
9116
|
numericIndexes: /* @__PURE__ */ new Map(),
|
|
@@ -8977,6 +9120,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8977
9120
|
patternIndexes: /* @__PURE__ */ new Map(),
|
|
8978
9121
|
sizeIndexes: /* @__PURE__ */ new Map(),
|
|
8979
9122
|
fieldNameTable,
|
|
9123
|
+
partialWords: createGrowingPartialWords({ docStore }),
|
|
8980
9124
|
flatSchemaCache: null,
|
|
8981
9125
|
lastSchemaRef: null,
|
|
8982
9126
|
trackPositions,
|
|
@@ -8997,6 +9141,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8997
9141
|
state.geoIndexes.clear();
|
|
8998
9142
|
state.patternIndexes.clear();
|
|
8999
9143
|
state.sizeIndexes.clear();
|
|
9144
|
+
state.partialWords.clear();
|
|
9000
9145
|
state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
|
|
9001
9146
|
state.flatSchemaCache = null;
|
|
9002
9147
|
state.lastSchemaRef = null;
|
|
@@ -9025,7 +9170,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9025
9170
|
}
|
|
9026
9171
|
}
|
|
9027
9172
|
const internalId = state.docStore.ensureInternalId(docId);
|
|
9028
|
-
const { fieldLengths, tokensByField } = indexDocument(
|
|
9173
|
+
const { fieldLengths, tokensByField, partialSpellings } = indexDocument(
|
|
9029
9174
|
state,
|
|
9030
9175
|
docId,
|
|
9031
9176
|
document2,
|
|
@@ -9039,6 +9184,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9039
9184
|
state.docStore.store(docId, document2, fieldLengths);
|
|
9040
9185
|
}
|
|
9041
9186
|
state.stats.addDocument(fieldLengths, tokensByField);
|
|
9187
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, 1);
|
|
9042
9188
|
recordSortValues(state, internalId, document2);
|
|
9043
9189
|
},
|
|
9044
9190
|
remove(docId, schema, language, options) {
|
|
@@ -9049,7 +9195,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9049
9195
|
partitionId
|
|
9050
9196
|
});
|
|
9051
9197
|
}
|
|
9052
|
-
const { fieldLengths, tokensByField } = removeFromIndexes(
|
|
9198
|
+
const { fieldLengths, tokensByField, partialSpellings } = removeFromIndexes(
|
|
9053
9199
|
state,
|
|
9054
9200
|
docId,
|
|
9055
9201
|
stored,
|
|
@@ -9060,6 +9206,9 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9060
9206
|
const internalId = state.docStore.getInternalId(docId);
|
|
9061
9207
|
state.docStore.remove(docId);
|
|
9062
9208
|
state.stats.removeDocument(fieldLengths, tokensByField);
|
|
9209
|
+
if (internalId !== void 0) {
|
|
9210
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, -1);
|
|
9211
|
+
}
|
|
9063
9212
|
forgetSortValues(state, internalId);
|
|
9064
9213
|
},
|
|
9065
9214
|
beginBatch() {
|
|
@@ -9095,29 +9244,19 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9095
9244
|
const fields = schemaFieldsOf(schema);
|
|
9096
9245
|
const needsTextReindex = textFieldsChanged(stored.fields, document2, fields);
|
|
9097
9246
|
if (needsTextReindex) {
|
|
9098
|
-
const
|
|
9099
|
-
|
|
9100
|
-
docId,
|
|
9101
|
-
stored,
|
|
9102
|
-
fields,
|
|
9103
|
-
language,
|
|
9104
|
-
options
|
|
9105
|
-
);
|
|
9106
|
-
state.stats.removeDocument(oldFieldLengths, oldTokens);
|
|
9247
|
+
const previous = removeFromIndexes(state, docId, stored, fields, language, options);
|
|
9248
|
+
state.stats.removeDocument(previous.fieldLengths, previous.tokensByField);
|
|
9107
9249
|
const previousInternalId = state.docStore.getInternalId(docId);
|
|
9250
|
+
if (previousInternalId !== void 0) {
|
|
9251
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), previousInternalId, previous.partialSpellings, -1);
|
|
9252
|
+
}
|
|
9108
9253
|
state.docStore.remove(docId);
|
|
9109
9254
|
forgetSortValues(state, previousInternalId);
|
|
9110
9255
|
const internalId = state.docStore.ensureInternalId(docId);
|
|
9111
|
-
const
|
|
9112
|
-
|
|
9113
|
-
|
|
9114
|
-
|
|
9115
|
-
fields,
|
|
9116
|
-
language,
|
|
9117
|
-
options
|
|
9118
|
-
);
|
|
9119
|
-
state.docStore.store(docId, document2, newFieldLengths);
|
|
9120
|
-
state.stats.addDocument(newFieldLengths, newTokens);
|
|
9256
|
+
const next = indexDocument(state, docId, document2, fields, language, options);
|
|
9257
|
+
state.docStore.store(docId, document2, next.fieldLengths);
|
|
9258
|
+
state.stats.addDocument(next.fieldLengths, next.tokensByField);
|
|
9259
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, next.partialSpellings, 1);
|
|
9121
9260
|
recordSortValues(state, internalId, document2);
|
|
9122
9261
|
} else {
|
|
9123
9262
|
updateFieldIndexOnly(state, docId, stored.fields, document2, fields);
|
|
@@ -9188,7 +9327,13 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9188
9327
|
return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
|
|
9189
9328
|
},
|
|
9190
9329
|
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
9191
|
-
return expandTermPrefix(state, surfacePrefix, stemmedToken, maxExpansions);
|
|
9330
|
+
return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
|
|
9331
|
+
},
|
|
9332
|
+
expandPartialWords(analysis, requests) {
|
|
9333
|
+
return expandPartialWords([state], analysis, requests);
|
|
9334
|
+
},
|
|
9335
|
+
preparePartialWords(analysis) {
|
|
9336
|
+
expandPartialWords([state], analysis, []);
|
|
9192
9337
|
},
|
|
9193
9338
|
serialize(indexName, totalPartitions, language, schema) {
|
|
9194
9339
|
return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
|
|
@@ -9939,6 +10084,8 @@ var MAX_SORT_FIELDS = 8;
|
|
|
9939
10084
|
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
9940
10085
|
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
9941
10086
|
var PREFIX_MAX_EXPANSIONS = 50;
|
|
10087
|
+
var PARTIAL_WORDS_ADDED_PER_QUERY = 50;
|
|
10088
|
+
var PARTIAL_WORD_MIN_CODE_POINTS = 3;
|
|
9942
10089
|
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
9943
10090
|
var MAX_TOLERANCE = 10;
|
|
9944
10091
|
var MAX_PREFIX_LENGTH = 1024;
|
|
@@ -10256,36 +10403,120 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
|
|
|
10256
10403
|
return { paginated: sliced, nextCursor };
|
|
10257
10404
|
}
|
|
10258
10405
|
|
|
10259
|
-
// src/search/
|
|
10260
|
-
function
|
|
10261
|
-
const term = params.term;
|
|
10262
|
-
if (term === void 0) return void 0;
|
|
10406
|
+
// src/search/term-groups.ts
|
|
10407
|
+
function writtenWordsByPosition(term, language, analysis) {
|
|
10263
10408
|
const unstemmed = tokenize(term, language, {
|
|
10264
10409
|
stem: false,
|
|
10265
10410
|
removeStopWords: true,
|
|
10266
|
-
customTokenizer:
|
|
10267
|
-
stopWordOverride:
|
|
10411
|
+
customTokenizer: analysis?.customTokenizer,
|
|
10412
|
+
stopWordOverride: analysis?.stopWords
|
|
10268
10413
|
});
|
|
10269
|
-
|
|
10270
|
-
for (const
|
|
10271
|
-
if (
|
|
10272
|
-
|
|
10273
|
-
|
|
10414
|
+
const written = /* @__PURE__ */ new Map();
|
|
10415
|
+
for (const token of unstemmed.tokens) {
|
|
10416
|
+
if (!written.has(token.position)) written.set(token.position, token.token);
|
|
10417
|
+
}
|
|
10418
|
+
return written;
|
|
10419
|
+
}
|
|
10420
|
+
function groupFor(groups, token, typoMatches) {
|
|
10421
|
+
let group = groups.get(token);
|
|
10422
|
+
if (group === void 0) {
|
|
10423
|
+
group = { token, members: [{ token, factor: 1, partialFactor: 1 }], typoMatches };
|
|
10424
|
+
groups.set(token, group);
|
|
10425
|
+
}
|
|
10426
|
+
return group;
|
|
10427
|
+
}
|
|
10428
|
+
function addCompletions(groups, partition, lastToken, typedPrefix) {
|
|
10429
|
+
if (typedPrefix === void 0 || typedPrefix.length === 0) return;
|
|
10430
|
+
const group = groupFor(groups, lastToken.token, false);
|
|
10431
|
+
const ownLength = codePointCount(lastToken.token);
|
|
10432
|
+
for (const completion of partition.expandTermPrefix(typedPrefix, lastToken.token, PREFIX_MAX_EXPANSIONS)) {
|
|
10433
|
+
if (completion === lastToken.token) continue;
|
|
10434
|
+
const factor = Math.min(1, ownLength / codePointCount(completion));
|
|
10435
|
+
group.members.push({ token: completion, factor, partialFactor: factor });
|
|
10436
|
+
}
|
|
10437
|
+
}
|
|
10438
|
+
function addPartialWords(group, added) {
|
|
10439
|
+
const membersByToken = /* @__PURE__ */ new Map();
|
|
10440
|
+
for (const member of group.members) membersByToken.set(member.token, member);
|
|
10441
|
+
for (const word of added) {
|
|
10442
|
+
const member = membersByToken.get(word.token);
|
|
10443
|
+
if (member === void 0) {
|
|
10444
|
+
group.members.push({ token: word.token, factor: 0, partialFactor: word.factor });
|
|
10445
|
+
} else if (word.factor > member.partialFactor) {
|
|
10446
|
+
member.partialFactor = word.factor;
|
|
10274
10447
|
}
|
|
10275
10448
|
}
|
|
10276
|
-
|
|
10277
|
-
|
|
10278
|
-
|
|
10279
|
-
|
|
10280
|
-
|
|
10449
|
+
}
|
|
10450
|
+
function partialWordAnalysisOf(fields, language, analysis) {
|
|
10451
|
+
return {
|
|
10452
|
+
fields,
|
|
10453
|
+
language,
|
|
10454
|
+
options: {
|
|
10455
|
+
stopWordOverride: analysis?.stopWords,
|
|
10456
|
+
customTokenizer: analysis?.customTokenizer,
|
|
10457
|
+
collectSurfaces: true
|
|
10458
|
+
}
|
|
10459
|
+
};
|
|
10460
|
+
}
|
|
10461
|
+
function searchesAPartialField(params, partialFields) {
|
|
10462
|
+
if (params.fields === void 0) return true;
|
|
10463
|
+
for (const field of params.fields) {
|
|
10464
|
+
if (partialFields.includes(field)) return true;
|
|
10281
10465
|
}
|
|
10282
|
-
|
|
10283
|
-
|
|
10284
|
-
|
|
10466
|
+
return false;
|
|
10467
|
+
}
|
|
10468
|
+
function partialWordRequests(params, queryTokens, written, prefixToken) {
|
|
10469
|
+
const eligible = [];
|
|
10470
|
+
for (const queryToken of queryTokens) {
|
|
10471
|
+
const word = written.get(queryToken.position);
|
|
10472
|
+
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
10473
|
+
eligible.push({ token: queryToken.token, written: word });
|
|
10474
|
+
if (eligible.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
10475
|
+
}
|
|
10476
|
+
const count = eligible.length;
|
|
10477
|
+
if (count === 0) return [];
|
|
10478
|
+
const evenShare = Math.floor(PARTIAL_WORDS_ADDED_PER_QUERY / count);
|
|
10479
|
+
const remainder = PARTIAL_WORDS_ADDED_PER_QUERY % count;
|
|
10480
|
+
return eligible.map((entry, order) => ({
|
|
10481
|
+
token: entry.token,
|
|
10482
|
+
request: {
|
|
10483
|
+
written: entry.written,
|
|
10484
|
+
ownToken: entry.token,
|
|
10485
|
+
share: evenShare + (order < remainder ? 1 : 0),
|
|
10486
|
+
tolerance: entry.token === prefixToken ? 0 : params.tolerance ?? 0,
|
|
10487
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
10488
|
+
}
|
|
10489
|
+
}));
|
|
10490
|
+
}
|
|
10491
|
+
function resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis) {
|
|
10492
|
+
const term = params.term;
|
|
10493
|
+
if (term === void 0 || params.exact === true) return {};
|
|
10494
|
+
const usesPrefix = params.prefix === true;
|
|
10495
|
+
const partialFields = params.partialWords === true ? partialFieldsOf(schema) : [];
|
|
10496
|
+
const usesPartialWords = partialFields.length > 0 && searchesAPartialField(params, partialFields);
|
|
10497
|
+
if (!usesPrefix && !usesPartialWords) return {};
|
|
10498
|
+
const written = writtenWordsByPosition(term, language, analysis);
|
|
10499
|
+
const groups = /* @__PURE__ */ new Map();
|
|
10500
|
+
if (usesPrefix) addCompletions(groups, partition, lastToken, written.get(lastToken.position));
|
|
10501
|
+
const prefixToken = groups.has(lastToken.token) ? lastToken.token : void 0;
|
|
10502
|
+
if (usesPartialWords) {
|
|
10503
|
+
const requests = partialWordRequests(params, queryTokens, written, prefixToken);
|
|
10504
|
+
const added = partition.expandPartialWords(
|
|
10505
|
+
partialWordAnalysisOf(partialFields, language, analysis),
|
|
10506
|
+
requests.map((entry) => entry.request)
|
|
10507
|
+
);
|
|
10508
|
+
for (let at = 0; at < requests.length; at++) {
|
|
10509
|
+
if (added[at].length === 0) continue;
|
|
10510
|
+
const token = requests[at].token;
|
|
10511
|
+
addPartialWords(groupFor(groups, token, token !== prefixToken), added[at]);
|
|
10285
10512
|
}
|
|
10286
10513
|
}
|
|
10287
|
-
|
|
10514
|
+
const termGroups = [...groups.values()];
|
|
10515
|
+
if (termGroups.length === 0) return {};
|
|
10516
|
+
return usesPartialWords ? { termGroups, partialFields: [...partialFields] } : { termGroups };
|
|
10288
10517
|
}
|
|
10518
|
+
|
|
10519
|
+
// src/search/fulltext.ts
|
|
10289
10520
|
function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
10290
10521
|
if (!params.term || params.term.trim().length === 0) return null;
|
|
10291
10522
|
if (params.fields && params.fields.length === 0) return null;
|
|
@@ -10301,11 +10532,6 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
10301
10532
|
});
|
|
10302
10533
|
if (queryTokenResult.tokens.length === 0) return null;
|
|
10303
10534
|
const queryTokens = deduplicateTokens(queryTokenResult.tokens);
|
|
10304
|
-
let prefixExpansion;
|
|
10305
|
-
if (params.prefix === true && params.exact !== true) {
|
|
10306
|
-
const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
|
|
10307
|
-
prefixExpansion = resolvePrefixExpansion(partition, params, language, options, queryTokens, lastToken);
|
|
10308
|
-
}
|
|
10309
10535
|
let filterBitset;
|
|
10310
10536
|
if (params.filters) {
|
|
10311
10537
|
filterBitset = partition.applyFiltersBitset(
|
|
@@ -10315,14 +10541,17 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
10315
10541
|
);
|
|
10316
10542
|
if (bitsetIsEmpty(filterBitset)) return null;
|
|
10317
10543
|
}
|
|
10318
|
-
|
|
10544
|
+
const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
|
|
10545
|
+
const groups = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, options);
|
|
10546
|
+
return { queryTokens, ...groups, filterBitset };
|
|
10319
10547
|
}
|
|
10320
10548
|
function fulltextMatches(partition, params, language, schema, options) {
|
|
10321
10549
|
const prepared = prepareFulltextQuery(partition, params, language, schema, options);
|
|
10322
10550
|
if (prepared === null) return null;
|
|
10323
10551
|
return partition.searchFulltextMatches({
|
|
10324
10552
|
queryTokens: prepared.queryTokens,
|
|
10325
|
-
|
|
10553
|
+
termGroups: prepared.termGroups,
|
|
10554
|
+
partialFields: prepared.partialFields,
|
|
10326
10555
|
fields: params.fields,
|
|
10327
10556
|
tolerance: params.tolerance ?? 0,
|
|
10328
10557
|
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
@@ -10335,14 +10564,15 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
10335
10564
|
if (prepared === null) {
|
|
10336
10565
|
return { scored: [], totalMatched: 0 };
|
|
10337
10566
|
}
|
|
10338
|
-
const { queryTokens,
|
|
10567
|
+
const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
|
|
10339
10568
|
const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
10340
10569
|
const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
|
|
10341
10570
|
const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
|
|
10342
10571
|
const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
|
|
10343
10572
|
const rawResult = partition.searchFulltext({
|
|
10344
10573
|
queryTokens,
|
|
10345
|
-
|
|
10574
|
+
termGroups,
|
|
10575
|
+
partialFields,
|
|
10346
10576
|
fields: params.fields,
|
|
10347
10577
|
boost: params.boost,
|
|
10348
10578
|
tolerance: params.tolerance ?? 0,
|
|
@@ -10366,7 +10596,8 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
10366
10596
|
params.tolerance ?? 0,
|
|
10367
10597
|
params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
10368
10598
|
params.exact ?? false,
|
|
10369
|
-
|
|
10599
|
+
termGroups,
|
|
10600
|
+
partialFields ?? []
|
|
10370
10601
|
);
|
|
10371
10602
|
}
|
|
10372
10603
|
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
@@ -10408,45 +10639,48 @@ function deduplicateTokens(tokens) {
|
|
|
10408
10639
|
}
|
|
10409
10640
|
return result;
|
|
10410
10641
|
}
|
|
10411
|
-
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact,
|
|
10642
|
+
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
|
|
10412
10643
|
const requiredCount = policy === "all" ? queryTokens.length : policy;
|
|
10413
10644
|
if (requiredCount <= 0) return scored;
|
|
10414
10645
|
if (requiredCount > queryTokens.length) return [];
|
|
10415
|
-
const
|
|
10646
|
+
const groups = /* @__PURE__ */ new Map();
|
|
10647
|
+
for (const group of termGroups ?? []) {
|
|
10648
|
+
const partialOnly = /* @__PURE__ */ new Set();
|
|
10649
|
+
for (const member of group.members) {
|
|
10650
|
+
if (member.factor === 0) partialOnly.add(member.token);
|
|
10651
|
+
}
|
|
10652
|
+
groups.set(group.token, {
|
|
10653
|
+
members: new Set(group.members.map((member) => member.token)),
|
|
10654
|
+
partialOnly,
|
|
10655
|
+
typoMatches: group.typoMatches
|
|
10656
|
+
});
|
|
10657
|
+
}
|
|
10416
10658
|
return scored.filter((doc) => {
|
|
10417
|
-
const matched = countDocTermMatches(
|
|
10418
|
-
doc,
|
|
10419
|
-
queryTokens,
|
|
10420
|
-
tolerance,
|
|
10421
|
-
prefixLength,
|
|
10422
|
-
exact,
|
|
10423
|
-
prefixExpansion?.token,
|
|
10424
|
-
expansionTerms
|
|
10425
|
-
);
|
|
10659
|
+
const matched = countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields);
|
|
10426
10660
|
return matched >= requiredCount;
|
|
10427
10661
|
});
|
|
10428
10662
|
}
|
|
10429
|
-
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact,
|
|
10663
|
+
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
|
|
10430
10664
|
const indexTokens = Object.keys(doc.idf);
|
|
10431
10665
|
if (indexTokens.length === 0) return 0;
|
|
10432
10666
|
let count = 0;
|
|
10433
10667
|
for (const qt of queryTokens) {
|
|
10434
|
-
|
|
10435
|
-
|
|
10436
|
-
|
|
10437
|
-
}
|
|
10438
|
-
continue;
|
|
10439
|
-
}
|
|
10440
|
-
if (queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact)) {
|
|
10441
|
-
count++;
|
|
10442
|
-
}
|
|
10668
|
+
const group = groups.get(qt.token);
|
|
10669
|
+
const satisfied = group === void 0 ? queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact) : groupSatisfied(group, doc, indexTokens, partialFields) || group.typoMatches && queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact);
|
|
10670
|
+
if (satisfied) count++;
|
|
10443
10671
|
}
|
|
10444
10672
|
return count;
|
|
10445
10673
|
}
|
|
10446
|
-
function
|
|
10674
|
+
function matchedInPartialField(doc, token, partialFields) {
|
|
10675
|
+
for (const field of partialFields) {
|
|
10676
|
+
if (doc.termFrequencies[`${field}:${token}`] !== void 0) return true;
|
|
10677
|
+
}
|
|
10678
|
+
return false;
|
|
10679
|
+
}
|
|
10680
|
+
function groupSatisfied(group, doc, indexTokens, partialFields) {
|
|
10447
10681
|
for (const indexToken of indexTokens) {
|
|
10448
|
-
if (indexToken
|
|
10449
|
-
if (
|
|
10682
|
+
if (!group.members.has(indexToken)) continue;
|
|
10683
|
+
if (!group.partialOnly.has(indexToken) || matchedInPartialField(doc, indexToken, partialFields)) return true;
|
|
10450
10684
|
}
|
|
10451
10685
|
return false;
|
|
10452
10686
|
}
|
|
@@ -10598,7 +10832,7 @@ function frozenSegmentBytes(segment) {
|
|
|
10598
10832
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
10599
10833
|
}
|
|
10600
10834
|
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
10601
|
-
return bytes;
|
|
10835
|
+
return bytes + segment.partialWords.bytes();
|
|
10602
10836
|
}
|
|
10603
10837
|
|
|
10604
10838
|
// src/core/partition/composite/compaction.ts
|
|
@@ -10733,20 +10967,19 @@ function subMatchesView(matches, layout, subIndex) {
|
|
|
10733
10967
|
|
|
10734
10968
|
// src/core/partition/composite/stats.ts
|
|
10735
10969
|
function collectCandidateTerms(subs, params) {
|
|
10736
|
-
const { queryTokens,
|
|
10970
|
+
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
10737
10971
|
const terms = /* @__PURE__ */ new Set();
|
|
10738
10972
|
for (const qt of queryTokens) {
|
|
10739
|
-
|
|
10740
|
-
|
|
10741
|
-
for (const
|
|
10742
|
-
continue;
|
|
10973
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
10974
|
+
if (group !== void 0) {
|
|
10975
|
+
for (const member of group.members) terms.add(member.token);
|
|
10743
10976
|
}
|
|
10744
|
-
if (exact || tolerance === 0) {
|
|
10977
|
+
if (group === void 0 && (exact || tolerance === 0)) {
|
|
10745
10978
|
terms.add(qt.token);
|
|
10746
10979
|
continue;
|
|
10747
10980
|
}
|
|
10748
10981
|
for (const sub of subs) {
|
|
10749
|
-
for (const match of sub.invertedIdx
|
|
10982
|
+
for (const match of typoMatchesFor(sub.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
10750
10983
|
terms.add(match.token);
|
|
10751
10984
|
}
|
|
10752
10985
|
}
|
|
@@ -10756,11 +10989,7 @@ function collectCandidateTerms(subs, params) {
|
|
|
10756
10989
|
function resolveQueryTermFrequencies(subs, params) {
|
|
10757
10990
|
const frequencies = /* @__PURE__ */ Object.create(null);
|
|
10758
10991
|
for (const term of collectCandidateTerms(subs, params)) {
|
|
10759
|
-
|
|
10760
|
-
for (const sub of subs) {
|
|
10761
|
-
const list = sub.invertedIdx.lookup(term);
|
|
10762
|
-
if (list !== void 0) total += list.docIdSet.size;
|
|
10763
|
-
}
|
|
10992
|
+
const total = partitionDocumentFrequency(subs, term);
|
|
10764
10993
|
if (total > 0) frequencies[term] = total;
|
|
10765
10994
|
}
|
|
10766
10995
|
return frequencies;
|
|
@@ -10946,9 +11175,10 @@ function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
|
|
|
10946
11175
|
|
|
10947
11176
|
// src/core/partition/composite/suggest.ts
|
|
10948
11177
|
function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
11178
|
+
if (limit <= 0) return [];
|
|
10949
11179
|
const merged = /* @__PURE__ */ new Map();
|
|
10950
11180
|
for (const sub of subs) {
|
|
10951
|
-
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix,
|
|
11181
|
+
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, Number.MAX_SAFE_INTEGER)) {
|
|
10952
11182
|
let entry = merged.get(suggestion.token);
|
|
10953
11183
|
if (entry === void 0) {
|
|
10954
11184
|
entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
|
|
@@ -10973,19 +11203,6 @@ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
|
10973
11203
|
if (suggestions.length > limit) suggestions.length = limit;
|
|
10974
11204
|
return suggestions;
|
|
10975
11205
|
}
|
|
10976
|
-
function compositeExpandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
|
|
10977
|
-
const expansions = [];
|
|
10978
|
-
const seen = /* @__PURE__ */ new Set();
|
|
10979
|
-
for (const sub of subs) {
|
|
10980
|
-
for (const term of expandTermPrefix(sub, surfacePrefix, stemmedToken, maxExpansions)) {
|
|
10981
|
-
if (seen.has(term)) continue;
|
|
10982
|
-
seen.add(term);
|
|
10983
|
-
expansions.push(term);
|
|
10984
|
-
if (expansions.length === maxExpansions) return expansions;
|
|
10985
|
-
}
|
|
10986
|
-
}
|
|
10987
|
-
return expansions;
|
|
10988
|
-
}
|
|
10989
11206
|
|
|
10990
11207
|
// src/core/partition/composite/index.ts
|
|
10991
11208
|
function isCompositePartition(partition) {
|
|
@@ -11189,7 +11406,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11189
11406
|
return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
|
|
11190
11407
|
},
|
|
11191
11408
|
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
11192
|
-
return
|
|
11409
|
+
return expandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
|
|
11410
|
+
},
|
|
11411
|
+
expandPartialWords(analysis, requests) {
|
|
11412
|
+
return expandPartialWords(subs(), analysis, requests);
|
|
11413
|
+
},
|
|
11414
|
+
preparePartialWords(analysis) {
|
|
11415
|
+
expandPartialWords(subs(), analysis, []);
|
|
11193
11416
|
},
|
|
11194
11417
|
get(docId, projection) {
|
|
11195
11418
|
const owner = frozenOwner(docId);
|
|
@@ -11250,6 +11473,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11250
11473
|
}
|
|
11251
11474
|
|
|
11252
11475
|
// src/partitioning/insert-options.ts
|
|
11476
|
+
function resolvePartialWordAnalysis(config, language, analysis) {
|
|
11477
|
+
const fields = partialFieldsOf(config.schema);
|
|
11478
|
+
if (fields.length === 0) return null;
|
|
11479
|
+
return { fields, language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
11480
|
+
}
|
|
11253
11481
|
function resolvePartitionInsertOptions(config, analysis, options) {
|
|
11254
11482
|
const applyStrict = config.strict === true;
|
|
11255
11483
|
const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
|
|
@@ -11327,8 +11555,10 @@ function setNestedValue(obj, path, value) {
|
|
|
11327
11555
|
// src/partitioning/manager/index.ts
|
|
11328
11556
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
11329
11557
|
requirePatternSearchFor(config.schema);
|
|
11558
|
+
requirePartialWordSearchFor(config.schema);
|
|
11330
11559
|
const count = initialPartitionCount ?? 1;
|
|
11331
11560
|
const analysis = resolveIndexAnalysis(config);
|
|
11561
|
+
const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
11332
11562
|
const trackPositions = config.trackPositions ?? true;
|
|
11333
11563
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
11334
11564
|
let partitions = [];
|
|
@@ -11348,6 +11578,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11348
11578
|
function resolveInsertOptions(options) {
|
|
11349
11579
|
return resolvePartitionInsertOptions(config, analysis, options);
|
|
11350
11580
|
}
|
|
11581
|
+
function preparePartialWords(partitionId) {
|
|
11582
|
+
if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
|
|
11583
|
+
}
|
|
11351
11584
|
function locateDocument(docId) {
|
|
11352
11585
|
return docPartitionMap.get(docId);
|
|
11353
11586
|
}
|
|
@@ -11403,6 +11636,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11403
11636
|
get analysis() {
|
|
11404
11637
|
return analysis;
|
|
11405
11638
|
},
|
|
11639
|
+
get partialWordAnalysis() {
|
|
11640
|
+
return partialWordAnalysis;
|
|
11641
|
+
},
|
|
11406
11642
|
getPartition(partitionId) {
|
|
11407
11643
|
validatePartitionId(partitionId);
|
|
11408
11644
|
return partitions[partitionId];
|
|
@@ -11479,6 +11715,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11479
11715
|
rebuildTextIndex(partitionId) {
|
|
11480
11716
|
validatePartitionId(partitionId);
|
|
11481
11717
|
partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
|
|
11718
|
+
preparePartialWords(partitionId);
|
|
11482
11719
|
},
|
|
11483
11720
|
beginBatchRemove() {
|
|
11484
11721
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -11580,6 +11817,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11580
11817
|
deserializePartition(partitionId, data) {
|
|
11581
11818
|
validatePartitionId(partitionId);
|
|
11582
11819
|
partitions[partitionId].deserialize(data, config.schema);
|
|
11820
|
+
preparePartialWords(partitionId);
|
|
11583
11821
|
rebuildDocPartitionMapForPartition(partitionId);
|
|
11584
11822
|
},
|
|
11585
11823
|
getAggregateStats() {
|
|
@@ -18050,6 +18288,15 @@ function decodeStoredDocument(bytes, discardedFields) {
|
|
|
18050
18288
|
}
|
|
18051
18289
|
return document2;
|
|
18052
18290
|
}
|
|
18291
|
+
function* storedDocumentsOf(data) {
|
|
18292
|
+
const count = data.offsets.length - 1;
|
|
18293
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
18294
|
+
yield decodeStoredDocument(
|
|
18295
|
+
data.blob.subarray(data.offsets[ordinal], data.offsets[ordinal + 1]),
|
|
18296
|
+
data.discardedFields
|
|
18297
|
+
);
|
|
18298
|
+
}
|
|
18299
|
+
}
|
|
18053
18300
|
function decodeEveryStoredDocument(data) {
|
|
18054
18301
|
const count = data.offsets.length - 1;
|
|
18055
18302
|
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
@@ -18093,6 +18340,9 @@ function wrapEncodedDocumentTable(data) {
|
|
|
18093
18340
|
decodedBytes += documentValueBytes(document2);
|
|
18094
18341
|
}
|
|
18095
18342
|
return document2;
|
|
18343
|
+
},
|
|
18344
|
+
readAt(ordinal) {
|
|
18345
|
+
return decoded[ordinal] ?? decodeStoredDocument(blob.subarray(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
|
|
18096
18346
|
}
|
|
18097
18347
|
};
|
|
18098
18348
|
}
|
|
@@ -18106,6 +18356,9 @@ function wrapDocumentArray(documents) {
|
|
|
18106
18356
|
byteLength: residentBytes,
|
|
18107
18357
|
docAt(ordinal) {
|
|
18108
18358
|
return documents[ordinal];
|
|
18359
|
+
},
|
|
18360
|
+
readAt(ordinal) {
|
|
18361
|
+
return documents[ordinal];
|
|
18109
18362
|
}
|
|
18110
18363
|
};
|
|
18111
18364
|
}
|
|
@@ -18492,6 +18745,10 @@ function createFrozenInvertedReader(table, views) {
|
|
|
18492
18745
|
if (at < 0) return void 0;
|
|
18493
18746
|
return views.viewReadOnce(table.payloadSlot(at), table.documentFrequencyAt(at));
|
|
18494
18747
|
},
|
|
18748
|
+
documentFrequency(token) {
|
|
18749
|
+
const at = table.find(token);
|
|
18750
|
+
return at < 0 ? 0 : table.documentFrequencyAt(at);
|
|
18751
|
+
},
|
|
18495
18752
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
18496
18753
|
if (tolerance === 0) {
|
|
18497
18754
|
const at = table.find(token);
|
|
@@ -18814,6 +19071,35 @@ function sharedSurfaceTable(payload) {
|
|
|
18814
19071
|
if (table.counts.length === 0) return null;
|
|
18815
19072
|
return { blob: sharedUint8(table.blob), offsets: sharedUint32(table.offsets), counts: sharedUint32(table.counts) };
|
|
18816
19073
|
}
|
|
19074
|
+
function sharePartialWordTable(table) {
|
|
19075
|
+
if (typeof SharedArrayBuffer !== "function") return table;
|
|
19076
|
+
return {
|
|
19077
|
+
fields: [...table.fields],
|
|
19078
|
+
spellings: sharedUint8(table.spellings),
|
|
19079
|
+
spellingOffsets: sharedUint32(table.spellingOffsets),
|
|
19080
|
+
spellingTokens: sharedUint32(table.spellingTokens),
|
|
19081
|
+
spellingCounts: sharedUint32(table.spellingCounts),
|
|
19082
|
+
runs: sharedUint32(table.runs),
|
|
19083
|
+
runOffsets: sharedUint32(table.runOffsets),
|
|
19084
|
+
runEntries: sharedUint32(table.runEntries)
|
|
19085
|
+
};
|
|
19086
|
+
}
|
|
19087
|
+
function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
|
|
19088
|
+
let source = spellings;
|
|
19089
|
+
if (source === null && analysis !== null && analysis.fields.length > 0) {
|
|
19090
|
+
source = encodePartialSpellings(analysis.fields, spellingCountsOf(storedDocumentsOf(documentData), analysis));
|
|
19091
|
+
}
|
|
19092
|
+
if (source === null || source.fields.length === 0) return null;
|
|
19093
|
+
const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
|
|
19094
|
+
return table === null ? null : sharePartialWordTable(table);
|
|
19095
|
+
}
|
|
19096
|
+
function shareSegmentArrays(arrays, analysis, segmentId) {
|
|
19097
|
+
return {
|
|
19098
|
+
...arrays,
|
|
19099
|
+
segmentId: segmentId ?? generateId(),
|
|
19100
|
+
partialWords: sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis)
|
|
19101
|
+
};
|
|
19102
|
+
}
|
|
18817
19103
|
function segmentArraysOf(payload, documentData) {
|
|
18818
19104
|
const tokenTable = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
18819
19105
|
const idTable = encodeExternalIdTableData(payload.docIds);
|
|
@@ -18842,11 +19128,11 @@ function segmentArraysOf(payload, documentData) {
|
|
|
18842
19128
|
surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable
|
|
18843
19129
|
};
|
|
18844
19130
|
}
|
|
18845
|
-
function freezeSegmentShared(payload, documents, segmentId) {
|
|
19131
|
+
function freezeSegmentShared(payload, documents, segmentId, analysis = null) {
|
|
18846
19132
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
18847
|
-
return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId);
|
|
19133
|
+
return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId, analysis);
|
|
18848
19134
|
}
|
|
18849
|
-
function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
19135
|
+
function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null) {
|
|
18850
19136
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
18851
19137
|
const tokenData = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
18852
19138
|
const idData = encodeExternalIdTableData(payload.docIds);
|
|
@@ -18913,7 +19199,8 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
|
18913
19199
|
offsets: sharedUint32(entry.offsets),
|
|
18914
19200
|
postings: sharedUint32(entry.postings)
|
|
18915
19201
|
})),
|
|
18916
|
-
surfaceTable: sharedSurfaceTable(payload)
|
|
19202
|
+
surfaceTable: sharedSurfaceTable(payload),
|
|
19203
|
+
partialWords: sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis)
|
|
18917
19204
|
};
|
|
18918
19205
|
}
|
|
18919
19206
|
|
|
@@ -18949,7 +19236,12 @@ function buildGeoReaders(entries) {
|
|
|
18949
19236
|
}
|
|
18950
19237
|
return readers;
|
|
18951
19238
|
}
|
|
18952
|
-
function
|
|
19239
|
+
function* survivingDocuments(documentSource, tombstones) {
|
|
19240
|
+
for (let ordinal = 0; ordinal < documentSource.count; ordinal++) {
|
|
19241
|
+
if (!tombstones.has(ordinal)) yield documentSource.readAt(ordinal);
|
|
19242
|
+
}
|
|
19243
|
+
}
|
|
19244
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
|
|
18953
19245
|
const tombstones = createFrozenTombstones();
|
|
18954
19246
|
const postingViews = createFrozenPostingViews(source, tombstones);
|
|
18955
19247
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
@@ -18957,6 +19249,10 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
18957
19249
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
18958
19250
|
}
|
|
18959
19251
|
const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
|
|
19252
|
+
const partialWords = createFrozenPartialWords(partialWordTable, source.tokenTable, {
|
|
19253
|
+
documentAt: (ordinal) => documentSource.readAt(ordinal),
|
|
19254
|
+
survivors: () => survivingDocuments(documentSource, tombstones)
|
|
19255
|
+
});
|
|
18960
19256
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
18961
19257
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
18962
19258
|
const sizeIndexes = /* @__PURE__ */ new Map();
|
|
@@ -18989,6 +19285,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
18989
19285
|
patternIndexes,
|
|
18990
19286
|
sizeIndexes,
|
|
18991
19287
|
fieldNameTable,
|
|
19288
|
+
partialWords,
|
|
18992
19289
|
trackPositions: source.positionOffsets !== null,
|
|
18993
19290
|
flatSchemaCache: null,
|
|
18994
19291
|
lastSchemaRef: null,
|
|
@@ -19008,6 +19305,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
19008
19305
|
if (ordinal === void 0) return false;
|
|
19009
19306
|
const added = tombstones.add(ordinal);
|
|
19010
19307
|
if (added) {
|
|
19308
|
+
partialWords.afterRemoval(ordinal);
|
|
19011
19309
|
docStore.releaseSortedDocIds();
|
|
19012
19310
|
segment.sortColumns = null;
|
|
19013
19311
|
segment.scoreBuffer = null;
|
|
@@ -19023,18 +19321,20 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
19023
19321
|
return segment;
|
|
19024
19322
|
}
|
|
19025
19323
|
function createFrozenSegment(payload, documents, segmentId) {
|
|
19324
|
+
const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
|
|
19026
19325
|
return assembleFrozenSegment(
|
|
19027
19326
|
segmentId ?? generateId(),
|
|
19028
19327
|
{
|
|
19029
19328
|
...payload,
|
|
19030
|
-
tokenTable
|
|
19329
|
+
tokenTable,
|
|
19031
19330
|
idTable: buildExternalIdTable(payload.docIds),
|
|
19032
19331
|
documentTable: null,
|
|
19033
19332
|
readSurfaceForms: () => payload.surfaceForms,
|
|
19034
19333
|
docFrequencies: () => payload.docFrequencies
|
|
19035
19334
|
},
|
|
19036
19335
|
wrapDocumentArray(documents),
|
|
19037
|
-
null
|
|
19336
|
+
null,
|
|
19337
|
+
payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, tokenTable)
|
|
19038
19338
|
);
|
|
19039
19339
|
}
|
|
19040
19340
|
function createSharedFrozenSegment(snapshot) {
|
|
@@ -19056,7 +19356,8 @@ function createSharedFrozenSegment(snapshot) {
|
|
|
19056
19356
|
}
|
|
19057
19357
|
},
|
|
19058
19358
|
wrapEncodedDocumentTable(snapshot.documentTable),
|
|
19059
|
-
snapshot
|
|
19359
|
+
snapshot,
|
|
19360
|
+
snapshot.partialWords
|
|
19060
19361
|
);
|
|
19061
19362
|
}
|
|
19062
19363
|
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
@@ -19638,10 +19939,10 @@ function mergeDocuments2(inputs, documentCount) {
|
|
|
19638
19939
|
}
|
|
19639
19940
|
return { blob, offsets };
|
|
19640
19941
|
}
|
|
19641
|
-
function mergeFrozenSegments(segments) {
|
|
19942
|
+
function mergeFrozenSegments(segments, analysis) {
|
|
19642
19943
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
19643
19944
|
const merged = mergeSegmentContents(segments.map((segment) => ({ segment })));
|
|
19644
|
-
return freezeEncodedSegmentShared(merged.payload, merged.documents);
|
|
19945
|
+
return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis);
|
|
19645
19946
|
}
|
|
19646
19947
|
function mergeSegmentContents(selections) {
|
|
19647
19948
|
const docIds = [];
|
|
@@ -19670,6 +19971,7 @@ function mergeSegmentContents(selections) {
|
|
|
19670
19971
|
totalFieldLengths: lengths.totalFieldLengths,
|
|
19671
19972
|
docFrequencies: postings.docFrequencies,
|
|
19672
19973
|
surfaceForms: mergeSurfaceForms2(inputs),
|
|
19974
|
+
partialSpellings: null,
|
|
19673
19975
|
...mergeFieldIndexes2(inputs)
|
|
19674
19976
|
};
|
|
19675
19977
|
return { payload, documents: mergeDocuments2(inputs, documentCount) };
|
|
@@ -19970,14 +20272,18 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
19970
20272
|
patternIndexes: /* @__PURE__ */ new Map(),
|
|
19971
20273
|
sizeIndexes: /* @__PURE__ */ new Map()
|
|
19972
20274
|
};
|
|
20275
|
+
const partialFields = partialFieldsOf(schema);
|
|
20276
|
+
const spellings = /* @__PURE__ */ new Map();
|
|
20277
|
+
let fieldSpellings = null;
|
|
19973
20278
|
let positionOffset = 0;
|
|
19974
20279
|
const addToken = (token, position, surface) => {
|
|
19975
20280
|
postings.addToken(token, positionOffset + position);
|
|
19976
20281
|
surfaces.count(surface, token);
|
|
20282
|
+
if (fieldSpellings !== null) countSpelling(fieldSpellings, token, surface ?? token, 1);
|
|
19977
20283
|
};
|
|
19978
20284
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
19979
20285
|
const document2 = documents[ordinal].document;
|
|
19980
|
-
for (const { path: fieldPath, base: fieldType, pattern } of fields) {
|
|
20286
|
+
for (const { path: fieldPath, base: fieldType, pattern, partial } of fields) {
|
|
19981
20287
|
const value = getNestedValue(document2, fieldPath);
|
|
19982
20288
|
if (value === void 0 || value === null) continue;
|
|
19983
20289
|
recordValueSize(filters.sizeIndexes, ordinal, fieldPath, fieldType, value);
|
|
@@ -19986,6 +20292,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
19986
20292
|
indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
|
|
19987
20293
|
continue;
|
|
19988
20294
|
}
|
|
20295
|
+
fieldSpellings = partial ? spellings : null;
|
|
19989
20296
|
postings.beginField();
|
|
19990
20297
|
positionOffset = 0;
|
|
19991
20298
|
for (const text of fieldType === "string" ? [value] : value) {
|
|
@@ -20006,6 +20313,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
20006
20313
|
fieldLengthColumns: lengths.columns,
|
|
20007
20314
|
totalFieldLengths: lengths.totalsByName(),
|
|
20008
20315
|
surfaceForms: surfaces.encode(),
|
|
20316
|
+
partialSpellings: partialFields.length === 0 ? null : encodePartialSpellings(partialFields, spellings),
|
|
20009
20317
|
...encodeFieldIndexes(filters, identity)
|
|
20010
20318
|
};
|
|
20011
20319
|
}
|
|
@@ -20072,7 +20380,7 @@ function runSegmentAction(entry, action) {
|
|
|
20072
20380
|
}
|
|
20073
20381
|
case "compactSegments": {
|
|
20074
20382
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
20075
|
-
return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds));
|
|
20383
|
+
return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds), entry.manager.partialWordAnalysis);
|
|
20076
20384
|
}
|
|
20077
20385
|
case "swapSegments": {
|
|
20078
20386
|
requireComposite(entry, action.indexName, action.partitionId, "swap");
|
|
@@ -23465,7 +23773,7 @@ function selectionOfMask(segment, keep) {
|
|
|
23465
23773
|
}
|
|
23466
23774
|
function frozenLiveTail(live, shared2) {
|
|
23467
23775
|
if (live.count() === 0) return null;
|
|
23468
|
-
const payload = live.encodeSegment();
|
|
23776
|
+
const payload = { ...live.encodeSegment(), partialSpellings: null };
|
|
23469
23777
|
const documents = survivorDocumentsOf(partitionStateOf(live), payload);
|
|
23470
23778
|
if (shared2) {
|
|
23471
23779
|
const snapshot = freezeSegmentShared(payload, documents);
|
|
@@ -24981,6 +25289,7 @@ function emptyPayload() {
|
|
|
24981
25289
|
totalFieldLengths: {},
|
|
24982
25290
|
docFrequencies: {},
|
|
24983
25291
|
surfaceForms: null,
|
|
25292
|
+
partialSpellings: null,
|
|
24984
25293
|
numeric: [],
|
|
24985
25294
|
boolean: [],
|
|
24986
25295
|
enums: [],
|
|
@@ -25086,7 +25395,7 @@ function chooseWindow(segments, wantedLength) {
|
|
|
25086
25395
|
return null;
|
|
25087
25396
|
}
|
|
25088
25397
|
function frozenOf(contents) {
|
|
25089
|
-
return createSharedFrozenSegment(
|
|
25398
|
+
return createSharedFrozenSegment(shareSegmentArrays(contents.arrays, null));
|
|
25090
25399
|
}
|
|
25091
25400
|
function applyTimeOrder(frozen, contents) {
|
|
25092
25401
|
for (let newer = 1; newer < frozen.length; newer++) {
|
|
@@ -25354,7 +25663,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
25354
25663
|
const contents = await readSegmentFile(store, ref);
|
|
25355
25664
|
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
25356
25665
|
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
25357
|
-
const segment = createSharedFrozenSegment(
|
|
25666
|
+
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
25358
25667
|
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
25359
25668
|
if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
|
|
25360
25669
|
await yieldToEventLoop6();
|
|
@@ -25787,7 +26096,7 @@ function validateHnswMetric2(value) {
|
|
|
25787
26096
|
}
|
|
25788
26097
|
return void 0;
|
|
25789
26098
|
}
|
|
25790
|
-
function
|
|
26099
|
+
function storedDocumentsOf2(raw) {
|
|
25791
26100
|
requireValidFieldTypes(raw.schema ?? {});
|
|
25792
26101
|
const documents = {};
|
|
25793
26102
|
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
@@ -25801,11 +26110,11 @@ function storedDocumentsOf(raw) {
|
|
|
25801
26110
|
}
|
|
25802
26111
|
function documentsOfPayloadV2(data) {
|
|
25803
26112
|
const raw = decode(data);
|
|
25804
|
-
return { documents:
|
|
26113
|
+
return { documents: storedDocumentsOf2(raw), schema: raw.schema ?? {} };
|
|
25805
26114
|
}
|
|
25806
26115
|
function deserializePayloadV2(data) {
|
|
25807
26116
|
const raw = decode(data);
|
|
25808
|
-
const documents =
|
|
26117
|
+
const documents = storedDocumentsOf2(raw);
|
|
25809
26118
|
const fieldNames = raw.inverted_index?.field_names ?? [];
|
|
25810
26119
|
const invertedIndex = {};
|
|
25811
26120
|
for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
|
|
@@ -27873,6 +28182,16 @@ function validatePartitionConfig(partitions) {
|
|
|
27873
28182
|
});
|
|
27874
28183
|
}
|
|
27875
28184
|
}
|
|
28185
|
+
function requireSurfaceFormsForPartialFields(schema, surfaceForms) {
|
|
28186
|
+
if (surfaceForms !== false) return;
|
|
28187
|
+
const partialFields = partialFieldsOf(schema);
|
|
28188
|
+
if (partialFields.length === 0) return;
|
|
28189
|
+
throw new NarsilError(
|
|
28190
|
+
ErrorCodes.CONFIG_INVALID,
|
|
28191
|
+
`Field "${partialFields[0]}" declares "partial", which compares words as the documents wrote them, so its index needs surfaceForms left on`,
|
|
28192
|
+
{ fields: [...partialFields], surfaceForms: false }
|
|
28193
|
+
);
|
|
28194
|
+
}
|
|
27876
28195
|
function validatePatternValueLimit(limit) {
|
|
27877
28196
|
if (limit === void 0) return;
|
|
27878
28197
|
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < MIN_PATTERN_VALUE_LIMIT || limit > MAX_PATTERN_VALUE_LIMIT) {
|
|
@@ -27987,6 +28306,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
27987
28306
|
try {
|
|
27988
28307
|
indexConfig.schema = validateSchema(indexConfig.schema);
|
|
27989
28308
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
28309
|
+
requireSurfaceFormsForPartialFields(indexConfig.schema, indexConfig.surfaceForms);
|
|
27990
28310
|
} catch (err) {
|
|
27991
28311
|
throw recoveryFailure(metadata, err);
|
|
27992
28312
|
}
|
|
@@ -28700,7 +29020,7 @@ function checkHeapAfterRecovery(core) {
|
|
|
28700
29020
|
}
|
|
28701
29021
|
|
|
28702
29022
|
// src/core/partition/frozen/share.ts
|
|
28703
|
-
function shareFrozenSegment(segment) {
|
|
29023
|
+
function shareFrozenSegment(segment, analysis) {
|
|
28704
29024
|
if (segment.sharedSnapshot !== null) {
|
|
28705
29025
|
return { snapshot: segment.sharedSnapshot, tombstonedDocIds: segment.tombstonedDocIds() };
|
|
28706
29026
|
}
|
|
@@ -28714,7 +29034,7 @@ function shareFrozenSegment(segment) {
|
|
|
28714
29034
|
}
|
|
28715
29035
|
return stored.fields;
|
|
28716
29036
|
});
|
|
28717
|
-
const snapshot = freezeSegmentShared(payload, documents, segment.segmentId);
|
|
29037
|
+
const snapshot = freezeSegmentShared(payload, documents, segment.segmentId, analysis);
|
|
28718
29038
|
return snapshot === null ? null : { snapshot, tombstonedDocIds: [] };
|
|
28719
29039
|
}
|
|
28720
29040
|
|
|
@@ -28725,7 +29045,7 @@ function captureComposite(manager, partitionId) {
|
|
|
28725
29045
|
const segmentIds = partition.frozenSegmentSizes().map((size) => size.segmentId);
|
|
28726
29046
|
const frozen = [];
|
|
28727
29047
|
for (const segment of partition.frozenSegmentsById(segmentIds)) {
|
|
28728
|
-
const shared2 = shareFrozenSegment(segment);
|
|
29048
|
+
const shared2 = shareFrozenSegment(segment, manager.partialWordAnalysis);
|
|
28729
29049
|
if (shared2 === null) return null;
|
|
28730
29050
|
frozen.push(shared2);
|
|
28731
29051
|
}
|
|
@@ -29130,7 +29450,7 @@ function liveTailCount(manager, partitionId) {
|
|
|
29130
29450
|
}
|
|
29131
29451
|
function freezeIntoSharedSegment(manager, partitionId) {
|
|
29132
29452
|
return manager.freezeLiveTail(partitionId, (payload, documents) => {
|
|
29133
|
-
const snapshot = freezeSegmentShared(payload, documents);
|
|
29453
|
+
const snapshot = freezeSegmentShared(payload, documents, void 0, manager.partialWordAnalysis);
|
|
29134
29454
|
return snapshot === null ? null : createSharedFrozenSegment(snapshot);
|
|
29135
29455
|
});
|
|
29136
29456
|
}
|
|
@@ -29253,7 +29573,7 @@ async function compactPartitionSegments2(state, indexName, manager, partitionId,
|
|
|
29253
29573
|
const mainCanMerge = composite !== null && mainHoldsAll(composite, segmentIds);
|
|
29254
29574
|
let snapshot = await compactOnWorker(state, indexName, partitionId, segmentIds);
|
|
29255
29575
|
if (snapshot === null && mainCanMerge && composite !== null) {
|
|
29256
|
-
snapshot = mergeFrozenSegments(composite.frozenSegmentsById(segmentIds));
|
|
29576
|
+
snapshot = mergeFrozenSegments(composite.frozenSegmentsById(segmentIds), manager.partialWordAnalysis);
|
|
29257
29577
|
}
|
|
29258
29578
|
if (snapshot === null) {
|
|
29259
29579
|
if (!mainCanMerge) forgetLedgerSegments(state, indexName, partitionId, segmentIds);
|
|
@@ -31226,7 +31546,9 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
31226
31546
|
if (appliesCreationLimits) requireCreatableSchema(schema);
|
|
31227
31547
|
const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
31228
31548
|
requirePatternSearchFor(schema);
|
|
31549
|
+
requirePartialWordSearchFor(schema);
|
|
31229
31550
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
31551
|
+
requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
|
|
31230
31552
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
31231
31553
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
31232
31554
|
validateBM25Params(indexConfig.bm25);
|
|
@@ -31256,10 +31578,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
31256
31578
|
const fields = indexConfig.embedding.fields;
|
|
31257
31579
|
if (typeof configuredAdapter === "string") {
|
|
31258
31580
|
embeddingAdapterName = configuredAdapter;
|
|
31259
|
-
const
|
|
31260
|
-
if (
|
|
31581
|
+
const registered3 = core.embeddingAdapters.get(configuredAdapter);
|
|
31582
|
+
if (registered3 !== void 0) {
|
|
31261
31583
|
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
31262
|
-
{ fields, adapter:
|
|
31584
|
+
{ fields, adapter: registered3 },
|
|
31263
31585
|
indexConfig.schema,
|
|
31264
31586
|
config?.embedding
|
|
31265
31587
|
);
|
|
@@ -34552,6 +34874,7 @@ function prepareRestoredIndex(configuration, deps, reject2) {
|
|
|
34552
34874
|
const language = getLanguage(languageName);
|
|
34553
34875
|
const schema = validateSchema(configuration.schema);
|
|
34554
34876
|
validatePatternValueLimit(configFields.patternValueLimit);
|
|
34877
|
+
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
34555
34878
|
validateVectorPromotion(configFields.vectorPromotion);
|
|
34556
34879
|
validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
|
|
34557
34880
|
if (configFields.required !== void 0 && configFields.required.length > 0) {
|
|
@@ -35109,7 +35432,7 @@ function boundVectorOf(vector) {
|
|
|
35109
35432
|
return bound;
|
|
35110
35433
|
}
|
|
35111
35434
|
var FILTERS_SLOT = 2;
|
|
35112
|
-
var BOUND_SLOT_COUNT =
|
|
35435
|
+
var BOUND_SLOT_COUNT = 16;
|
|
35113
35436
|
function bindingOf(values) {
|
|
35114
35437
|
const stream = { bytes: new Uint8Array(256), length: 0 };
|
|
35115
35438
|
for (const value of values) {
|
|
@@ -35128,6 +35451,7 @@ function queryBindingOf(params) {
|
|
|
35128
35451
|
params.tolerance,
|
|
35129
35452
|
params.prefixLength,
|
|
35130
35453
|
params.prefix,
|
|
35454
|
+
params.partialWords,
|
|
35131
35455
|
params.exact,
|
|
35132
35456
|
params.pinned,
|
|
35133
35457
|
params.mode,
|
|
@@ -35296,9 +35620,18 @@ function requireKnownMode(params) {
|
|
|
35296
35620
|
{ mode: String(params.mode), modes: [...SEARCH_MODES] }
|
|
35297
35621
|
);
|
|
35298
35622
|
}
|
|
35623
|
+
function requireBooleanPartialWords(params) {
|
|
35624
|
+
const setting = params.partialWords;
|
|
35625
|
+
if (setting === void 0 || typeof setting === "boolean") return;
|
|
35626
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The "partialWords" query option takes true or false', {
|
|
35627
|
+
option: "partialWords",
|
|
35628
|
+
received: setting === null ? "null" : typeof setting
|
|
35629
|
+
});
|
|
35630
|
+
}
|
|
35299
35631
|
function requireValidQueryOptions(params, config) {
|
|
35300
35632
|
const strict = config.strict === true;
|
|
35301
35633
|
requireValidTypoTolerance(params);
|
|
35634
|
+
requireBooleanPartialWords(params);
|
|
35302
35635
|
requireSortableFields(params.sort, config.schema, strict);
|
|
35303
35636
|
requireValidQueryFilter(params.filters, config.schema);
|
|
35304
35637
|
requireUsableQueryFields(params, config.schema, strict);
|
|
@@ -35650,6 +35983,31 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
|
|
|
35650
35983
|
if (needsTrailingEllipsis) snippet = `${snippet}...`;
|
|
35651
35984
|
return { snippet, visiblePositions };
|
|
35652
35985
|
}
|
|
35986
|
+
function containsPartialWord(spelling, words) {
|
|
35987
|
+
for (const word of words) {
|
|
35988
|
+
if (spelling.length > word.length && spelling.includes(word)) return true;
|
|
35989
|
+
}
|
|
35990
|
+
return false;
|
|
35991
|
+
}
|
|
35992
|
+
function partialWordMatcher(text, language, partial) {
|
|
35993
|
+
const indexed = tokenize(text, language, {
|
|
35994
|
+
stem: true,
|
|
35995
|
+
removeStopWords: true,
|
|
35996
|
+
collectSurfaces: true,
|
|
35997
|
+
customTokenizer: partial.customTokenizer,
|
|
35998
|
+
stopWordOverride: partial.stopWordOverride
|
|
35999
|
+
});
|
|
36000
|
+
const positions = /* @__PURE__ */ new Set();
|
|
36001
|
+
const spellings = /* @__PURE__ */ new Set();
|
|
36002
|
+
for (let at = 0; at < indexed.tokens.length; at++) {
|
|
36003
|
+
const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
|
|
36004
|
+
if (!containsPartialWord(spelling, partial.words)) continue;
|
|
36005
|
+
positions.add(indexed.tokens[at].position);
|
|
36006
|
+
spellings.add(spelling);
|
|
36007
|
+
}
|
|
36008
|
+
if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
|
|
36009
|
+
return (position) => positions.has(position);
|
|
36010
|
+
}
|
|
35653
36011
|
function highlightField(text, queryTokens, language, options) {
|
|
35654
36012
|
const preTag = options?.preTag ?? "<mark>";
|
|
35655
36013
|
const postTag = options?.postTag ?? "</mark>";
|
|
@@ -35686,11 +36044,13 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
35686
36044
|
matchByFieldToken.set(fieldToken, matches);
|
|
35687
36045
|
return matches;
|
|
35688
36046
|
};
|
|
36047
|
+
const matchesInside = options?.partialWords === void 0 ? null : partialWordMatcher(text, language, options.partialWords);
|
|
35689
36048
|
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
35690
36049
|
if (i >= charOffsets.length) break;
|
|
35691
36050
|
const fieldToken = fieldResult.tokens[i].token;
|
|
35692
36051
|
const prefixMatch = prefixToken !== void 0 && fieldToken.startsWith(prefixToken);
|
|
35693
|
-
|
|
36052
|
+
const insideMatch = matchesInside?.(fieldResult.tokens[i].position, fieldToken) === true;
|
|
36053
|
+
if (prefixMatch || insideMatch || matchesQuery(fieldToken)) {
|
|
35694
36054
|
matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
|
|
35695
36055
|
}
|
|
35696
36056
|
}
|
|
@@ -35722,7 +36082,39 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
35722
36082
|
}
|
|
35723
36083
|
|
|
35724
36084
|
// src/engine/query/highlight.ts
|
|
35725
|
-
function
|
|
36085
|
+
function writtenWordsOf(params, language, analysis) {
|
|
36086
|
+
return tokenize(params.term ?? "", language, {
|
|
36087
|
+
stem: false,
|
|
36088
|
+
removeStopWords: true,
|
|
36089
|
+
customTokenizer: analysis.customTokenizer,
|
|
36090
|
+
stopWordOverride: analysis.stopWords
|
|
36091
|
+
}).tokens;
|
|
36092
|
+
}
|
|
36093
|
+
function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
36094
|
+
if (params.partialWords !== true || params.exact === true) return null;
|
|
36095
|
+
const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
|
|
36096
|
+
if (fields.length === 0) return null;
|
|
36097
|
+
const writtenAt = /* @__PURE__ */ new Map();
|
|
36098
|
+
for (const word of written) {
|
|
36099
|
+
if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
|
|
36100
|
+
}
|
|
36101
|
+
const seenTokens = /* @__PURE__ */ new Set();
|
|
36102
|
+
const words = [];
|
|
36103
|
+
for (const queryToken of queryTokens) {
|
|
36104
|
+
if (seenTokens.has(queryToken.token)) continue;
|
|
36105
|
+
seenTokens.add(queryToken.token);
|
|
36106
|
+
const word = writtenAt.get(queryToken.position);
|
|
36107
|
+
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
36108
|
+
words.push(word);
|
|
36109
|
+
if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
36110
|
+
}
|
|
36111
|
+
if (words.length === 0) return null;
|
|
36112
|
+
return {
|
|
36113
|
+
fields: new Set(fields),
|
|
36114
|
+
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
36115
|
+
};
|
|
36116
|
+
}
|
|
36117
|
+
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument) {
|
|
35726
36118
|
if (!params.highlight) return;
|
|
35727
36119
|
const queryTokenResult = tokenize(params.term ?? "", language, {
|
|
35728
36120
|
stem: true,
|
|
@@ -35730,17 +36122,15 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
35730
36122
|
customTokenizer: analysis.customTokenizer,
|
|
35731
36123
|
stopWordOverride: analysis.stopWords
|
|
35732
36124
|
});
|
|
36125
|
+
const queryTokens = queryTokenResult.tokens;
|
|
36126
|
+
const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
|
|
36127
|
+
const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
|
|
36128
|
+
const written = usesPrefix || usesPartialWords ? writtenWordsOf(params, language, analysis) : [];
|
|
35733
36129
|
let prefixToken;
|
|
35734
36130
|
let completedToken;
|
|
35735
|
-
if (
|
|
35736
|
-
const lastToken =
|
|
35737
|
-
|
|
35738
|
-
stem: false,
|
|
35739
|
-
removeStopWords: true,
|
|
35740
|
-
customTokenizer: analysis.customTokenizer,
|
|
35741
|
-
stopWordOverride: analysis.stopWords
|
|
35742
|
-
});
|
|
35743
|
-
for (const t of unstemmed.tokens) {
|
|
36131
|
+
if (usesPrefix) {
|
|
36132
|
+
const lastToken = queryTokens[queryTokens.length - 1];
|
|
36133
|
+
for (const t of written) {
|
|
35744
36134
|
if (t.position === lastToken.position) {
|
|
35745
36135
|
prefixToken = t.token;
|
|
35746
36136
|
completedToken = lastToken.token;
|
|
@@ -35748,6 +36138,7 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
35748
36138
|
}
|
|
35749
36139
|
}
|
|
35750
36140
|
}
|
|
36141
|
+
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
35751
36142
|
for (const hit of hits) {
|
|
35752
36143
|
const highlights = {};
|
|
35753
36144
|
const projected = hit.document;
|
|
@@ -35759,14 +36150,15 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
35759
36150
|
fieldValue = stored?.[field];
|
|
35760
36151
|
}
|
|
35761
36152
|
if (typeof fieldValue === "string") {
|
|
35762
|
-
highlights[field] = highlightField(fieldValue,
|
|
36153
|
+
highlights[field] = highlightField(fieldValue, queryTokens, language, {
|
|
35763
36154
|
preTag: params.highlight.preTag,
|
|
35764
36155
|
postTag: params.highlight.postTag,
|
|
35765
36156
|
maxSnippetLength: params.highlight.maxSnippetLength,
|
|
35766
36157
|
prefixToken,
|
|
35767
36158
|
completedToken,
|
|
35768
36159
|
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
35769
|
-
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
36160
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
36161
|
+
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0
|
|
35770
36162
|
});
|
|
35771
36163
|
}
|
|
35772
36164
|
}
|
|
@@ -36276,7 +36668,7 @@ async function executeQuery(params, callerContext) {
|
|
|
36276
36668
|
if (params.highlight) {
|
|
36277
36669
|
const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
|
|
36278
36670
|
const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
|
|
36279
|
-
applyHighlights(paginated, params, language, manager.analysis, readStoredDocument);
|
|
36671
|
+
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument);
|
|
36280
36672
|
}
|
|
36281
36673
|
const elapsed = now() - startTime;
|
|
36282
36674
|
return {
|
|
@@ -36868,4 +37260,4 @@ async function createNarsil(config) {
|
|
|
36868
37260
|
// src/index.ts
|
|
36869
37261
|
var VERSION = "0.1.0";
|
|
36870
37262
|
|
|
36871
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerPatternSearch, registerStopWords, registerTokenizer, resetNormalizationCache };
|
|
37263
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerTokenizer, resetNormalizationCache };
|