@delali/narsil 0.3.1-next.77 → 0.3.1-next.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-HH6ALOUZ.mjs → chunk-55WEJKWT.mjs} +117 -17
- package/dist/{chunk-DFFQOA7S.mjs → chunk-DFW3U5UW.mjs} +134 -159
- package/dist/{chunk-V3B24XR7.mjs → chunk-IOZPBNSG.mjs} +2 -1
- package/dist/{chunk-D4EF7J3I.mjs → chunk-P2BEMMOD.mjs} +193 -58
- package/dist/{chunk-PXDY5VRT.mjs → chunk-PJWG547Z.mjs} +2 -2
- package/dist/{chunk-356UOTK2.mjs → chunk-PXDQFYD5.mjs} +7 -1
- package/dist/{chunk-G7IP3UDZ.mjs → chunk-R6WBLBTC.mjs} +45 -26
- package/dist/{chunk-PJU6EF22.mjs → chunk-RTR44V2A.mjs} +267 -108
- package/dist/{chunk-NO3ABFLE.mjs → chunk-WFVOBZS4.mjs} +2 -49
- package/dist/{chunk-BT62MUM5.mjs → chunk-XSZTF4ZQ.mjs} +1 -1
- package/dist/{chunk-BT6BI4GV.mjs → chunk-Z3BXCZXS.mjs} +1 -1
- package/dist/{direct-executor-Q7PNCGZB.mjs → direct-executor-WB2HKU4N.mjs} +6 -6
- package/dist/distribution.mjs +10 -10
- package/dist/index.browser.mjs +488 -260
- package/dist/index.mjs +9 -9
- package/dist/partial-words.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +9 -9
- package/dist/server.mjs +6 -6
- package/dist/{thread-YOTNJEVY.mjs → thread-XBDVXKUE.mjs} +6 -6
- package/dist/workers/entry.mjs +3 -3
- package/package.json +7 -7
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { tokenizeOptions, getNestedValue,
|
|
1
|
+
import { tokenizeOptions, getNestedValue, visitTokens } from './chunk-WFVOBZS4.mjs';
|
|
2
2
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-IBR6634K.mjs';
|
|
4
4
|
|
|
@@ -77,6 +77,7 @@ function expandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
|
|
|
77
77
|
// src/core/partition/partial-words/text.ts
|
|
78
78
|
var encoder = new TextEncoder();
|
|
79
79
|
var LAST_BASIC_PLANE_CODE_POINT = 65535;
|
|
80
|
+
var BASIC_PLANE_SIZE = 65536;
|
|
80
81
|
var UTF8_CONTINUATION_MASK = 192;
|
|
81
82
|
var UTF8_CONTINUATION_BITS = 128;
|
|
82
83
|
function codePointsOf(text) {
|
|
@@ -92,6 +93,13 @@ function codePointsOf(text) {
|
|
|
92
93
|
function runKeyAt(points, at) {
|
|
93
94
|
return String.fromCodePoint(points[at], points[at + 1], points[at + 2]);
|
|
94
95
|
}
|
|
96
|
+
function packedRunKeyAt(points, at) {
|
|
97
|
+
const first = points[at];
|
|
98
|
+
const second = points[at + 1];
|
|
99
|
+
const third = points[at + 2];
|
|
100
|
+
if ((first | second | third) > LAST_BASIC_PLANE_CODE_POINT) return String.fromCodePoint(first, second, third);
|
|
101
|
+
return first * BASIC_PLANE_SIZE * BASIC_PLANE_SIZE + second * BASIC_PLANE_SIZE + third;
|
|
102
|
+
}
|
|
95
103
|
function partialWordProbe(text) {
|
|
96
104
|
return { text, codePoints: codePointsOf(text), utf8: encoder.encode(text) };
|
|
97
105
|
}
|
|
@@ -149,15 +157,35 @@ function expandPartialWords(subs, analysis, requests) {
|
|
|
149
157
|
|
|
150
158
|
// src/core/partition/partial-words/constants.ts
|
|
151
159
|
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
152
|
-
var GROWING_PARTIAL_ENTRY_BYTES =
|
|
160
|
+
var GROWING_PARTIAL_ENTRY_BYTES = 104;
|
|
161
|
+
var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
|
|
162
|
+
var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
|
|
153
163
|
var GROWING_PARTIAL_RUN_BYTES = 120;
|
|
154
164
|
var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
|
|
155
|
-
var GROWING_PARTIAL_COUNTED_SPELLING_BYTES = 64;
|
|
156
165
|
var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
|
|
157
166
|
|
|
167
|
+
// src/core/partition/constants.ts
|
|
168
|
+
var TARGET_BLOCK_ENTRIES = 128;
|
|
169
|
+
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
170
|
+
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
171
|
+
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
172
|
+
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
173
|
+
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
174
|
+
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
175
|
+
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
176
|
+
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
177
|
+
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
178
|
+
|
|
158
179
|
// src/core/partition/frozen/string-blob.ts
|
|
159
180
|
var SURROGATE = /[\ud800-\udfff]/;
|
|
181
|
+
var UTF8_BYTES_PER_UTF16_UNIT = 3;
|
|
160
182
|
var encoder2 = new TextEncoder();
|
|
183
|
+
var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
|
|
184
|
+
function encodeLookupKey(text) {
|
|
185
|
+
if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder2.encode(text);
|
|
186
|
+
const { written } = encoder2.encodeInto(text, lookupKeyBytes);
|
|
187
|
+
return lookupKeyBytes.subarray(0, written);
|
|
188
|
+
}
|
|
161
189
|
function encodeOneByOne(strings) {
|
|
162
190
|
const encoded = new Array(strings.length);
|
|
163
191
|
let blobLength = 0;
|
|
@@ -218,7 +246,97 @@ function codePointOrder(strings) {
|
|
|
218
246
|
return order;
|
|
219
247
|
}
|
|
220
248
|
|
|
249
|
+
// src/core/partition/partial-words/spellings.ts
|
|
250
|
+
var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
|
|
251
|
+
function codePointCountOf(text) {
|
|
252
|
+
let count = 0;
|
|
253
|
+
for (let index = 0; index < text.length; count++) {
|
|
254
|
+
index += (text.codePointAt(index) ?? 0) > LAST_BASIC_PLANE_CODE_POINT2 ? 2 : 1;
|
|
255
|
+
}
|
|
256
|
+
return count;
|
|
257
|
+
}
|
|
258
|
+
function holdsRun(spelling) {
|
|
259
|
+
if (spelling.length > 2 * PARTIAL_WORD_RUN_CODE_POINTS) return true;
|
|
260
|
+
if (spelling.length <= PARTIAL_WORD_RUN_CODE_POINTS) return false;
|
|
261
|
+
return codePointCountOf(spelling) > PARTIAL_WORD_RUN_CODE_POINTS;
|
|
262
|
+
}
|
|
263
|
+
function countSpelling(counts, token, spelling, change) {
|
|
264
|
+
if (!holdsRun(spelling)) return;
|
|
265
|
+
let byToken = counts.get(token);
|
|
266
|
+
if (byToken === void 0) {
|
|
267
|
+
if (change <= 0) return;
|
|
268
|
+
byToken = /* @__PURE__ */ new Map();
|
|
269
|
+
counts.set(token, byToken);
|
|
270
|
+
}
|
|
271
|
+
const next = (byToken.get(spelling) ?? 0) + change;
|
|
272
|
+
if (next > 0) {
|
|
273
|
+
byToken.set(spelling, next);
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
byToken.delete(spelling);
|
|
277
|
+
if (byToken.size === 0) counts.delete(token);
|
|
278
|
+
}
|
|
279
|
+
function textsOf(value) {
|
|
280
|
+
if (typeof value === "string") return [value];
|
|
281
|
+
if (!Array.isArray(value)) return [];
|
|
282
|
+
return value.filter((element) => typeof element === "string");
|
|
283
|
+
}
|
|
284
|
+
function countDocumentSpellings(counts, document, analysis, direction) {
|
|
285
|
+
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
286
|
+
const countOccurrence = (token, _position, surface) => {
|
|
287
|
+
countSpelling(counts, token, surface ?? token, direction);
|
|
288
|
+
};
|
|
289
|
+
for (const field of analysis.fields) {
|
|
290
|
+
for (const text of textsOf(getNestedValue(document, field))) {
|
|
291
|
+
visitTokens(text, analysis.language, tokenizing, countOccurrence);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
function spellingCountsOf(documents, analysis) {
|
|
296
|
+
const counts = /* @__PURE__ */ new Map();
|
|
297
|
+
for (const document of documents) countDocumentSpellings(counts, document, analysis, 1);
|
|
298
|
+
return counts;
|
|
299
|
+
}
|
|
300
|
+
function encodePartialSpellings(fields, counts) {
|
|
301
|
+
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
302
|
+
for (const [token, byToken] of counts) {
|
|
303
|
+
for (const [spelling, count] of byToken) {
|
|
304
|
+
encoded.tokens.push(token);
|
|
305
|
+
encoded.spellings.push(spelling);
|
|
306
|
+
encoded.counts.push(count);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return encoded;
|
|
310
|
+
}
|
|
311
|
+
function sameFields(left, right) {
|
|
312
|
+
if (left.length !== right.length) return false;
|
|
313
|
+
for (let at = 0; at < left.length; at++) {
|
|
314
|
+
if (left[at] !== right[at]) return false;
|
|
315
|
+
}
|
|
316
|
+
return true;
|
|
317
|
+
}
|
|
318
|
+
|
|
221
319
|
// src/core/partition/partial-words/frozen-table.ts
|
|
320
|
+
var decoder = new TextDecoder();
|
|
321
|
+
function compareEntries(left, right) {
|
|
322
|
+
return compareCodePoints(left.spelling, right.spelling) || left.tokenIndex - right.tokenIndex;
|
|
323
|
+
}
|
|
324
|
+
function orderedEntriesOf(source, tokenTable) {
|
|
325
|
+
const entries = [];
|
|
326
|
+
for (let at = 0; at < source.tokens.length; at++) {
|
|
327
|
+
if (source.counts[at] <= 0) continue;
|
|
328
|
+
const tokenIndex = tokenTable.find(source.tokens[at]);
|
|
329
|
+
if (tokenIndex >= 0) entries.push({ spelling: source.spellings[at], tokenIndex, count: source.counts[at] });
|
|
330
|
+
}
|
|
331
|
+
entries.sort(compareEntries);
|
|
332
|
+
const distinct = [];
|
|
333
|
+
for (const entry of entries) {
|
|
334
|
+
const last = distinct[distinct.length - 1];
|
|
335
|
+
if (last !== void 0 && compareEntries(last, entry) === 0) last.count += entry.count;
|
|
336
|
+
else distinct.push(entry);
|
|
337
|
+
}
|
|
338
|
+
return distinct;
|
|
339
|
+
}
|
|
222
340
|
function buildPartialWordTable(source, tokenTable) {
|
|
223
341
|
const spellings = [];
|
|
224
342
|
const spellingTokens = [];
|
|
@@ -238,11 +356,7 @@ function buildPartialWordTable(source, tokenTable) {
|
|
|
238
356
|
else if (list[list.length - 1] !== entry) list.push(entry);
|
|
239
357
|
}
|
|
240
358
|
};
|
|
241
|
-
for (
|
|
242
|
-
if (source.counts[at] <= 0) continue;
|
|
243
|
-
const tokenIndex = tokenTable.find(source.tokens[at]);
|
|
244
|
-
if (tokenIndex >= 0) addSpelling(source.spellings[at], tokenIndex, source.counts[at]);
|
|
245
|
-
}
|
|
359
|
+
for (const entry of orderedEntriesOf(source, tokenTable)) addSpelling(entry.spelling, entry.tokenIndex, entry.count);
|
|
246
360
|
const runKeys = [...runLists.keys()].sort(compareCodePoints);
|
|
247
361
|
const runs = new Uint32Array(runKeys.length * PARTIAL_WORD_RUN_CODE_POINTS);
|
|
248
362
|
const runOffsets = new Uint32Array(runKeys.length + 1);
|
|
@@ -268,6 +382,13 @@ function buildPartialWordTable(source, tokenTable) {
|
|
|
268
382
|
runEntries
|
|
269
383
|
};
|
|
270
384
|
}
|
|
385
|
+
function countTableSpellings(counts, table, tokenTable) {
|
|
386
|
+
const { spellings, spellingOffsets, spellingTokens, spellingCounts } = table;
|
|
387
|
+
for (let entry = 0; entry < spellingTokens.length; entry++) {
|
|
388
|
+
const spelling = decoder.decode(spellings.subarray(spellingOffsets[entry], spellingOffsets[entry + 1]));
|
|
389
|
+
countSpelling(counts, tokenTable.tokenAt(spellingTokens[entry]), spelling, spellingCounts[entry]);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
271
392
|
function findRun(runs, first, second, third) {
|
|
272
393
|
let low = 0;
|
|
273
394
|
let high = runs.length / PARTIAL_WORD_RUN_CODE_POINTS;
|
|
@@ -339,67 +460,30 @@ function readPartialWordTable(table, tokenTable, removedCounts) {
|
|
|
339
460
|
};
|
|
340
461
|
}
|
|
341
462
|
|
|
342
|
-
// src/core/partition/partial-words/
|
|
343
|
-
function
|
|
344
|
-
let
|
|
345
|
-
|
|
346
|
-
if (
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
}
|
|
350
|
-
|
|
351
|
-
if (next > 0) {
|
|
352
|
-
byToken.set(spelling, next);
|
|
353
|
-
return;
|
|
354
|
-
}
|
|
355
|
-
byToken.delete(spelling);
|
|
356
|
-
if (byToken.size === 0) counts.delete(token);
|
|
357
|
-
}
|
|
358
|
-
function textsOf(value) {
|
|
359
|
-
if (typeof value === "string") return [value];
|
|
360
|
-
if (!Array.isArray(value)) return [];
|
|
361
|
-
return value.filter((element) => typeof element === "string");
|
|
362
|
-
}
|
|
363
|
-
function countDocumentSpellings(counts, document, analysis, direction) {
|
|
364
|
-
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
365
|
-
for (const field of analysis.fields) {
|
|
366
|
-
for (const text of textsOf(getNestedValue(document, field))) {
|
|
367
|
-
for (const occurrence of tokenizeIterator(text, analysis.language, tokenizing)) {
|
|
368
|
-
countSpelling(counts, occurrence.token, occurrence.surface ?? occurrence.token, direction);
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
}
|
|
373
|
-
function spellingCountsOf(documents, analysis) {
|
|
374
|
-
const counts = /* @__PURE__ */ new Map();
|
|
375
|
-
for (const document of documents) countDocumentSpellings(counts, document, analysis, 1);
|
|
376
|
-
return counts;
|
|
377
|
-
}
|
|
378
|
-
function encodePartialSpellings(fields, counts) {
|
|
379
|
-
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
380
|
-
for (const [token, byToken] of counts) {
|
|
381
|
-
for (const [spelling, count] of byToken) {
|
|
382
|
-
encoded.tokens.push(token);
|
|
383
|
-
encoded.spellings.push(spelling);
|
|
384
|
-
encoded.counts.push(count);
|
|
385
|
-
}
|
|
386
|
-
}
|
|
387
|
-
return encoded;
|
|
463
|
+
// src/core/partition/partial-words/frozen-slot.ts
|
|
464
|
+
function tombstonedOrdinalsOf(documents) {
|
|
465
|
+
let marks = null;
|
|
466
|
+
for (let ordinal = 0; ordinal < documents.count; ordinal++) {
|
|
467
|
+
if (!documents.isTombstoned(ordinal)) continue;
|
|
468
|
+
marks ??= new Uint8Array(documents.count);
|
|
469
|
+
marks[ordinal] = 1;
|
|
470
|
+
}
|
|
471
|
+
return marks;
|
|
388
472
|
}
|
|
389
|
-
function
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
if (left[at] !== right[at]) return false;
|
|
473
|
+
function addCounts(target, source) {
|
|
474
|
+
for (const [token, bySpelling] of source) {
|
|
475
|
+
for (const [spelling, count] of bySpelling) countSpelling(target, token, spelling, count);
|
|
393
476
|
}
|
|
394
|
-
return true;
|
|
395
477
|
}
|
|
396
|
-
|
|
397
|
-
// src/core/partition/partial-words/frozen-slot.ts
|
|
398
478
|
function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
399
479
|
let table = initial;
|
|
480
|
+
let uncounted = null;
|
|
400
481
|
let removedCounts = /* @__PURE__ */ new Map();
|
|
401
482
|
let reader = initial === null ? null : readPartialWordTable(initial, tokenTable, removedCounts);
|
|
402
483
|
let pendingRemovals = [];
|
|
484
|
+
function isCounted(ordinal) {
|
|
485
|
+
return uncounted === null || uncounted[ordinal] === 0;
|
|
486
|
+
}
|
|
403
487
|
function subtractPendingRemovals(current, analysis) {
|
|
404
488
|
if (pendingRemovals.length === 0) return;
|
|
405
489
|
const removed = /* @__PURE__ */ new Map();
|
|
@@ -412,23 +496,64 @@ function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
|
412
496
|
}
|
|
413
497
|
}
|
|
414
498
|
}
|
|
499
|
+
function rebuild(analysis) {
|
|
500
|
+
uncounted = tombstonedOrdinalsOf(documents);
|
|
501
|
+
const counts = /* @__PURE__ */ new Map();
|
|
502
|
+
for (let ordinal = 0; ordinal < documents.count; ordinal++) {
|
|
503
|
+
if (isCounted(ordinal)) countDocumentSpellings(counts, documents.documentAt(ordinal), analysis, 1);
|
|
504
|
+
}
|
|
505
|
+
const rebuilt = buildPartialWordTable(encodePartialSpellings(analysis.fields, counts), tokenTable);
|
|
506
|
+
table = rebuilt;
|
|
507
|
+
removedCounts = /* @__PURE__ */ new Map();
|
|
508
|
+
pendingRemovals = [];
|
|
509
|
+
const rebuiltReader = readPartialWordTable(rebuilt, tokenTable, removedCounts);
|
|
510
|
+
reader = rebuiltReader;
|
|
511
|
+
return rebuiltReader;
|
|
512
|
+
}
|
|
513
|
+
function correctionsAgainstTable(mergedOrdinals) {
|
|
514
|
+
let corrections = 0;
|
|
515
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
516
|
+
if (mergedOrdinals[ordinal] >= 0 !== isCounted(ordinal)) corrections += 1;
|
|
517
|
+
}
|
|
518
|
+
return corrections;
|
|
519
|
+
}
|
|
520
|
+
function keptCount(mergedOrdinals) {
|
|
521
|
+
let kept = 0;
|
|
522
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
523
|
+
if (mergedOrdinals[ordinal] >= 0) kept += 1;
|
|
524
|
+
}
|
|
525
|
+
return kept;
|
|
526
|
+
}
|
|
415
527
|
return {
|
|
416
528
|
readerFor(analysis) {
|
|
417
529
|
if (analysis.fields.length === 0) return null;
|
|
418
|
-
if (table === null || reader === null || !sameFields(table.fields, analysis.fields))
|
|
419
|
-
const counts = spellingCountsOf(documents.survivors(), analysis);
|
|
420
|
-
table = buildPartialWordTable(encodePartialSpellings(analysis.fields, counts), tokenTable);
|
|
421
|
-
removedCounts = /* @__PURE__ */ new Map();
|
|
422
|
-
pendingRemovals = [];
|
|
423
|
-
reader = readPartialWordTable(table, tokenTable, removedCounts);
|
|
424
|
-
return reader;
|
|
425
|
-
}
|
|
530
|
+
if (table === null || reader === null || !sameFields(table.fields, analysis.fields)) return rebuild(analysis);
|
|
426
531
|
subtractPendingRemovals(reader, analysis);
|
|
427
532
|
return reader;
|
|
428
533
|
},
|
|
429
534
|
afterRemoval(ordinal) {
|
|
430
535
|
if (table !== null) pendingRemovals.push(ordinal);
|
|
431
536
|
},
|
|
537
|
+
tableOfEveryDocument(analysis) {
|
|
538
|
+
if (table === null || uncounted !== null || !sameFields(table.fields, analysis.fields)) return null;
|
|
539
|
+
return table;
|
|
540
|
+
},
|
|
541
|
+
countKeptSpellings(counts, analysis, mergedOrdinals) {
|
|
542
|
+
if (table === null || !sameFields(table.fields, analysis.fields) || correctionsAgainstTable(mergedOrdinals) > keptCount(mergedOrdinals)) {
|
|
543
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
544
|
+
if (mergedOrdinals[ordinal] >= 0) countDocumentSpellings(counts, documents.documentAt(ordinal), analysis, 1);
|
|
545
|
+
}
|
|
546
|
+
return;
|
|
547
|
+
}
|
|
548
|
+
const corrected = /* @__PURE__ */ new Map();
|
|
549
|
+
countTableSpellings(corrected, table, tokenTable);
|
|
550
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
551
|
+
const kept = mergedOrdinals[ordinal] >= 0;
|
|
552
|
+
if (kept === isCounted(ordinal)) continue;
|
|
553
|
+
countDocumentSpellings(corrected, documents.documentAt(ordinal), analysis, kept ? 1 : -1);
|
|
554
|
+
}
|
|
555
|
+
addCounts(counts, corrected);
|
|
556
|
+
},
|
|
432
557
|
spellings() {
|
|
433
558
|
return null;
|
|
434
559
|
},
|
|
@@ -444,94 +569,127 @@ function createGrowingPartialWordSlot(view) {
|
|
|
444
569
|
let fields = [];
|
|
445
570
|
let everyDocumentPending = false;
|
|
446
571
|
const pending = /* @__PURE__ */ new Set();
|
|
447
|
-
const counts = /* @__PURE__ */ new Map();
|
|
448
572
|
const entriesByToken = /* @__PURE__ */ new Map();
|
|
449
573
|
const entrySpellings = [];
|
|
450
574
|
const entryTokens = [];
|
|
451
575
|
const entryLengths = [];
|
|
576
|
+
const entryCounts = [];
|
|
577
|
+
const entryInRuns = [];
|
|
452
578
|
const freeEntries = [];
|
|
579
|
+
const entriesOutsideRuns = [];
|
|
453
580
|
const runs = /* @__PURE__ */ new Map();
|
|
454
581
|
let liveEntries = 0;
|
|
455
582
|
let runMembers = 0;
|
|
456
|
-
let countedSpellings = 0;
|
|
457
583
|
function resetSpellings() {
|
|
458
584
|
pending.clear();
|
|
459
|
-
counts.clear();
|
|
460
|
-
countedSpellings = 0;
|
|
461
585
|
entriesByToken.clear();
|
|
462
586
|
entrySpellings.length = 0;
|
|
463
587
|
entryTokens.length = 0;
|
|
464
588
|
entryLengths.length = 0;
|
|
589
|
+
entryCounts.length = 0;
|
|
590
|
+
entryInRuns.length = 0;
|
|
465
591
|
freeEntries.length = 0;
|
|
592
|
+
entriesOutsideRuns.length = 0;
|
|
466
593
|
runs.clear();
|
|
467
594
|
liveEntries = 0;
|
|
468
595
|
runMembers = 0;
|
|
469
596
|
}
|
|
470
|
-
function addEntry(token, spelling) {
|
|
471
|
-
const
|
|
472
|
-
if (
|
|
597
|
+
function addEntry(byToken, token, spelling, count) {
|
|
598
|
+
const length = codePointCountOf(spelling);
|
|
599
|
+
if (length <= PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
473
600
|
const entry = freeEntries.pop() ?? entrySpellings.length;
|
|
474
601
|
entrySpellings[entry] = spelling;
|
|
475
602
|
entryTokens[entry] = token;
|
|
476
|
-
entryLengths[entry] =
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
603
|
+
entryLengths[entry] = length;
|
|
604
|
+
entryCounts[entry] = count;
|
|
605
|
+
entryInRuns[entry] = false;
|
|
606
|
+
entriesOutsideRuns.push(entry);
|
|
607
|
+
if (entriesOutsideRuns.length > GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY * entrySpellings.length) {
|
|
608
|
+
dropFreedAndRepeatedPendingEntries();
|
|
481
609
|
}
|
|
482
610
|
byToken.set(spelling, entry);
|
|
483
611
|
liveEntries++;
|
|
612
|
+
}
|
|
613
|
+
function dropFreedAndRepeatedPendingEntries() {
|
|
614
|
+
let kept = 0;
|
|
615
|
+
for (const entry of entriesOutsideRuns) {
|
|
616
|
+
if (entryLengths[entry] === 0 || entryInRuns[entry]) continue;
|
|
617
|
+
entryInRuns[entry] = true;
|
|
618
|
+
entriesOutsideRuns[kept++] = entry;
|
|
619
|
+
}
|
|
620
|
+
entriesOutsideRuns.length = kept;
|
|
621
|
+
for (let at = 0; at < kept; at++) entryInRuns[entriesOutsideRuns[at]] = false;
|
|
622
|
+
}
|
|
623
|
+
function addToRuns(entry) {
|
|
624
|
+
const points = codePointsOf(entrySpellings[entry]);
|
|
484
625
|
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
485
|
-
const key =
|
|
626
|
+
const key = packedRunKeyAt(points, at);
|
|
486
627
|
let members = runs.get(key);
|
|
487
628
|
if (members === void 0) {
|
|
488
629
|
members = /* @__PURE__ */ new Set();
|
|
489
630
|
runs.set(key, members);
|
|
490
631
|
}
|
|
491
|
-
|
|
632
|
+
const size = members.size;
|
|
492
633
|
members.add(entry);
|
|
493
|
-
runMembers++;
|
|
634
|
+
if (members.size !== size) runMembers++;
|
|
494
635
|
}
|
|
636
|
+
entryInRuns[entry] = true;
|
|
495
637
|
}
|
|
496
|
-
function
|
|
497
|
-
const
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
638
|
+
function addEntriesToRuns() {
|
|
639
|
+
for (const entry of entriesOutsideRuns) {
|
|
640
|
+
if (entryLengths[entry] > 0 && !entryInRuns[entry]) addToRuns(entry);
|
|
641
|
+
}
|
|
642
|
+
entriesOutsideRuns.length = 0;
|
|
643
|
+
}
|
|
644
|
+
function removeFromRuns(entry) {
|
|
645
|
+
const points = codePointsOf(entrySpellings[entry]);
|
|
501
646
|
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
502
|
-
const key =
|
|
647
|
+
const key = packedRunKeyAt(points, at);
|
|
503
648
|
const members = runs.get(key);
|
|
504
649
|
if (members === void 0 || !members.delete(entry)) continue;
|
|
505
650
|
runMembers--;
|
|
506
651
|
if (members.size === 0) runs.delete(key);
|
|
507
652
|
}
|
|
653
|
+
entryInRuns[entry] = false;
|
|
654
|
+
}
|
|
655
|
+
function removeEntry(byToken, token, spelling, entry) {
|
|
656
|
+
if (entryInRuns[entry]) removeFromRuns(entry);
|
|
508
657
|
byToken.delete(spelling);
|
|
509
658
|
if (byToken.size === 0) entriesByToken.delete(token);
|
|
510
659
|
entrySpellings[entry] = "";
|
|
511
660
|
entryTokens[entry] = "";
|
|
512
661
|
entryLengths[entry] = 0;
|
|
662
|
+
entryCounts[entry] = 0;
|
|
513
663
|
freeEntries.push(entry);
|
|
514
664
|
liveEntries--;
|
|
515
665
|
}
|
|
516
666
|
function change(token, spelling, delta) {
|
|
517
|
-
let byToken =
|
|
518
|
-
const
|
|
519
|
-
|
|
520
|
-
|
|
667
|
+
let byToken = entriesByToken.get(token);
|
|
668
|
+
const entry = byToken?.get(spelling);
|
|
669
|
+
if (byToken !== void 0 && entry !== void 0) {
|
|
670
|
+
const after = entryCounts[entry] + delta;
|
|
671
|
+
if (after > 0) entryCounts[entry] = after;
|
|
672
|
+
else removeEntry(byToken, token, spelling, entry);
|
|
673
|
+
return;
|
|
674
|
+
}
|
|
675
|
+
if (delta <= 0) return;
|
|
521
676
|
if (byToken === void 0) {
|
|
522
677
|
byToken = /* @__PURE__ */ new Map();
|
|
523
|
-
|
|
678
|
+
entriesByToken.set(token, byToken);
|
|
524
679
|
}
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
680
|
+
addEntry(byToken, token, spelling, delta);
|
|
681
|
+
if (byToken.size === 0) entriesByToken.delete(token);
|
|
682
|
+
}
|
|
683
|
+
function encodedSpellings() {
|
|
684
|
+
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
685
|
+
for (const [token, byToken] of entriesByToken) {
|
|
686
|
+
for (const [spelling, entry] of byToken) {
|
|
687
|
+
encoded.tokens.push(token);
|
|
688
|
+
encoded.spellings.push(spelling);
|
|
689
|
+
encoded.counts.push(entryCounts[entry]);
|
|
690
|
+
}
|
|
534
691
|
}
|
|
692
|
+
return encoded;
|
|
535
693
|
}
|
|
536
694
|
function applyCounts(source, direction) {
|
|
537
695
|
for (const [token, byToken] of source) {
|
|
@@ -572,7 +730,7 @@ function createGrowingPartialWordSlot(view) {
|
|
|
572
730
|
if (points.length < PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
573
731
|
let narrowest = null;
|
|
574
732
|
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
575
|
-
const members = runs.get(
|
|
733
|
+
const members = runs.get(packedRunKeyAt(points, at));
|
|
576
734
|
if (members === void 0) return;
|
|
577
735
|
if (narrowest === null || members.size < narrowest.size) narrowest = members;
|
|
578
736
|
}
|
|
@@ -593,11 +751,12 @@ function createGrowingPartialWordSlot(view) {
|
|
|
593
751
|
if (analysis.fields.length === 0) return null;
|
|
594
752
|
adoptFields(analysis.fields);
|
|
595
753
|
resolvePending(analysis);
|
|
754
|
+
addEntriesToRuns();
|
|
596
755
|
return reader;
|
|
597
756
|
},
|
|
598
757
|
spellings() {
|
|
599
758
|
if (fields.length === 0 || everyDocumentPending || pending.size > 0) return null;
|
|
600
|
-
return
|
|
759
|
+
return encodedSpellings();
|
|
601
760
|
},
|
|
602
761
|
afterDocument(documentFields, internalId, spellings, direction) {
|
|
603
762
|
if (documentFields.length === 0) return;
|
|
@@ -641,7 +800,7 @@ function createGrowingPartialWordSlot(view) {
|
|
|
641
800
|
resetSpellings();
|
|
642
801
|
},
|
|
643
802
|
bytes() {
|
|
644
|
-
return liveEntries * GROWING_PARTIAL_ENTRY_BYTES + runs.size * GROWING_PARTIAL_RUN_BYTES + runMembers * GROWING_PARTIAL_RUN_MEMBER_BYTES +
|
|
803
|
+
return liveEntries * GROWING_PARTIAL_ENTRY_BYTES + runs.size * GROWING_PARTIAL_RUN_BYTES + runMembers * GROWING_PARTIAL_RUN_MEMBER_BYTES + entriesOutsideRuns.length * GROWING_PARTIAL_PENDING_ENTRY_BYTES;
|
|
645
804
|
}
|
|
646
805
|
};
|
|
647
806
|
return slot;
|
|
@@ -656,4 +815,4 @@ var partialWordSearchModule = {
|
|
|
656
815
|
expand: expandPartialWords
|
|
657
816
|
};
|
|
658
817
|
|
|
659
|
-
export { codePointOrder, countSpelling, encodePartialSpellings, encodeStringBlob, expandTermPrefix, partialWordSearchModule, partitionDocumentFrequency, spellingCountsOf, suggestDisplayTerms };
|
|
818
|
+
export { PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, codePointOrder, countSpelling, encodeLookupKey, encodePartialSpellings, encodeStringBlob, expandTermPrefix, partialWordSearchModule, partitionDocumentFrequency, sameFields, spellingCountsOf, suggestDisplayTerms };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isWordIndexedBase, flattenSchema } from './chunk-
|
|
1
|
+
import { isWordIndexedBase, flattenSchema } from './chunk-PXDQFYD5.mjs';
|
|
2
2
|
import { createNarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/tokenizer/constants.ts
|
|
@@ -409,53 +409,6 @@ function tokenize(text, language, options) {
|
|
|
409
409
|
});
|
|
410
410
|
return { tokens, originalTokens, surfaces };
|
|
411
411
|
}
|
|
412
|
-
function* tokenizeIterator(text, language, options) {
|
|
413
|
-
const {
|
|
414
|
-
stem = true,
|
|
415
|
-
removeStopWords = true,
|
|
416
|
-
removeDiacritics = false,
|
|
417
|
-
collectSurfaces = false,
|
|
418
|
-
customTokenizer,
|
|
419
|
-
stopWordOverride
|
|
420
|
-
} = options ?? {};
|
|
421
|
-
if (customTokenizer) {
|
|
422
|
-
const customResult = customTokenizer.tokenize(text);
|
|
423
|
-
for (const entry of customResult) {
|
|
424
|
-
yield entry;
|
|
425
|
-
}
|
|
426
|
-
return;
|
|
427
|
-
}
|
|
428
|
-
const normalized = normalizeForSplitting(text);
|
|
429
|
-
const rawParts = splitText(normalized, language);
|
|
430
|
-
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
431
|
-
const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
|
|
432
|
-
const stemsTokens = stem && language.stemmer !== null;
|
|
433
|
-
const wantSurfaces = collectSurfaces && stemsTokens;
|
|
434
|
-
const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
|
|
435
|
-
const stripPossessives = language.tokenizer?.stripPossessive ?? false;
|
|
436
|
-
let position = 0;
|
|
437
|
-
for (const part of rawParts) {
|
|
438
|
-
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
439
|
-
if (candidate.length < minLength) {
|
|
440
|
-
position++;
|
|
441
|
-
continue;
|
|
442
|
-
}
|
|
443
|
-
if (stopWords.has(candidate)) {
|
|
444
|
-
position++;
|
|
445
|
-
continue;
|
|
446
|
-
}
|
|
447
|
-
const processed = transformToken(candidate, language, stem, effectiveDiacritics);
|
|
448
|
-
if (processed.length > 0) {
|
|
449
|
-
if (wantSurfaces) {
|
|
450
|
-
const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
|
|
451
|
-
yield { token: processed, position, surface: surface === processed ? void 0 : surface };
|
|
452
|
-
} else {
|
|
453
|
-
yield { token: processed, position };
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
|
-
position++;
|
|
457
|
-
}
|
|
458
|
-
}
|
|
459
412
|
|
|
460
413
|
// src/core/partition/utils.ts
|
|
461
414
|
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
@@ -529,4 +482,4 @@ function getFieldValueByInternalId(docStore, internalId, fieldPath) {
|
|
|
529
482
|
return getFieldValueForDoc(docStore, externalId, fieldPath);
|
|
530
483
|
}
|
|
531
484
|
|
|
532
|
-
export { clearNormalizationCache, configureNormalizationCache, getFieldValueByInternalId, getFieldValueForDoc, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, resetNormalizationCache, textFieldsChanged, tokenize,
|
|
485
|
+
export { clearNormalizationCache, configureNormalizationCache, getFieldValueByInternalId, getFieldValueForDoc, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, resetNormalizationCache, textFieldsChanged, tokenize, tokenizeOptions, visitTokens };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-
|
|
1
|
+
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-PXDQFYD5.mjs';
|
|
2
2
|
import { ErrorCodes, NarsilError } from './chunk-I64LXKO3.mjs';
|
|
3
3
|
|
|
4
4
|
// src/narsil/internals.ts
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MAX_FILTER_DEPTH, MAX_SORT_FIELDS } from './chunk-
|
|
1
|
+
import { MAX_FILTER_DEPTH, MAX_SORT_FIELDS } from './chunk-IOZPBNSG.mjs';
|
|
2
2
|
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-2E35UM3G.mjs';
|
|
3
3
|
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
4
4
|
import { mkdtemp, open, rm } from 'fs/promises';
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
export { createDirectExecutor } from './chunk-
|
|
2
|
-
import './chunk-
|
|
1
|
+
export { createDirectExecutor } from './chunk-DFW3U5UW.mjs';
|
|
2
|
+
import './chunk-P2BEMMOD.mjs';
|
|
3
3
|
import './chunk-4MQ5F4IL.mjs';
|
|
4
|
-
import './chunk-
|
|
4
|
+
import './chunk-IOZPBNSG.mjs';
|
|
5
5
|
import './chunk-TGP5M5R7.mjs';
|
|
6
6
|
import './chunk-AMGDG4ZJ.mjs';
|
|
7
7
|
import './chunk-ZJ4CSAB2.mjs';
|
|
8
8
|
import './chunk-2E35UM3G.mjs';
|
|
9
9
|
import './chunk-4344I4MR.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import './chunk-
|
|
12
|
-
import './chunk-
|
|
10
|
+
import './chunk-RTR44V2A.mjs';
|
|
11
|
+
import './chunk-WFVOBZS4.mjs';
|
|
12
|
+
import './chunk-PXDQFYD5.mjs';
|
|
13
13
|
import './chunk-KEHK2O6P.mjs';
|
|
14
14
|
import './chunk-GES7U2PM.mjs';
|
|
15
15
|
import './chunk-U2BKYEC7.mjs';
|