@delali/narsil 0.3.1-next.77 → 0.3.1-next.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { tokenizeOptions, getNestedValue, tokenizeIterator } from './chunk-NO3ABFLE.mjs';
1
+ import { tokenizeOptions, getNestedValue, visitTokens } from './chunk-WFVOBZS4.mjs';
2
2
  import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
3
3
  import { compareCodePoints } from './chunk-IBR6634K.mjs';
4
4
 
@@ -77,6 +77,7 @@ function expandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
77
77
  // src/core/partition/partial-words/text.ts
78
78
  var encoder = new TextEncoder();
79
79
  var LAST_BASIC_PLANE_CODE_POINT = 65535;
80
+ var BASIC_PLANE_SIZE = 65536;
80
81
  var UTF8_CONTINUATION_MASK = 192;
81
82
  var UTF8_CONTINUATION_BITS = 128;
82
83
  function codePointsOf(text) {
@@ -92,6 +93,13 @@ function codePointsOf(text) {
92
93
  function runKeyAt(points, at) {
93
94
  return String.fromCodePoint(points[at], points[at + 1], points[at + 2]);
94
95
  }
96
+ function packedRunKeyAt(points, at) {
97
+ const first = points[at];
98
+ const second = points[at + 1];
99
+ const third = points[at + 2];
100
+ if ((first | second | third) > LAST_BASIC_PLANE_CODE_POINT) return String.fromCodePoint(first, second, third);
101
+ return first * BASIC_PLANE_SIZE * BASIC_PLANE_SIZE + second * BASIC_PLANE_SIZE + third;
102
+ }
95
103
  function partialWordProbe(text) {
96
104
  return { text, codePoints: codePointsOf(text), utf8: encoder.encode(text) };
97
105
  }
@@ -149,15 +157,35 @@ function expandPartialWords(subs, analysis, requests) {
149
157
 
150
158
  // src/core/partition/partial-words/constants.ts
151
159
  var PARTIAL_WORD_RUN_CODE_POINTS = 3;
152
- var GROWING_PARTIAL_ENTRY_BYTES = 96;
160
+ var GROWING_PARTIAL_ENTRY_BYTES = 104;
161
+ var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
162
+ var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
153
163
  var GROWING_PARTIAL_RUN_BYTES = 120;
154
164
  var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
155
- var GROWING_PARTIAL_COUNTED_SPELLING_BYTES = 64;
156
165
  var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
157
166
 
167
+ // src/core/partition/constants.ts
168
+ var TARGET_BLOCK_ENTRIES = 128;
169
+ var PRUNING_REJECTION_SLACK = 1e-12;
170
+ var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
171
+ var SEGMENT_TOKEN_TABLE_START = 1 << 12;
172
+ var SEGMENT_POSTING_LOG_START = 1 << 14;
173
+ var SEGMENT_POSITION_POOL_START = 1 << 16;
174
+ var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
175
+ var LOOKUP_KEY_SCRATCH_BYTES = 4096;
176
+ var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
177
+ var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
178
+
158
179
  // src/core/partition/frozen/string-blob.ts
159
180
  var SURROGATE = /[\ud800-\udfff]/;
181
+ var UTF8_BYTES_PER_UTF16_UNIT = 3;
160
182
  var encoder2 = new TextEncoder();
183
+ var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
184
+ function encodeLookupKey(text) {
185
+ if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder2.encode(text);
186
+ const { written } = encoder2.encodeInto(text, lookupKeyBytes);
187
+ return lookupKeyBytes.subarray(0, written);
188
+ }
161
189
  function encodeOneByOne(strings) {
162
190
  const encoded = new Array(strings.length);
163
191
  let blobLength = 0;
@@ -218,7 +246,97 @@ function codePointOrder(strings) {
218
246
  return order;
219
247
  }
220
248
 
249
+ // src/core/partition/partial-words/spellings.ts
250
+ var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
251
+ function codePointCountOf(text) {
252
+ let count = 0;
253
+ for (let index = 0; index < text.length; count++) {
254
+ index += (text.codePointAt(index) ?? 0) > LAST_BASIC_PLANE_CODE_POINT2 ? 2 : 1;
255
+ }
256
+ return count;
257
+ }
258
+ function holdsRun(spelling) {
259
+ if (spelling.length > 2 * PARTIAL_WORD_RUN_CODE_POINTS) return true;
260
+ if (spelling.length <= PARTIAL_WORD_RUN_CODE_POINTS) return false;
261
+ return codePointCountOf(spelling) > PARTIAL_WORD_RUN_CODE_POINTS;
262
+ }
263
+ function countSpelling(counts, token, spelling, change) {
264
+ if (!holdsRun(spelling)) return;
265
+ let byToken = counts.get(token);
266
+ if (byToken === void 0) {
267
+ if (change <= 0) return;
268
+ byToken = /* @__PURE__ */ new Map();
269
+ counts.set(token, byToken);
270
+ }
271
+ const next = (byToken.get(spelling) ?? 0) + change;
272
+ if (next > 0) {
273
+ byToken.set(spelling, next);
274
+ return;
275
+ }
276
+ byToken.delete(spelling);
277
+ if (byToken.size === 0) counts.delete(token);
278
+ }
279
+ function textsOf(value) {
280
+ if (typeof value === "string") return [value];
281
+ if (!Array.isArray(value)) return [];
282
+ return value.filter((element) => typeof element === "string");
283
+ }
284
+ function countDocumentSpellings(counts, document, analysis, direction) {
285
+ const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
286
+ const countOccurrence = (token, _position, surface) => {
287
+ countSpelling(counts, token, surface ?? token, direction);
288
+ };
289
+ for (const field of analysis.fields) {
290
+ for (const text of textsOf(getNestedValue(document, field))) {
291
+ visitTokens(text, analysis.language, tokenizing, countOccurrence);
292
+ }
293
+ }
294
+ }
295
+ function spellingCountsOf(documents, analysis) {
296
+ const counts = /* @__PURE__ */ new Map();
297
+ for (const document of documents) countDocumentSpellings(counts, document, analysis, 1);
298
+ return counts;
299
+ }
300
+ function encodePartialSpellings(fields, counts) {
301
+ const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
302
+ for (const [token, byToken] of counts) {
303
+ for (const [spelling, count] of byToken) {
304
+ encoded.tokens.push(token);
305
+ encoded.spellings.push(spelling);
306
+ encoded.counts.push(count);
307
+ }
308
+ }
309
+ return encoded;
310
+ }
311
+ function sameFields(left, right) {
312
+ if (left.length !== right.length) return false;
313
+ for (let at = 0; at < left.length; at++) {
314
+ if (left[at] !== right[at]) return false;
315
+ }
316
+ return true;
317
+ }
318
+
221
319
  // src/core/partition/partial-words/frozen-table.ts
320
+ var decoder = new TextDecoder();
321
+ function compareEntries(left, right) {
322
+ return compareCodePoints(left.spelling, right.spelling) || left.tokenIndex - right.tokenIndex;
323
+ }
324
+ function orderedEntriesOf(source, tokenTable) {
325
+ const entries = [];
326
+ for (let at = 0; at < source.tokens.length; at++) {
327
+ if (source.counts[at] <= 0) continue;
328
+ const tokenIndex = tokenTable.find(source.tokens[at]);
329
+ if (tokenIndex >= 0) entries.push({ spelling: source.spellings[at], tokenIndex, count: source.counts[at] });
330
+ }
331
+ entries.sort(compareEntries);
332
+ const distinct = [];
333
+ for (const entry of entries) {
334
+ const last = distinct[distinct.length - 1];
335
+ if (last !== void 0 && compareEntries(last, entry) === 0) last.count += entry.count;
336
+ else distinct.push(entry);
337
+ }
338
+ return distinct;
339
+ }
222
340
  function buildPartialWordTable(source, tokenTable) {
223
341
  const spellings = [];
224
342
  const spellingTokens = [];
@@ -238,11 +356,7 @@ function buildPartialWordTable(source, tokenTable) {
238
356
  else if (list[list.length - 1] !== entry) list.push(entry);
239
357
  }
240
358
  };
241
- for (let at = 0; at < source.tokens.length; at++) {
242
- if (source.counts[at] <= 0) continue;
243
- const tokenIndex = tokenTable.find(source.tokens[at]);
244
- if (tokenIndex >= 0) addSpelling(source.spellings[at], tokenIndex, source.counts[at]);
245
- }
359
+ for (const entry of orderedEntriesOf(source, tokenTable)) addSpelling(entry.spelling, entry.tokenIndex, entry.count);
246
360
  const runKeys = [...runLists.keys()].sort(compareCodePoints);
247
361
  const runs = new Uint32Array(runKeys.length * PARTIAL_WORD_RUN_CODE_POINTS);
248
362
  const runOffsets = new Uint32Array(runKeys.length + 1);
@@ -268,6 +382,13 @@ function buildPartialWordTable(source, tokenTable) {
268
382
  runEntries
269
383
  };
270
384
  }
385
+ function countTableSpellings(counts, table, tokenTable) {
386
+ const { spellings, spellingOffsets, spellingTokens, spellingCounts } = table;
387
+ for (let entry = 0; entry < spellingTokens.length; entry++) {
388
+ const spelling = decoder.decode(spellings.subarray(spellingOffsets[entry], spellingOffsets[entry + 1]));
389
+ countSpelling(counts, tokenTable.tokenAt(spellingTokens[entry]), spelling, spellingCounts[entry]);
390
+ }
391
+ }
271
392
  function findRun(runs, first, second, third) {
272
393
  let low = 0;
273
394
  let high = runs.length / PARTIAL_WORD_RUN_CODE_POINTS;
@@ -339,67 +460,30 @@ function readPartialWordTable(table, tokenTable, removedCounts) {
339
460
  };
340
461
  }
341
462
 
342
- // src/core/partition/partial-words/spellings.ts
343
- function countSpelling(counts, token, spelling, change) {
344
- let byToken = counts.get(token);
345
- if (byToken === void 0) {
346
- if (change <= 0) return;
347
- byToken = /* @__PURE__ */ new Map();
348
- counts.set(token, byToken);
349
- }
350
- const next = (byToken.get(spelling) ?? 0) + change;
351
- if (next > 0) {
352
- byToken.set(spelling, next);
353
- return;
354
- }
355
- byToken.delete(spelling);
356
- if (byToken.size === 0) counts.delete(token);
357
- }
358
- function textsOf(value) {
359
- if (typeof value === "string") return [value];
360
- if (!Array.isArray(value)) return [];
361
- return value.filter((element) => typeof element === "string");
362
- }
363
- function countDocumentSpellings(counts, document, analysis, direction) {
364
- const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
365
- for (const field of analysis.fields) {
366
- for (const text of textsOf(getNestedValue(document, field))) {
367
- for (const occurrence of tokenizeIterator(text, analysis.language, tokenizing)) {
368
- countSpelling(counts, occurrence.token, occurrence.surface ?? occurrence.token, direction);
369
- }
370
- }
371
- }
372
- }
373
- function spellingCountsOf(documents, analysis) {
374
- const counts = /* @__PURE__ */ new Map();
375
- for (const document of documents) countDocumentSpellings(counts, document, analysis, 1);
376
- return counts;
377
- }
378
- function encodePartialSpellings(fields, counts) {
379
- const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
380
- for (const [token, byToken] of counts) {
381
- for (const [spelling, count] of byToken) {
382
- encoded.tokens.push(token);
383
- encoded.spellings.push(spelling);
384
- encoded.counts.push(count);
385
- }
386
- }
387
- return encoded;
463
+ // src/core/partition/partial-words/frozen-slot.ts
464
+ function tombstonedOrdinalsOf(documents) {
465
+ let marks = null;
466
+ for (let ordinal = 0; ordinal < documents.count; ordinal++) {
467
+ if (!documents.isTombstoned(ordinal)) continue;
468
+ marks ??= new Uint8Array(documents.count);
469
+ marks[ordinal] = 1;
470
+ }
471
+ return marks;
388
472
  }
389
- function sameFields(left, right) {
390
- if (left.length !== right.length) return false;
391
- for (let at = 0; at < left.length; at++) {
392
- if (left[at] !== right[at]) return false;
473
+ function addCounts(target, source) {
474
+ for (const [token, bySpelling] of source) {
475
+ for (const [spelling, count] of bySpelling) countSpelling(target, token, spelling, count);
393
476
  }
394
- return true;
395
477
  }
396
-
397
- // src/core/partition/partial-words/frozen-slot.ts
398
478
  function createFrozenPartialWordSlot(initial, tokenTable, documents) {
399
479
  let table = initial;
480
+ let uncounted = null;
400
481
  let removedCounts = /* @__PURE__ */ new Map();
401
482
  let reader = initial === null ? null : readPartialWordTable(initial, tokenTable, removedCounts);
402
483
  let pendingRemovals = [];
484
+ function isCounted(ordinal) {
485
+ return uncounted === null || uncounted[ordinal] === 0;
486
+ }
403
487
  function subtractPendingRemovals(current, analysis) {
404
488
  if (pendingRemovals.length === 0) return;
405
489
  const removed = /* @__PURE__ */ new Map();
@@ -412,23 +496,64 @@ function createFrozenPartialWordSlot(initial, tokenTable, documents) {
412
496
  }
413
497
  }
414
498
  }
499
+ function rebuild(analysis) {
500
+ uncounted = tombstonedOrdinalsOf(documents);
501
+ const counts = /* @__PURE__ */ new Map();
502
+ for (let ordinal = 0; ordinal < documents.count; ordinal++) {
503
+ if (isCounted(ordinal)) countDocumentSpellings(counts, documents.documentAt(ordinal), analysis, 1);
504
+ }
505
+ const rebuilt = buildPartialWordTable(encodePartialSpellings(analysis.fields, counts), tokenTable);
506
+ table = rebuilt;
507
+ removedCounts = /* @__PURE__ */ new Map();
508
+ pendingRemovals = [];
509
+ const rebuiltReader = readPartialWordTable(rebuilt, tokenTable, removedCounts);
510
+ reader = rebuiltReader;
511
+ return rebuiltReader;
512
+ }
513
+ function correctionsAgainstTable(mergedOrdinals) {
514
+ let corrections = 0;
515
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
516
+ if (mergedOrdinals[ordinal] >= 0 !== isCounted(ordinal)) corrections += 1;
517
+ }
518
+ return corrections;
519
+ }
520
+ function keptCount(mergedOrdinals) {
521
+ let kept = 0;
522
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
523
+ if (mergedOrdinals[ordinal] >= 0) kept += 1;
524
+ }
525
+ return kept;
526
+ }
415
527
  return {
416
528
  readerFor(analysis) {
417
529
  if (analysis.fields.length === 0) return null;
418
- if (table === null || reader === null || !sameFields(table.fields, analysis.fields)) {
419
- const counts = spellingCountsOf(documents.survivors(), analysis);
420
- table = buildPartialWordTable(encodePartialSpellings(analysis.fields, counts), tokenTable);
421
- removedCounts = /* @__PURE__ */ new Map();
422
- pendingRemovals = [];
423
- reader = readPartialWordTable(table, tokenTable, removedCounts);
424
- return reader;
425
- }
530
+ if (table === null || reader === null || !sameFields(table.fields, analysis.fields)) return rebuild(analysis);
426
531
  subtractPendingRemovals(reader, analysis);
427
532
  return reader;
428
533
  },
429
534
  afterRemoval(ordinal) {
430
535
  if (table !== null) pendingRemovals.push(ordinal);
431
536
  },
537
+ tableOfEveryDocument(analysis) {
538
+ if (table === null || uncounted !== null || !sameFields(table.fields, analysis.fields)) return null;
539
+ return table;
540
+ },
541
+ countKeptSpellings(counts, analysis, mergedOrdinals) {
542
+ if (table === null || !sameFields(table.fields, analysis.fields) || correctionsAgainstTable(mergedOrdinals) > keptCount(mergedOrdinals)) {
543
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
544
+ if (mergedOrdinals[ordinal] >= 0) countDocumentSpellings(counts, documents.documentAt(ordinal), analysis, 1);
545
+ }
546
+ return;
547
+ }
548
+ const corrected = /* @__PURE__ */ new Map();
549
+ countTableSpellings(corrected, table, tokenTable);
550
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
551
+ const kept = mergedOrdinals[ordinal] >= 0;
552
+ if (kept === isCounted(ordinal)) continue;
553
+ countDocumentSpellings(corrected, documents.documentAt(ordinal), analysis, kept ? 1 : -1);
554
+ }
555
+ addCounts(counts, corrected);
556
+ },
432
557
  spellings() {
433
558
  return null;
434
559
  },
@@ -444,94 +569,127 @@ function createGrowingPartialWordSlot(view) {
444
569
  let fields = [];
445
570
  let everyDocumentPending = false;
446
571
  const pending = /* @__PURE__ */ new Set();
447
- const counts = /* @__PURE__ */ new Map();
448
572
  const entriesByToken = /* @__PURE__ */ new Map();
449
573
  const entrySpellings = [];
450
574
  const entryTokens = [];
451
575
  const entryLengths = [];
576
+ const entryCounts = [];
577
+ const entryInRuns = [];
452
578
  const freeEntries = [];
579
+ const entriesOutsideRuns = [];
453
580
  const runs = /* @__PURE__ */ new Map();
454
581
  let liveEntries = 0;
455
582
  let runMembers = 0;
456
- let countedSpellings = 0;
457
583
  function resetSpellings() {
458
584
  pending.clear();
459
- counts.clear();
460
- countedSpellings = 0;
461
585
  entriesByToken.clear();
462
586
  entrySpellings.length = 0;
463
587
  entryTokens.length = 0;
464
588
  entryLengths.length = 0;
589
+ entryCounts.length = 0;
590
+ entryInRuns.length = 0;
465
591
  freeEntries.length = 0;
592
+ entriesOutsideRuns.length = 0;
466
593
  runs.clear();
467
594
  liveEntries = 0;
468
595
  runMembers = 0;
469
596
  }
470
- function addEntry(token, spelling) {
471
- const points = codePointsOf(spelling);
472
- if (points.length <= PARTIAL_WORD_RUN_CODE_POINTS) return;
597
+ function addEntry(byToken, token, spelling, count) {
598
+ const length = codePointCountOf(spelling);
599
+ if (length <= PARTIAL_WORD_RUN_CODE_POINTS) return;
473
600
  const entry = freeEntries.pop() ?? entrySpellings.length;
474
601
  entrySpellings[entry] = spelling;
475
602
  entryTokens[entry] = token;
476
- entryLengths[entry] = points.length;
477
- let byToken = entriesByToken.get(token);
478
- if (byToken === void 0) {
479
- byToken = /* @__PURE__ */ new Map();
480
- entriesByToken.set(token, byToken);
603
+ entryLengths[entry] = length;
604
+ entryCounts[entry] = count;
605
+ entryInRuns[entry] = false;
606
+ entriesOutsideRuns.push(entry);
607
+ if (entriesOutsideRuns.length > GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY * entrySpellings.length) {
608
+ dropFreedAndRepeatedPendingEntries();
481
609
  }
482
610
  byToken.set(spelling, entry);
483
611
  liveEntries++;
612
+ }
613
+ function dropFreedAndRepeatedPendingEntries() {
614
+ let kept = 0;
615
+ for (const entry of entriesOutsideRuns) {
616
+ if (entryLengths[entry] === 0 || entryInRuns[entry]) continue;
617
+ entryInRuns[entry] = true;
618
+ entriesOutsideRuns[kept++] = entry;
619
+ }
620
+ entriesOutsideRuns.length = kept;
621
+ for (let at = 0; at < kept; at++) entryInRuns[entriesOutsideRuns[at]] = false;
622
+ }
623
+ function addToRuns(entry) {
624
+ const points = codePointsOf(entrySpellings[entry]);
484
625
  for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
485
- const key = runKeyAt(points, at);
626
+ const key = packedRunKeyAt(points, at);
486
627
  let members = runs.get(key);
487
628
  if (members === void 0) {
488
629
  members = /* @__PURE__ */ new Set();
489
630
  runs.set(key, members);
490
631
  }
491
- if (members.has(entry)) continue;
632
+ const size = members.size;
492
633
  members.add(entry);
493
- runMembers++;
634
+ if (members.size !== size) runMembers++;
494
635
  }
636
+ entryInRuns[entry] = true;
495
637
  }
496
- function removeEntry(token, spelling) {
497
- const byToken = entriesByToken.get(token);
498
- const entry = byToken?.get(spelling);
499
- if (byToken === void 0 || entry === void 0) return;
500
- const points = codePointsOf(spelling);
638
+ function addEntriesToRuns() {
639
+ for (const entry of entriesOutsideRuns) {
640
+ if (entryLengths[entry] > 0 && !entryInRuns[entry]) addToRuns(entry);
641
+ }
642
+ entriesOutsideRuns.length = 0;
643
+ }
644
+ function removeFromRuns(entry) {
645
+ const points = codePointsOf(entrySpellings[entry]);
501
646
  for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
502
- const key = runKeyAt(points, at);
647
+ const key = packedRunKeyAt(points, at);
503
648
  const members = runs.get(key);
504
649
  if (members === void 0 || !members.delete(entry)) continue;
505
650
  runMembers--;
506
651
  if (members.size === 0) runs.delete(key);
507
652
  }
653
+ entryInRuns[entry] = false;
654
+ }
655
+ function removeEntry(byToken, token, spelling, entry) {
656
+ if (entryInRuns[entry]) removeFromRuns(entry);
508
657
  byToken.delete(spelling);
509
658
  if (byToken.size === 0) entriesByToken.delete(token);
510
659
  entrySpellings[entry] = "";
511
660
  entryTokens[entry] = "";
512
661
  entryLengths[entry] = 0;
662
+ entryCounts[entry] = 0;
513
663
  freeEntries.push(entry);
514
664
  liveEntries--;
515
665
  }
516
666
  function change(token, spelling, delta) {
517
- let byToken = counts.get(token);
518
- const before = byToken?.get(spelling) ?? 0;
519
- const after = Math.max(0, before + delta);
520
- if (after === before) return;
667
+ let byToken = entriesByToken.get(token);
668
+ const entry = byToken?.get(spelling);
669
+ if (byToken !== void 0 && entry !== void 0) {
670
+ const after = entryCounts[entry] + delta;
671
+ if (after > 0) entryCounts[entry] = after;
672
+ else removeEntry(byToken, token, spelling, entry);
673
+ return;
674
+ }
675
+ if (delta <= 0) return;
521
676
  if (byToken === void 0) {
522
677
  byToken = /* @__PURE__ */ new Map();
523
- counts.set(token, byToken);
678
+ entriesByToken.set(token, byToken);
524
679
  }
525
- if (after > 0) byToken.set(spelling, after);
526
- else byToken.delete(spelling);
527
- if (byToken.size === 0) counts.delete(token);
528
- if (before === 0) {
529
- countedSpellings++;
530
- addEntry(token, spelling);
531
- } else if (after === 0) {
532
- countedSpellings--;
533
- removeEntry(token, spelling);
680
+ addEntry(byToken, token, spelling, delta);
681
+ if (byToken.size === 0) entriesByToken.delete(token);
682
+ }
683
+ function encodedSpellings() {
684
+ const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
685
+ for (const [token, byToken] of entriesByToken) {
686
+ for (const [spelling, entry] of byToken) {
687
+ encoded.tokens.push(token);
688
+ encoded.spellings.push(spelling);
689
+ encoded.counts.push(entryCounts[entry]);
690
+ }
534
691
  }
692
+ return encoded;
535
693
  }
536
694
  function applyCounts(source, direction) {
537
695
  for (const [token, byToken] of source) {
@@ -572,7 +730,7 @@ function createGrowingPartialWordSlot(view) {
572
730
  if (points.length < PARTIAL_WORD_RUN_CODE_POINTS) return;
573
731
  let narrowest = null;
574
732
  for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
575
- const members = runs.get(runKeyAt(points, at));
733
+ const members = runs.get(packedRunKeyAt(points, at));
576
734
  if (members === void 0) return;
577
735
  if (narrowest === null || members.size < narrowest.size) narrowest = members;
578
736
  }
@@ -593,11 +751,12 @@ function createGrowingPartialWordSlot(view) {
593
751
  if (analysis.fields.length === 0) return null;
594
752
  adoptFields(analysis.fields);
595
753
  resolvePending(analysis);
754
+ addEntriesToRuns();
596
755
  return reader;
597
756
  },
598
757
  spellings() {
599
758
  if (fields.length === 0 || everyDocumentPending || pending.size > 0) return null;
600
- return encodePartialSpellings(fields, counts);
759
+ return encodedSpellings();
601
760
  },
602
761
  afterDocument(documentFields, internalId, spellings, direction) {
603
762
  if (documentFields.length === 0) return;
@@ -641,7 +800,7 @@ function createGrowingPartialWordSlot(view) {
641
800
  resetSpellings();
642
801
  },
643
802
  bytes() {
644
- return liveEntries * GROWING_PARTIAL_ENTRY_BYTES + runs.size * GROWING_PARTIAL_RUN_BYTES + runMembers * GROWING_PARTIAL_RUN_MEMBER_BYTES + countedSpellings * GROWING_PARTIAL_COUNTED_SPELLING_BYTES;
803
+ return liveEntries * GROWING_PARTIAL_ENTRY_BYTES + runs.size * GROWING_PARTIAL_RUN_BYTES + runMembers * GROWING_PARTIAL_RUN_MEMBER_BYTES + entriesOutsideRuns.length * GROWING_PARTIAL_PENDING_ENTRY_BYTES;
645
804
  }
646
805
  };
647
806
  return slot;
@@ -656,4 +815,4 @@ var partialWordSearchModule = {
656
815
  expand: expandPartialWords
657
816
  };
658
817
 
659
- export { codePointOrder, countSpelling, encodePartialSpellings, encodeStringBlob, expandTermPrefix, partialWordSearchModule, partitionDocumentFrequency, spellingCountsOf, suggestDisplayTerms };
818
+ export { PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, codePointOrder, countSpelling, encodeLookupKey, encodePartialSpellings, encodeStringBlob, expandTermPrefix, partialWordSearchModule, partitionDocumentFrequency, sameFields, spellingCountsOf, suggestDisplayTerms };
@@ -1,4 +1,4 @@
1
- import { isWordIndexedBase, flattenSchema } from './chunk-356UOTK2.mjs';
1
+ import { isWordIndexedBase, flattenSchema } from './chunk-PXDQFYD5.mjs';
2
2
  import { createNarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
3
3
 
4
4
  // src/core/tokenizer/constants.ts
@@ -409,53 +409,6 @@ function tokenize(text, language, options) {
409
409
  });
410
410
  return { tokens, originalTokens, surfaces };
411
411
  }
412
- function* tokenizeIterator(text, language, options) {
413
- const {
414
- stem = true,
415
- removeStopWords = true,
416
- removeDiacritics = false,
417
- collectSurfaces = false,
418
- customTokenizer,
419
- stopWordOverride
420
- } = options ?? {};
421
- if (customTokenizer) {
422
- const customResult = customTokenizer.tokenize(text);
423
- for (const entry of customResult) {
424
- yield entry;
425
- }
426
- return;
427
- }
428
- const normalized = normalizeForSplitting(text);
429
- const rawParts = splitText(normalized, language);
430
- const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
431
- const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
432
- const stemsTokens = stem && language.stemmer !== null;
433
- const wantSurfaces = collectSurfaces && stemsTokens;
434
- const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
435
- const stripPossessives = language.tokenizer?.stripPossessive ?? false;
436
- let position = 0;
437
- for (const part of rawParts) {
438
- const candidate = stripPossessives ? stripPossessive(part) : part;
439
- if (candidate.length < minLength) {
440
- position++;
441
- continue;
442
- }
443
- if (stopWords.has(candidate)) {
444
- position++;
445
- continue;
446
- }
447
- const processed = transformToken(candidate, language, stem, effectiveDiacritics);
448
- if (processed.length > 0) {
449
- if (wantSurfaces) {
450
- const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
451
- yield { token: processed, position, surface: surface === processed ? void 0 : surface };
452
- } else {
453
- yield { token: processed, position };
454
- }
455
- }
456
- position++;
457
- }
458
- }
459
412
 
460
413
  // src/core/partition/utils.ts
461
414
  function getOrCreateFieldNameIndex(table, fieldName) {
@@ -529,4 +482,4 @@ function getFieldValueByInternalId(docStore, internalId, fieldPath) {
529
482
  return getFieldValueForDoc(docStore, externalId, fieldPath);
530
483
  }
531
484
 
532
- export { clearNormalizationCache, configureNormalizationCache, getFieldValueByInternalId, getFieldValueForDoc, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, resetNormalizationCache, textFieldsChanged, tokenize, tokenizeIterator, tokenizeOptions, visitTokens };
485
+ export { clearNormalizationCache, configureNormalizationCache, getFieldValueByInternalId, getFieldValueForDoc, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, resetNormalizationCache, textFieldsChanged, tokenize, tokenizeOptions, visitTokens };
@@ -1,4 +1,4 @@
1
- import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-356UOTK2.mjs';
1
+ import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-PXDQFYD5.mjs';
2
2
  import { ErrorCodes, NarsilError } from './chunk-I64LXKO3.mjs';
3
3
 
4
4
  // src/narsil/internals.ts
@@ -1,4 +1,4 @@
1
- import { MAX_FILTER_DEPTH, MAX_SORT_FIELDS } from './chunk-V3B24XR7.mjs';
1
+ import { MAX_FILTER_DEPTH, MAX_SORT_FIELDS } from './chunk-IOZPBNSG.mjs';
2
2
  import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-2E35UM3G.mjs';
3
3
  import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-I64LXKO3.mjs';
4
4
  import { mkdtemp, open, rm } from 'fs/promises';
@@ -1,15 +1,15 @@
1
- export { createDirectExecutor } from './chunk-DFFQOA7S.mjs';
2
- import './chunk-D4EF7J3I.mjs';
1
+ export { createDirectExecutor } from './chunk-DFW3U5UW.mjs';
2
+ import './chunk-P2BEMMOD.mjs';
3
3
  import './chunk-4MQ5F4IL.mjs';
4
- import './chunk-V3B24XR7.mjs';
4
+ import './chunk-IOZPBNSG.mjs';
5
5
  import './chunk-TGP5M5R7.mjs';
6
6
  import './chunk-AMGDG4ZJ.mjs';
7
7
  import './chunk-ZJ4CSAB2.mjs';
8
8
  import './chunk-2E35UM3G.mjs';
9
9
  import './chunk-4344I4MR.mjs';
10
- import './chunk-PJU6EF22.mjs';
11
- import './chunk-NO3ABFLE.mjs';
12
- import './chunk-356UOTK2.mjs';
10
+ import './chunk-RTR44V2A.mjs';
11
+ import './chunk-WFVOBZS4.mjs';
12
+ import './chunk-PXDQFYD5.mjs';
13
13
  import './chunk-KEHK2O6P.mjs';
14
14
  import './chunk-GES7U2PM.mjs';
15
15
  import './chunk-U2BKYEC7.mjs';