@delali/narsil 0.3.1-next.87 → 0.3.1-next.88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
  2. package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
  3. package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
  4. package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
  5. package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
  6. package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
  7. package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
  8. package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
  9. package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
  10. package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
  11. package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
  12. package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
  13. package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
  14. package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
  15. package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
  16. package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
  17. package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
  18. package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
  19. package/dist/distribution.mjs +14 -27
  20. package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
  21. package/dist/index.browser.mjs +1 -1
  22. package/dist/index.mjs +5 -5
  23. package/dist/partial-words.mjs +1 -1
  24. package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
  25. package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
  26. package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
  27. package/dist/workers/entry.mjs +3 -3
  28. package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
  29. package/package.json +13 -8
@@ -1,18 +1,19 @@
1
- import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-SYRFSF6V.mjs';
1
+ import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, copyVectorsInto, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, projectIncludedPaths, readProjectedPath, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-X4SEXL4U.mjs';
2
2
  import { getLanguage } from './chunk-PR4O7CAP.mjs';
3
3
  import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-T6CNO72J.mjs';
4
4
  import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
5
- import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, getFieldValueByInternalId, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-PSUJCOSE.mjs';
5
+ import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION, visitDocumentSurfaceSpellings, visitDocumentSpellings, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-A3UB2MXL.mjs';
6
6
  import { schemaFieldsOf, partialFieldsOf, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-CVQWQVFX.mjs';
7
7
  import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
8
8
  import { isWordIndexedBase, parsedTypeOf, isListBase } from './chunk-BVOJIYRV.mjs';
9
- import { toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-R6A773WK.mjs';
9
+ import { toReducedSortValue, defaultSortMode, readSortField, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-R6A773WK.mjs';
10
10
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
11
11
  import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-R7OSDAFK.mjs';
12
+ import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
12
13
  import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
13
14
  import { totalmem } from 'os';
14
15
  import { getHeapStatistics } from 'v8';
15
- import { decode } from '@msgpack/msgpack';
16
+ import { Decoder, decode } from '@msgpack/msgpack';
16
17
 
17
18
  var MEGABYTE_BYTES = 1048576;
18
19
  var IN_MEGABYTES = /^--max-old-space-size=(\d+)$/;
@@ -145,7 +146,7 @@ function sharesABiggerBuffer(value) {
145
146
  return false;
146
147
  }
147
148
  function withOwnBuffers(value) {
148
- if (value === null || typeof value !== "object") return value;
149
+ if (value === null || typeof value !== "object" || value instanceof Date) return value;
149
150
  if (ArrayBuffer.isView(value)) return detachedView(value);
150
151
  if (Array.isArray(value)) {
151
152
  const copy2 = new Array(value.length);
@@ -170,6 +171,9 @@ function documentWithOwnBuffers(document) {
170
171
  if (!sharesABiggerBuffer(document)) return document;
171
172
  return withOwnBuffers(document);
172
173
  }
174
+ function valueWithOwnBuffers(value) {
175
+ return sharesABiggerBuffer(value) ? withOwnBuffers(value) : value;
176
+ }
173
177
  var STRING_HEADER_BYTES = 16;
174
178
  var TWO_BYTE_SAMPLE_LIMIT = 16;
175
179
  var NUMBER_BYTES = 16;
@@ -367,337 +371,926 @@ function createDocumentStore() {
367
371
  },
368
372
  resolver() {
369
373
  return idResolver;
374
+ },
375
+ fieldReader(fieldPath) {
376
+ return (internalId) => {
377
+ const docId = reverseMap[internalId];
378
+ const stored = docId === void 0 ? void 0 : docs.get(docId);
379
+ return stored === void 0 ? void 0 : getNestedValue(stored.fields, fieldPath);
380
+ };
381
+ },
382
+ projectedDocument(docId, projection) {
383
+ const stored = docs.get(docId);
384
+ return stored === void 0 ? void 0 : cloneProjected(stored.fields, projection);
370
385
  }
371
386
  };
372
387
  }
373
388
 
374
389
  // src/core/constants.ts
375
- var POSTING_LIST_INITIAL_CAPACITY = 8;
376
390
  var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
377
391
  var MAX_TERM_FREQUENCY = 65535;
378
392
 
379
- // src/core/posting-list.ts
380
- function createPostingList() {
381
- return {
382
- length: 0,
383
- docIds: [],
384
- termFrequencies: new Uint16Array(POSTING_LIST_INITIAL_CAPACITY),
385
- fieldNameIndices: new Uint16Array(POSTING_LIST_INITIAL_CAPACITY),
386
- positions: null,
387
- docIdSet: /* @__PURE__ */ new Set(),
388
- deletedDocs: /* @__PURE__ */ new Set(),
389
- totalTermFrequency: 0,
390
- structureRevision: 0,
391
- ordered: true
392
- };
393
- }
394
- function growTypedArrays(list) {
395
- const newCap = list.termFrequencies.length * 2;
396
- const newTF = new Uint16Array(newCap);
397
- newTF.set(list.termFrequencies);
398
- list.termFrequencies = newTF;
399
- const newFNI = new Uint16Array(newCap);
400
- newFNI.set(list.fieldNameIndices);
401
- list.fieldNameIndices = newFNI;
402
- }
403
- function compactList(list) {
404
- if (list.deletedDocs.size === 0) return;
405
- list.structureRevision++;
406
- let writeIdx = 0;
407
- let ascending = true;
408
- for (let i = 0; i < list.length; i++) {
409
- if (!list.deletedDocs.has(list.docIds[i])) {
410
- if (writeIdx > 0 && list.docIds[i] < list.docIds[writeIdx - 1]) ascending = false;
411
- if (writeIdx !== i) {
412
- list.docIds[writeIdx] = list.docIds[i];
413
- list.termFrequencies[writeIdx] = list.termFrequencies[i];
414
- list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
415
- if (list.positions) {
416
- list.positions[writeIdx] = list.positions[i];
417
- }
418
- }
419
- writeIdx++;
420
- }
421
- }
422
- list.docIds.length = writeIdx;
423
- if (list.positions) list.positions.length = writeIdx;
424
- list.length = writeIdx;
425
- list.deletedDocs.clear();
426
- list.ordered = ascending;
427
- }
428
- function compactDocEntries(list, internalId) {
429
- list.structureRevision++;
430
- let writeIdx = 0;
431
- let ascending = true;
432
- for (let i = 0; i < list.length; i++) {
433
- if (list.docIds[i] !== internalId) {
434
- if (writeIdx > 0 && list.docIds[i] < list.docIds[writeIdx - 1]) ascending = false;
435
- if (writeIdx !== i) {
436
- list.docIds[writeIdx] = list.docIds[i];
437
- list.termFrequencies[writeIdx] = list.termFrequencies[i];
438
- list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
439
- if (list.positions) {
440
- list.positions[writeIdx] = list.positions[i];
441
- }
442
- }
443
- writeIdx++;
444
- }
393
+ // src/core/inverted-index/constants.ts
394
+ var TERM_TABLE_START_CAPACITY = 1 << 10;
395
+ var POSTING_POOL_START_ENTRIES = 1 << 12;
396
+ var POSITION_POOL_START_VALUES = 1 << 12;
397
+ var RETIRED_DOCUMENTS_START_CAPACITY = 1 << 10;
398
+ var SLICE_GROWTH_FACTOR = 1.5;
399
+ var POOL_GROWTH_FACTOR = 1.5;
400
+ var POOL_VACUUM_FREE_FRACTION = 0.25;
401
+ var CACHED_VIEW_MIN_ENTRIES = 128;
402
+ var TERM_VACUUM_MIN_DEAD_TERMS = 1 << 10;
403
+ var POOL_VACUUM_MIN_FREE_ENTRIES = 1 << 14;
404
+ var POSITION_VACUUM_MIN_FREE_VALUES = 1 << 14;
405
+ var TERM_ENTRY_ESTIMATED_BYTES = 64;
406
+ var TOKEN_STRING_HEADER_BYTES = 16;
407
+
408
+ // src/core/inverted-index/pools.ts
409
+ var NO_SLICE = 255;
410
+ var EMPTY_POSITIONS = 0;
411
+ var MAX_POOL_ENTRIES = 2 ** 32 - 1;
412
+ var SLICE_CAPACITIES = (() => {
413
+ const capacities = [1];
414
+ while (capacities[capacities.length - 1] < MAX_POOL_ENTRIES) {
415
+ const previous = capacities[capacities.length - 1];
416
+ capacities.push(Math.min(MAX_POOL_ENTRIES, Math.max(previous + 1, Math.ceil(previous * SLICE_GROWTH_FACTOR))));
417
+ }
418
+ return Uint32Array.from(capacities);
419
+ })();
420
+ function sliceClassFor(entries) {
421
+ let low = 0;
422
+ let high = SLICE_CAPACITIES.length - 1;
423
+ while (low < high) {
424
+ const middle = low + high >>> 1;
425
+ if (SLICE_CAPACITIES[middle] < entries) low = middle + 1;
426
+ else high = middle;
427
+ }
428
+ return low;
429
+ }
430
+ function sliceCapacity(sliceClass) {
431
+ return sliceClass === NO_SLICE ? 0 : SLICE_CAPACITIES[sliceClass];
432
+ }
433
+ function grownCapacity(current, needed) {
434
+ return Math.min(MAX_POOL_ENTRIES, Math.max(needed, Math.ceil(current * POOL_GROWTH_FACTOR)));
435
+ }
436
+ var PostingPool = class {
437
+ docIds;
438
+ frequencies;
439
+ fieldIndices;
440
+ positionStarts = null;
441
+ end = 0;
442
+ epoch = 0;
443
+ freeEntries = 0;
444
+ freeLists = [];
445
+ constructor(capacity) {
446
+ this.docIds = new Uint32Array(capacity);
447
+ this.frequencies = new Uint16Array(capacity);
448
+ this.fieldIndices = new Uint16Array(capacity);
449
+ }
450
+ allocate(sliceClass) {
451
+ const size = sliceCapacity(sliceClass);
452
+ const free = this.freeLists[sliceClass];
453
+ if (free !== void 0) {
454
+ const reused = free.pop();
455
+ if (reused !== void 0) {
456
+ this.freeEntries -= size;
457
+ return reused;
458
+ }
459
+ }
460
+ const start = this.end;
461
+ this.reserve(start + size);
462
+ this.end = start + size;
463
+ return start;
464
+ }
465
+ release(start, sliceClass) {
466
+ if (sliceClass === NO_SLICE) return;
467
+ let free = this.freeLists[sliceClass];
468
+ if (free === void 0) {
469
+ free = [];
470
+ this.freeLists[sliceClass] = free;
471
+ }
472
+ free.push(start);
473
+ this.freeEntries += sliceCapacity(sliceClass);
474
+ }
475
+ trackPositions() {
476
+ if (this.positionStarts === null) {
477
+ this.positionStarts = new Uint32Array(this.docIds.length);
478
+ this.epoch++;
479
+ }
480
+ return this.positionStarts;
481
+ }
482
+ move(from, to, count) {
483
+ this.docIds.copyWithin(to, from, from + count);
484
+ this.frequencies.copyWithin(to, from, from + count);
485
+ this.fieldIndices.copyWithin(to, from, from + count);
486
+ if (this.positionStarts !== null) this.positionStarts.copyWithin(to, from, from + count);
487
+ }
488
+ byteLength() {
489
+ const positionBytes = this.positionStarts === null ? 0 : this.positionStarts.byteLength;
490
+ return this.docIds.byteLength + this.frequencies.byteLength + this.fieldIndices.byteLength + positionBytes;
491
+ }
492
+ reserve(needed) {
493
+ if (needed <= this.docIds.length) return;
494
+ const capacity = grownCapacity(this.docIds.length, needed);
495
+ this.docIds = grownCopy(this.docIds, new Uint32Array(capacity), this.end);
496
+ this.frequencies = grownCopy(this.frequencies, new Uint16Array(capacity), this.end);
497
+ this.fieldIndices = grownCopy(this.fieldIndices, new Uint16Array(capacity), this.end);
498
+ if (this.positionStarts !== null) {
499
+ this.positionStarts = grownCopy(this.positionStarts, new Uint32Array(capacity), this.end);
500
+ }
501
+ this.epoch++;
445
502
  }
446
- list.docIds.length = writeIdx;
447
- if (list.positions) list.positions.length = writeIdx;
448
- list.length = writeIdx;
449
- list.ordered = ascending;
450
- }
503
+ };
504
+ function grownCopy(source, target, used) {
505
+ target.set(source.subarray(0, used));
506
+ return target;
507
+ }
508
+ var PositionPool = class {
509
+ values;
510
+ end = 1;
511
+ freeValues = 0;
512
+ constructor(capacity) {
513
+ this.values = new Uint32Array(Math.max(1, capacity));
514
+ }
515
+ append(positions) {
516
+ const count = positions.length;
517
+ if (count === 0) return EMPTY_POSITIONS;
518
+ const start = this.end;
519
+ this.reserve(start + 1 + count);
520
+ const values = this.values;
521
+ values[start] = count;
522
+ for (let i = 0; i < count; i++) values[start + 1 + i] = positions[i];
523
+ this.end = start + 1 + count;
524
+ return start;
525
+ }
526
+ appendFrom(source, start) {
527
+ if (start === EMPTY_POSITIONS) return EMPTY_POSITIONS;
528
+ const count = source.values[start];
529
+ const target = this.end;
530
+ this.reserve(target + 1 + count);
531
+ this.values.set(source.values.subarray(start, start + 1 + count), target);
532
+ this.end = target + 1 + count;
533
+ return target;
534
+ }
535
+ countAt(start) {
536
+ return this.values[start];
537
+ }
538
+ read(start) {
539
+ const count = this.values[start];
540
+ const out = new Array(count);
541
+ for (let i = 0; i < count; i++) out[i] = this.values[start + 1 + i];
542
+ return out;
543
+ }
544
+ copyInto(start, target, offset) {
545
+ const count = this.values[start];
546
+ if (count > 0) target.set(this.values.subarray(start + 1, start + 1 + count), offset);
547
+ return count;
548
+ }
549
+ release(start) {
550
+ if (start === EMPTY_POSITIONS) return;
551
+ this.freeValues += this.values[start] + 1;
552
+ }
553
+ reserve(needed) {
554
+ if (needed <= this.values.length) return;
555
+ const capacity = grownCapacity(this.values.length, needed);
556
+ const grown = new Uint32Array(capacity);
557
+ grown.set(this.values.subarray(0, this.end));
558
+ this.values = grown;
559
+ }
560
+ };
451
561
 
452
- // src/core/inverted-index.ts
453
- function createInvertedIndex(fieldNameTable) {
454
- const index = /* @__PURE__ */ new Map();
455
- const charBuckets = /* @__PURE__ */ new Map();
456
- let batchMode = false;
457
- const batchDirtyTokens = /* @__PURE__ */ new Set();
458
- function trackToken(token) {
562
+ // src/core/inverted-index/terms.ts
563
+ var TERM_LIVE = 1;
564
+ var TERM_ORDERED = 2;
565
+ var TERM_POSITIONS = 4;
566
+ function grow(source, target) {
567
+ target.set(source);
568
+ return target;
569
+ }
570
+ var TermColumns = class _TermColumns {
571
+ constructor(startCapacity) {
572
+ this.startCapacity = startCapacity;
573
+ this.sliceStart = new Uint32Array(startCapacity);
574
+ this.sliceClass = new Uint8Array(startCapacity);
575
+ this.lengths = new Uint32Array(startCapacity);
576
+ this.docFrequency = new Uint32Array(startCapacity);
577
+ this.deadDocuments = new Uint32Array(startCapacity);
578
+ this.totalTermFrequency = new Float64Array(startCapacity);
579
+ this.revision = new Uint32Array(startCapacity);
580
+ this.flags = new Uint8Array(startCapacity);
581
+ }
582
+ ids = /* @__PURE__ */ new Map();
583
+ tokens = [];
584
+ sliceStart;
585
+ sliceClass;
586
+ lengths;
587
+ docFrequency;
588
+ deadDocuments;
589
+ totalTermFrequency;
590
+ revision;
591
+ flags;
592
+ count = 0;
593
+ liveCount = 0;
594
+ generation = 0;
595
+ tokenCodeUnits = 0;
596
+ buckets = /* @__PURE__ */ new Map();
597
+ get deadCount() {
598
+ return this.count - this.liveCount;
599
+ }
600
+ add(token) {
601
+ const ordinal = this.count;
602
+ if (ordinal === this.flags.length) this.growTo(ordinal * 2);
603
+ this.count = ordinal + 1;
604
+ this.liveCount++;
605
+ this.tokenCodeUnits += token.length;
606
+ this.ids.set(token, ordinal);
607
+ this.tokens.push(token);
608
+ this.sliceClass[ordinal] = NO_SLICE;
609
+ this.flags[ordinal] = TERM_LIVE | TERM_ORDERED;
610
+ this.track(token, ordinal);
611
+ return ordinal;
612
+ }
613
+ kill(ordinal) {
614
+ const token = this.tokens[ordinal];
615
+ this.ids.delete(token);
616
+ const bucket = token.length > 0 ? this.buckets.get(token[0]) : void 0;
617
+ if (bucket !== void 0) bucket.dead++;
618
+ this.tokens[ordinal] = "";
619
+ this.flags[ordinal] = 0;
620
+ this.lengths[ordinal] = 0;
621
+ this.docFrequency[ordinal] = 0;
622
+ this.deadDocuments[ordinal] = 0;
623
+ this.totalTermFrequency[ordinal] = 0;
624
+ this.sliceClass[ordinal] = NO_SLICE;
625
+ this.liveCount--;
626
+ this.tokenCodeUnits -= token.length;
627
+ }
628
+ isLive(ordinal) {
629
+ return (this.flags[ordinal] & TERM_LIVE) !== 0;
630
+ }
631
+ forEachLive(visit) {
632
+ const flags = this.flags;
633
+ for (let ordinal = 0; ordinal < this.count; ordinal++) {
634
+ if ((flags[ordinal] & TERM_LIVE) !== 0) visit(ordinal);
635
+ }
636
+ }
637
+ forEachStartingWith(prefix, visit) {
638
+ if (prefix.length === 0) {
639
+ this.forEachLive(visit);
640
+ return;
641
+ }
642
+ const bucket = this.buckets.get(prefix[0]);
643
+ if (bucket === void 0) return;
644
+ this.compactBucket(prefix[0], bucket);
645
+ const { ordinals, length } = bucket;
646
+ const flags = this.flags;
647
+ const tokens = this.tokens;
648
+ const checkPrefix = prefix.length > 1;
649
+ for (let i = 0; i < length; i++) {
650
+ const ordinal = ordinals[i];
651
+ if ((flags[ordinal] & TERM_LIVE) === 0) continue;
652
+ if (checkPrefix && !tokens[ordinal].startsWith(prefix)) continue;
653
+ visit(ordinal);
654
+ }
655
+ }
656
+ renumber() {
657
+ const remap = new Int32Array(this.count).fill(-1);
658
+ const capacity = Math.max(this.startCapacity, this.liveCount);
659
+ const next = new _TermColumns(capacity);
660
+ for (let ordinal = 0; ordinal < this.count; ordinal++) {
661
+ if ((this.flags[ordinal] & TERM_LIVE) === 0) continue;
662
+ const target = next.add(this.tokens[ordinal]);
663
+ remap[ordinal] = target;
664
+ next.sliceStart[target] = this.sliceStart[ordinal];
665
+ next.sliceClass[target] = this.sliceClass[ordinal];
666
+ next.lengths[target] = this.lengths[ordinal];
667
+ next.docFrequency[target] = this.docFrequency[ordinal];
668
+ next.deadDocuments[target] = this.deadDocuments[ordinal];
669
+ next.totalTermFrequency[target] = this.totalTermFrequency[ordinal];
670
+ next.revision[target] = this.revision[ordinal];
671
+ next.flags[target] = this.flags[ordinal];
672
+ }
673
+ this.ids = next.ids;
674
+ this.tokens = next.tokens;
675
+ this.sliceStart = next.sliceStart;
676
+ this.sliceClass = next.sliceClass;
677
+ this.lengths = next.lengths;
678
+ this.docFrequency = next.docFrequency;
679
+ this.deadDocuments = next.deadDocuments;
680
+ this.totalTermFrequency = next.totalTermFrequency;
681
+ this.revision = next.revision;
682
+ this.flags = next.flags;
683
+ this.buckets = next.buckets;
684
+ this.count = next.count;
685
+ this.liveCount = next.liveCount;
686
+ this.generation++;
687
+ return remap;
688
+ }
689
+ clear() {
690
+ this.ids = /* @__PURE__ */ new Map();
691
+ this.tokens = [];
692
+ this.buckets = /* @__PURE__ */ new Map();
693
+ this.count = 0;
694
+ this.liveCount = 0;
695
+ this.tokenCodeUnits = 0;
696
+ this.generation++;
697
+ this.sliceStart = new Uint32Array(this.startCapacity);
698
+ this.sliceClass = new Uint8Array(this.startCapacity);
699
+ this.lengths = new Uint32Array(this.startCapacity);
700
+ this.docFrequency = new Uint32Array(this.startCapacity);
701
+ this.deadDocuments = new Uint32Array(this.startCapacity);
702
+ this.totalTermFrequency = new Float64Array(this.startCapacity);
703
+ this.revision = new Uint32Array(this.startCapacity);
704
+ this.flags = new Uint8Array(this.startCapacity);
705
+ }
706
+ byteLength() {
707
+ let bucketBytes = 0;
708
+ for (const bucket of this.buckets.values()) bucketBytes += bucket.ordinals.byteLength;
709
+ return this.sliceStart.byteLength + this.sliceClass.byteLength + this.lengths.byteLength + this.docFrequency.byteLength + this.deadDocuments.byteLength + this.totalTermFrequency.byteLength + this.revision.byteLength + this.flags.byteLength + bucketBytes;
710
+ }
711
+ track(token, ordinal) {
459
712
  if (token.length === 0) return;
460
- const ch = token[0];
461
- let bucket = charBuckets.get(ch);
462
- if (!bucket) {
463
- bucket = /* @__PURE__ */ new Set();
464
- charBuckets.set(ch, bucket);
713
+ const first = token[0];
714
+ let bucket = this.buckets.get(first);
715
+ if (bucket === void 0) {
716
+ bucket = { ordinals: new Uint32Array(4), length: 0, dead: 0 };
717
+ this.buckets.set(first, bucket);
718
+ }
719
+ if (bucket.length === bucket.ordinals.length) {
720
+ bucket.ordinals = grow(bucket.ordinals, new Uint32Array(bucket.ordinals.length * 2));
721
+ }
722
+ bucket.ordinals[bucket.length++] = ordinal;
723
+ }
724
+ compactBucket(first, bucket) {
725
+ if (bucket.dead * 2 <= bucket.length) return;
726
+ const { ordinals } = bucket;
727
+ let kept = 0;
728
+ for (let i = 0; i < bucket.length; i++) {
729
+ const ordinal = ordinals[i];
730
+ if ((this.flags[ordinal] & TERM_LIVE) !== 0) ordinals[kept++] = ordinal;
731
+ }
732
+ bucket.length = kept;
733
+ bucket.dead = 0;
734
+ if (kept === 0) this.buckets.delete(first);
735
+ }
736
+ growTo(capacity) {
737
+ this.sliceStart = grow(this.sliceStart, new Uint32Array(capacity));
738
+ this.sliceClass = grow(this.sliceClass, new Uint8Array(capacity));
739
+ this.lengths = grow(this.lengths, new Uint32Array(capacity));
740
+ this.docFrequency = grow(this.docFrequency, new Uint32Array(capacity));
741
+ this.deadDocuments = grow(this.deadDocuments, new Uint32Array(capacity));
742
+ this.totalTermFrequency = grow(this.totalTermFrequency, new Float64Array(capacity));
743
+ this.revision = grow(this.revision, new Uint32Array(capacity));
744
+ this.flags = grow(this.flags, new Uint8Array(capacity));
745
+ }
746
+ };
747
+
748
+ // src/core/inverted-index/view.ts
749
+ var NO_TOMBSTONES = { size: 0, has: () => false };
750
+ var NO_DOCUMENT_IDS = new Uint32Array(0);
751
+ var NO_FREQUENCIES = new Uint16Array(0);
752
+ var DETACHED = -1;
753
+ var LivePostingView = class {
754
+ constructor(store, token, ordinal, keepsPositions) {
755
+ this.store = store;
756
+ this.token = token;
757
+ this.keepsPositions = keepsPositions;
758
+ this.ordinal = ordinal;
759
+ this.generation = store.terms.generation;
760
+ const view = this;
761
+ this.docIdSet = {
762
+ get size() {
763
+ const live = view.sync();
764
+ return live === DETACHED ? 0 : view.store.terms.docFrequency[live];
765
+ }
766
+ };
767
+ this.retiredRows = {
768
+ get size() {
769
+ const live = view.sync();
770
+ return live === DETACHED ? 0 : view.store.terms.deadDocuments[live];
771
+ },
772
+ has: (internalId) => store.isRetired(internalId)
773
+ };
774
+ }
775
+ docIdSet;
776
+ retiredRows;
777
+ ordinal;
778
+ generation;
779
+ syncedEpoch = -1;
780
+ syncedStart = -1;
781
+ syncedLength = 0;
782
+ syncedRevision = -1;
783
+ ids = NO_DOCUMENT_IDS;
784
+ frequencies = NO_FREQUENCIES;
785
+ fields = NO_FREQUENCIES;
786
+ materialized = null;
787
+ get length() {
788
+ this.sync();
789
+ return this.syncedLength;
790
+ }
791
+ get docIds() {
792
+ this.sync();
793
+ return this.ids;
794
+ }
795
+ get termFrequencies() {
796
+ this.sync();
797
+ return this.frequencies;
798
+ }
799
+ get fieldNameIndices() {
800
+ this.sync();
801
+ return this.fields;
802
+ }
803
+ get holdsPositions() {
804
+ const live = this.sync();
805
+ return live !== DETACHED && (this.store.terms.flags[live] & TERM_POSITIONS) !== 0;
806
+ }
807
+ get positions() {
808
+ if (!this.holdsPositions) return null;
809
+ if (this.materialized !== null) return this.materialized;
810
+ const starts = this.store.pool.positionStarts;
811
+ const rows = new Array(this.syncedLength);
812
+ for (let row = 0; row < this.syncedLength; row++) {
813
+ rows[row] = starts === null ? [] : this.store.positions.read(starts[this.syncedStart + row]);
814
+ }
815
+ if (this.keepsPositions) this.materialized = rows;
816
+ return rows;
817
+ }
818
+ positionCountAt(row) {
819
+ if (!this.holdsPositions) return 0;
820
+ const starts = this.store.pool.positionStarts;
821
+ return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
822
+ }
823
+ copyPositionsAt(row, target, offset) {
824
+ if (!this.holdsPositions) return 0;
825
+ const starts = this.store.pool.positionStarts;
826
+ return starts === null ? 0 : this.store.positions.copyInto(starts[this.syncedStart + row], target, offset);
827
+ }
828
+ get deletedDocs() {
829
+ const live = this.sync();
830
+ if (live === DETACHED || this.store.terms.deadDocuments[live] === 0) return NO_TOMBSTONES;
831
+ return this.retiredRows;
832
+ }
833
+ get totalTermFrequency() {
834
+ const live = this.sync();
835
+ return live === DETACHED ? 0 : this.store.terms.totalTermFrequency[live];
836
+ }
837
+ get structureRevision() {
838
+ const live = this.sync();
839
+ return live === DETACHED ? this.syncedRevision : this.store.terms.revision[live];
840
+ }
841
+ get ordered() {
842
+ const live = this.sync();
843
+ return live === DETACHED || (this.store.terms.flags[live] & TERM_ORDERED) !== 0;
844
+ }
845
+ sync() {
846
+ const terms = this.store.terms;
847
+ if (this.generation !== terms.generation) {
848
+ this.ordinal = terms.ids.get(this.token) ?? DETACHED;
849
+ this.generation = terms.generation;
850
+ }
851
+ const ordinal = this.ordinal;
852
+ if (ordinal === DETACHED || !terms.isLive(ordinal)) {
853
+ this.detach();
854
+ return DETACHED;
855
+ }
856
+ const pool = this.store.pool;
857
+ const start = terms.sliceStart[ordinal];
858
+ const length = terms.lengths[ordinal];
859
+ const revision = terms.revision[ordinal];
860
+ if (pool.epoch !== this.syncedEpoch || start !== this.syncedStart || length !== this.syncedLength || revision !== this.syncedRevision) {
861
+ this.ids = pool.docIds.subarray(start, start + length);
862
+ this.frequencies = pool.frequencies.subarray(start, start + length);
863
+ this.fields = pool.fieldIndices.subarray(start, start + length);
864
+ this.materialized = null;
865
+ this.syncedEpoch = pool.epoch;
866
+ this.syncedStart = start;
867
+ this.syncedLength = length;
868
+ this.syncedRevision = revision;
465
869
  }
466
- bucket.add(token);
870
+ return ordinal;
467
871
  }
468
- function untrackToken(token) {
469
- if (token.length === 0) return;
470
- const ch = token[0];
471
- const bucket = charBuckets.get(ch);
472
- if (bucket) {
473
- bucket.delete(token);
474
- if (bucket.size === 0) charBuckets.delete(ch);
872
+ detach() {
873
+ if (this.syncedStart === DETACHED && this.syncedLength === 0) return;
874
+ this.ids = NO_DOCUMENT_IDS;
875
+ this.frequencies = NO_FREQUENCIES;
876
+ this.fields = NO_FREQUENCIES;
877
+ this.materialized = null;
878
+ this.syncedStart = DETACHED;
879
+ this.syncedLength = 0;
880
+ this.syncedEpoch = -1;
881
+ }
882
+ };
883
+
884
+ // src/core/inverted-index/postings.ts
885
+ function holdsDocument(docIds, start, length, internalId, ordered) {
886
+ if (!ordered) {
887
+ for (let i = start; i < start + length; i++) {
888
+ if (docIds[i] === internalId) return true;
475
889
  }
890
+ return false;
476
891
  }
477
- function tokensStartingWith(prefix) {
478
- if (prefix.length === 0) return index.keys();
479
- const bucket = charBuckets.get(prefix[0]);
480
- if (!bucket) return [];
481
- if (prefix.length === 1) return bucket;
482
- const filtered = [];
483
- for (const t of bucket) {
484
- if (t.startsWith(prefix)) {
485
- filtered.push(t);
892
+ let low = start;
893
+ let high = start + length;
894
+ while (low < high) {
895
+ const middle = low + high >>> 1;
896
+ if (docIds[middle] < internalId) low = middle + 1;
897
+ else high = middle;
898
+ }
899
+ return low < start + length && docIds[low] === internalId;
900
+ }
901
+ var LivePostings = class {
902
+ terms = new TermColumns(TERM_TABLE_START_CAPACITY);
903
+ pool = new PostingPool(POSTING_POOL_START_ENTRIES);
904
+ positions = new PositionPool(POSITION_POOL_START_VALUES);
905
+ retired = new Uint8Array(RETIRED_DOCUMENTS_START_CAPACITY);
906
+ batchMode = false;
907
+ dirty = /* @__PURE__ */ new Set();
908
+ cachedViews = /* @__PURE__ */ new Map();
909
+ compactedSlack = 0;
910
+ view(ordinal, cached) {
911
+ if (!cached || this.terms.lengths[ordinal] < CACHED_VIEW_MIN_ENTRIES) {
912
+ return new LivePostingView(this, this.terms.tokens[ordinal], ordinal, cached);
913
+ }
914
+ let view = this.cachedViews.get(ordinal);
915
+ if (view === void 0) {
916
+ view = new LivePostingView(this, this.terms.tokens[ordinal], ordinal, true);
917
+ this.cachedViews.set(ordinal, view);
918
+ }
919
+ return view;
920
+ }
921
+ reserve(token, entries) {
922
+ const terms = this.terms;
923
+ const ordinal = terms.ids.get(token) ?? terms.add(token);
924
+ if (sliceCapacity(terms.sliceClass[ordinal]) < terms.lengths[ordinal] + entries) {
925
+ this.relocate(ordinal, terms.lengths[ordinal] + entries);
926
+ }
927
+ }
928
+ insert(token, internalId, termFrequency, fieldIndex, positions) {
929
+ const terms = this.terms;
930
+ if (this.isRetired(internalId)) this.purgeRetired();
931
+ const ordinal = terms.ids.get(token) ?? terms.add(token);
932
+ const length = terms.lengths[ordinal];
933
+ if (length >= sliceCapacity(terms.sliceClass[ordinal])) this.relocate(ordinal, length + 1);
934
+ const pool = this.pool;
935
+ const start = terms.sliceStart[ordinal];
936
+ const at = start + length;
937
+ const flags = terms.flags[ordinal];
938
+ let newDocument = true;
939
+ if (length > 0) {
940
+ const last = pool.docIds[at - 1];
941
+ const ordered = (flags & TERM_ORDERED) !== 0;
942
+ if (internalId === last) {
943
+ newDocument = false;
944
+ } else if (!ordered || internalId < last) {
945
+ newDocument = !holdsDocument(pool.docIds, start, length, internalId, ordered);
946
+ if (internalId < last) terms.flags[ordinal] = flags & ~TERM_ORDERED;
947
+ }
948
+ }
949
+ const stored = termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : termFrequency;
950
+ pool.docIds[at] = internalId;
951
+ pool.frequencies[at] = stored;
952
+ pool.fieldIndices[at] = fieldIndex;
953
+ terms.totalTermFrequency[ordinal] += stored;
954
+ if (positions !== null) {
955
+ const starts = pool.trackPositions();
956
+ if ((flags & TERM_POSITIONS) === 0) {
957
+ terms.flags[ordinal] |= TERM_POSITIONS;
958
+ starts.fill(EMPTY_POSITIONS, start, at);
959
+ }
960
+ starts[at] = this.positions.append(positions);
961
+ } else if ((flags & TERM_POSITIONS) !== 0) {
962
+ pool.trackPositions()[at] = EMPTY_POSITIONS;
963
+ }
964
+ if (newDocument) terms.docFrequency[ordinal]++;
965
+ terms.lengths[ordinal] = length + 1;
966
+ this.repackIfSparse();
967
+ }
968
+ remove(token, internalId, occurrences) {
969
+ const terms = this.terms;
970
+ const ordinal = terms.ids.get(token);
971
+ if (ordinal === void 0) return;
972
+ const length = terms.lengths[ordinal];
973
+ const ordered = (terms.flags[ordinal] & TERM_ORDERED) !== 0;
974
+ if (!holdsDocument(this.pool.docIds, terms.sliceStart[ordinal], length, internalId, ordered)) return;
975
+ terms.revision[ordinal]++;
976
+ terms.docFrequency[ordinal]--;
977
+ terms.deadDocuments[ordinal]++;
978
+ this.retire(internalId);
979
+ terms.totalTermFrequency[ordinal] = Math.max(0, terms.totalTermFrequency[ordinal] - occurrences);
980
+ if (terms.docFrequency[ordinal] === 0) {
981
+ this.dropTerm(ordinal);
982
+ return;
983
+ }
984
+ if (this.batchMode) {
985
+ this.dirty.add(ordinal);
986
+ return;
987
+ }
988
+ if (terms.deadDocuments[ordinal] / length > POSTING_LIST_COMPACTION_THRESHOLD) this.compact(ordinal);
989
+ this.repackIfSparse();
990
+ }
991
+ isRetired(internalId) {
992
+ return internalId < this.retired.length && this.retired[internalId] === 1;
993
+ }
994
+ beginBatch() {
995
+ this.batchMode = true;
996
+ this.dirty.clear();
997
+ }
998
+ endBatch() {
999
+ this.batchMode = false;
1000
+ const terms = this.terms;
1001
+ for (const ordinal of this.dirty) {
1002
+ if (!terms.isLive(ordinal)) continue;
1003
+ if (terms.deadDocuments[ordinal] / terms.lengths[ordinal] > POSTING_LIST_COMPACTION_THRESHOLD) {
1004
+ this.compact(ordinal);
1005
+ }
1006
+ }
1007
+ this.dirty.clear();
1008
+ this.repackIfSparse();
1009
+ }
1010
+ clear() {
1011
+ const epoch = this.pool.epoch + 1;
1012
+ this.terms.clear();
1013
+ this.pool = new PostingPool(POSTING_POOL_START_ENTRIES);
1014
+ this.pool.epoch = epoch;
1015
+ this.positions = new PositionPool(POSITION_POOL_START_VALUES);
1016
+ this.retired = new Uint8Array(RETIRED_DOCUMENTS_START_CAPACITY);
1017
+ this.dirty.clear();
1018
+ this.cachedViews.clear();
1019
+ this.compactedSlack = 0;
1020
+ }
1021
+ byteLength() {
1022
+ return this.terms.byteLength() + this.pool.byteLength() + this.positions.values.byteLength + this.retired.byteLength;
1023
+ }
1024
+ retire(internalId) {
1025
+ if (internalId >= this.retired.length) {
1026
+ const grown = new Uint8Array(Math.max(internalId + 1, this.retired.length * 2));
1027
+ grown.set(this.retired);
1028
+ this.retired = grown;
1029
+ }
1030
+ this.retired[internalId] = 1;
1031
+ }
1032
+ purgeRetired() {
1033
+ const terms = this.terms;
1034
+ for (let ordinal = 0; ordinal < terms.count; ordinal++) {
1035
+ if (terms.isLive(ordinal) && terms.deadDocuments[ordinal] > 0) this.compact(ordinal);
1036
+ }
1037
+ this.retired.fill(0);
1038
+ }
1039
+ compact(ordinal) {
1040
+ if (this.terms.deadDocuments[ordinal] === 0) return;
1041
+ const retired = this.retired;
1042
+ this.filterEntries(ordinal, (id) => id < retired.length && retired[id] === 1);
1043
+ this.terms.deadDocuments[ordinal] = 0;
1044
+ }
1045
+ filterEntries(ordinal, drop) {
1046
+ const terms = this.terms;
1047
+ const pool = this.pool;
1048
+ const start = terms.sliceStart[ordinal];
1049
+ const length = terms.lengths[ordinal];
1050
+ const starts = (terms.flags[ordinal] & TERM_POSITIONS) !== 0 ? pool.positionStarts : null;
1051
+ const { docIds, frequencies, fieldIndices } = pool;
1052
+ terms.revision[ordinal]++;
1053
+ let kept = 0;
1054
+ let ascending = true;
1055
+ for (let i = start; i < start + length; i++) {
1056
+ const internalId = docIds[i];
1057
+ if (drop(internalId)) {
1058
+ if (starts !== null) this.positions.release(starts[i]);
1059
+ continue;
1060
+ }
1061
+ const target = start + kept;
1062
+ if (kept > 0 && internalId < docIds[target - 1]) ascending = false;
1063
+ if (target !== i) {
1064
+ docIds[target] = internalId;
1065
+ frequencies[target] = frequencies[i];
1066
+ fieldIndices[target] = fieldIndices[i];
1067
+ if (starts !== null) starts[target] = starts[i];
1068
+ }
1069
+ kept++;
1070
+ }
1071
+ terms.lengths[ordinal] = kept;
1072
+ terms.flags[ordinal] = ascending ? terms.flags[ordinal] | TERM_ORDERED : terms.flags[ordinal] & ~TERM_ORDERED;
1073
+ this.compactedSlack += length - kept;
1074
+ }
1075
+ relocate(ordinal, needed) {
1076
+ const terms = this.terms;
1077
+ const previousClass = terms.sliceClass[ordinal];
1078
+ const previousStart = terms.sliceStart[ordinal];
1079
+ const length = terms.lengths[ordinal];
1080
+ const grownClass = previousClass === NO_SLICE ? 0 : previousClass + 1;
1081
+ const nextClass = Math.max(grownClass, sliceClassFor(needed));
1082
+ const nextStart = this.pool.allocate(nextClass);
1083
+ if (length > 0) this.pool.move(previousStart, nextStart, length);
1084
+ this.pool.release(previousStart, previousClass);
1085
+ terms.sliceStart[ordinal] = nextStart;
1086
+ terms.sliceClass[ordinal] = nextClass;
1087
+ }
1088
+ dropTerm(ordinal) {
1089
+ const terms = this.terms;
1090
+ const pool = this.pool;
1091
+ if ((terms.flags[ordinal] & TERM_POSITIONS) !== 0 && pool.positionStarts !== null) {
1092
+ const start = terms.sliceStart[ordinal];
1093
+ for (let i = start; i < start + terms.lengths[ordinal]; i++) this.positions.release(pool.positionStarts[i]);
1094
+ }
1095
+ pool.release(terms.sliceStart[ordinal], terms.sliceClass[ordinal]);
1096
+ this.dirty.delete(ordinal);
1097
+ this.cachedViews.delete(ordinal);
1098
+ terms.kill(ordinal);
1099
+ if (terms.deadCount >= TERM_VACUUM_MIN_DEAD_TERMS && terms.deadCount > terms.liveCount) this.renumberTerms();
1100
+ }
1101
+ renumberTerms() {
1102
+ const remap = this.terms.renumber();
1103
+ const dirty = /* @__PURE__ */ new Set();
1104
+ for (const ordinal of this.dirty) {
1105
+ if (remap[ordinal] >= 0) dirty.add(remap[ordinal]);
1106
+ }
1107
+ this.dirty = dirty;
1108
+ this.cachedViews.clear();
1109
+ }
1110
+ repackIfSparse() {
1111
+ const pool = this.pool;
1112
+ const freeEntries = pool.freeEntries + this.compactedSlack;
1113
+ const sparsePostings = freeEntries >= POOL_VACUUM_MIN_FREE_ENTRIES && freeEntries > pool.end * POOL_VACUUM_FREE_FRACTION;
1114
+ const positions = this.positions;
1115
+ const sparsePositions = positions.freeValues >= POSITION_VACUUM_MIN_FREE_VALUES && positions.freeValues > positions.end * POOL_VACUUM_FREE_FRACTION;
1116
+ if (sparsePostings || sparsePositions) this.repack(sparsePositions);
1117
+ }
1118
+ repack(rebuildPositions) {
1119
+ const terms = this.terms;
1120
+ const pool = this.pool;
1121
+ const positions = this.positions;
1122
+ let capacity = 0;
1123
+ for (let ordinal = 0; ordinal < terms.count; ordinal++) {
1124
+ if (terms.isLive(ordinal)) capacity += sliceCapacity(sliceClassFor(terms.lengths[ordinal]));
1125
+ }
1126
+ const nextPool = new PostingPool(Math.max(1, capacity));
1127
+ nextPool.epoch = pool.epoch + 1;
1128
+ const sourceStarts = pool.positionStarts;
1129
+ const targetStarts = sourceStarts === null ? null : nextPool.trackPositions();
1130
+ const nextPositions = rebuildPositions ? new PositionPool(positions.end - positions.freeValues) : positions;
1131
+ for (let ordinal = 0; ordinal < terms.count; ordinal++) {
1132
+ if (!terms.isLive(ordinal)) continue;
1133
+ const start = terms.sliceStart[ordinal];
1134
+ const length = terms.lengths[ordinal];
1135
+ const sliceClass = sliceClassFor(length);
1136
+ const target = nextPool.allocate(sliceClass);
1137
+ nextPool.docIds.set(pool.docIds.subarray(start, start + length), target);
1138
+ nextPool.frequencies.set(pool.frequencies.subarray(start, start + length), target);
1139
+ nextPool.fieldIndices.set(pool.fieldIndices.subarray(start, start + length), target);
1140
+ if ((terms.flags[ordinal] & TERM_POSITIONS) !== 0 && sourceStarts !== null && targetStarts !== null) {
1141
+ if (rebuildPositions) {
1142
+ for (let i = 0; i < length; i++)
1143
+ targetStarts[target + i] = nextPositions.appendFrom(positions, sourceStarts[start + i]);
1144
+ } else {
1145
+ targetStarts.set(sourceStarts.subarray(start, start + length), target);
1146
+ }
486
1147
  }
1148
+ terms.sliceStart[ordinal] = target;
1149
+ terms.sliceClass[ordinal] = sliceClass;
487
1150
  }
488
- return filtered;
1151
+ this.pool = nextPool;
1152
+ this.positions = nextPositions;
1153
+ this.compactedSlack = 0;
1154
+ }
1155
+ };
1156
+
1157
+ // src/core/inverted-index/index.ts
1158
+ function fieldIndexFor(fieldNameTable, fieldName) {
1159
+ let index = fieldNameTable.indexMap.get(fieldName);
1160
+ if (index === void 0) {
1161
+ index = fieldNameTable.names.length;
1162
+ fieldNameTable.names.push(fieldName);
1163
+ fieldNameTable.indexMap.set(fieldName, index);
489
1164
  }
490
- function getOrCreateList(token) {
491
- let list = index.get(token);
492
- if (!list) {
493
- list = createPostingList();
494
- index.set(token, list);
495
- trackToken(token);
1165
+ return index;
1166
+ }
1167
+ function createInvertedIndex(fieldNameTable) {
1168
+ const postings = new LivePostings();
1169
+ function* liveTokens() {
1170
+ const terms = postings.terms;
1171
+ const generation = terms.generation;
1172
+ for (let ordinal = 0; ordinal < terms.count && terms.generation === generation; ordinal++) {
1173
+ if (terms.isLive(ordinal)) yield terms.tokens[ordinal];
1174
+ }
1175
+ }
1176
+ function serializeTerm(ordinal, resolver) {
1177
+ const list = postings.view(ordinal, false);
1178
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, length } = list;
1179
+ const positions = list.positions;
1180
+ const entries = [];
1181
+ for (let i = 0; i < length; i++) {
1182
+ if (deletedDocs.size > 0 && deletedDocs.has(docIds[i])) continue;
1183
+ const externalId = resolver.toExternal(docIds[i]);
1184
+ if (externalId === void 0) continue;
1185
+ entries.push({
1186
+ docId: externalId,
1187
+ termFrequency: termFrequencies[i],
1188
+ fieldName: fieldNameTable.names[fieldNameIndices[i]],
1189
+ positions: positions === null ? [] : [...positions[i]]
1190
+ });
496
1191
  }
497
- return list;
1192
+ return entries;
498
1193
  }
499
1194
  return {
500
1195
  insert(token, internalId, termFrequency, fieldNameIndex2, positions) {
501
- const list = getOrCreateList(token);
502
- if (list.deletedDocs.size > 0 && list.deletedDocs.has(internalId)) {
503
- compactDocEntries(list, internalId);
504
- list.deletedDocs.delete(internalId);
505
- }
506
- if (list.length >= list.termFrequencies.length) {
507
- growTypedArrays(list);
508
- }
509
- const idx = list.length;
510
- if (idx > 0 && internalId < list.docIds[idx - 1]) list.ordered = false;
511
- list.docIds.push(internalId);
512
- list.termFrequencies[idx] = termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : termFrequency;
513
- list.fieldNameIndices[idx] = fieldNameIndex2;
514
- list.totalTermFrequency += list.termFrequencies[idx];
515
- if (positions !== null) {
516
- if (!list.positions) {
517
- list.positions = [];
518
- for (let j = 0; j < idx; j++) list.positions.push([]);
519
- }
520
- list.positions.push(positions);
521
- } else if (list.positions !== null) {
522
- list.positions.push([]);
523
- }
524
- if (!list.docIdSet.has(internalId)) {
525
- list.docIdSet.add(internalId);
526
- }
527
- list.length++;
1196
+ postings.insert(token, internalId, termFrequency, fieldNameIndex2, positions);
528
1197
  },
529
1198
  remove(token, internalId, occurrences) {
530
- const list = index.get(token);
531
- if (!list || !list.docIdSet.has(internalId)) return;
532
- list.structureRevision++;
533
- list.docIdSet.delete(internalId);
534
- list.deletedDocs.add(internalId);
535
- list.totalTermFrequency = Math.max(0, list.totalTermFrequency - occurrences);
536
- if (list.docIdSet.size === 0) {
537
- index.delete(token);
538
- untrackToken(token);
539
- if (batchMode) batchDirtyTokens.delete(token);
540
- return;
541
- }
542
- if (batchMode) {
543
- batchDirtyTokens.add(token);
544
- return;
545
- }
546
- if (list.deletedDocs.size / list.length > POSTING_LIST_COMPACTION_THRESHOLD) {
547
- compactList(list);
548
- }
1199
+ postings.remove(token, internalId, occurrences);
549
1200
  },
550
1201
  beginBatch() {
551
- batchMode = true;
552
- batchDirtyTokens.clear();
1202
+ postings.beginBatch();
553
1203
  },
554
1204
  endBatch() {
555
- batchMode = false;
556
- for (const token of batchDirtyTokens) {
557
- const list = index.get(token);
558
- if (!list) continue;
559
- if (list.deletedDocs.size / list.length > POSTING_LIST_COMPACTION_THRESHOLD) {
560
- compactList(list);
561
- }
562
- }
563
- batchDirtyTokens.clear();
1205
+ postings.endBatch();
564
1206
  },
565
1207
  lookup(token) {
566
- return index.get(token);
1208
+ const ordinal = postings.terms.ids.get(token);
1209
+ return ordinal === void 0 ? void 0 : postings.view(ordinal, true);
1210
+ },
1211
+ lookupReadOnce(token) {
1212
+ const ordinal = postings.terms.ids.get(token);
1213
+ return ordinal === void 0 ? void 0 : postings.view(ordinal, false);
567
1214
  },
568
1215
  documentFrequency(token) {
569
- return index.get(token)?.docIdSet.size ?? 0;
1216
+ const ordinal = postings.terms.ids.get(token);
1217
+ return ordinal === void 0 ? 0 : postings.terms.docFrequency[ordinal];
570
1218
  },
571
1219
  fuzzyLookup(token, tolerance, prefixLength) {
1220
+ const terms = postings.terms;
572
1221
  if (tolerance === 0) {
573
- const exact = index.get(token);
574
- return exact ? [{ token, postingList: exact }] : [];
1222
+ const exact = terms.ids.get(token);
1223
+ return exact === void 0 ? [] : [{ token, postingList: postings.view(exact, true) }];
575
1224
  }
576
1225
  const results = [];
577
- const candidates = tokensStartingWith(fuzzyPrefixOf(token, prefixLength));
578
- for (const candidate of candidates) {
579
- const { withinTolerance } = boundedLevenshtein(token, candidate, tolerance);
580
- if (withinTolerance) {
581
- const postingList = index.get(candidate);
582
- if (postingList) results.push({ token: candidate, postingList });
1226
+ terms.forEachStartingWith(fuzzyPrefixOf(token, prefixLength), (ordinal) => {
1227
+ const candidate = terms.tokens[ordinal];
1228
+ if (boundedLevenshtein(token, candidate, tolerance).withinTolerance) {
1229
+ results.push({ token: candidate, postingList: postings.view(ordinal, true) });
583
1230
  }
584
- }
1231
+ });
585
1232
  return results;
586
1233
  },
587
1234
  prefixSearch(prefix, limit) {
588
1235
  if (prefix.length === 0 || limit <= 0) return [];
589
- const candidates = tokensStartingWith(prefix);
1236
+ const terms = postings.terms;
590
1237
  const results = [];
591
- for (const term of candidates) {
592
- if (!term.startsWith(prefix)) continue;
593
- const list = index.get(term);
594
- if (!list) continue;
595
- results.push({ term, documentFrequency: list.docIdSet.size });
596
- }
1238
+ terms.forEachStartingWith(prefix, (ordinal) => {
1239
+ results.push({ term: terms.tokens[ordinal], documentFrequency: terms.docFrequency[ordinal] });
1240
+ });
597
1241
  results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.term, b.term));
598
1242
  if (results.length > limit) results.length = limit;
599
1243
  return results;
600
1244
  },
601
1245
  has(token) {
602
- return index.has(token);
1246
+ return postings.terms.ids.has(token);
603
1247
  },
604
1248
  tokens() {
605
- return index.keys();
1249
+ return liveTokens();
606
1250
  },
607
1251
  size() {
608
- return index.size;
1252
+ return postings.terms.liveCount;
609
1253
  },
610
1254
  clear() {
611
- index.clear();
612
- charBuckets.clear();
1255
+ postings.clear();
1256
+ },
1257
+ estimateBytes() {
1258
+ const terms = postings.terms;
1259
+ const perTerm = TERM_ENTRY_ESTIMATED_BYTES + TOKEN_STRING_HEADER_BYTES;
1260
+ return postings.byteLength() + terms.liveCount * perTerm + terms.tokenCodeUnits * 2;
613
1261
  },
614
1262
  serialize(resolver) {
615
1263
  const result = /* @__PURE__ */ Object.create(null);
616
- for (const [token, list] of index) {
617
- const postings = [];
618
- for (let i = 0; i < list.length; i++) {
619
- if (list.deletedDocs.size > 0 && list.deletedDocs.has(list.docIds[i])) continue;
620
- const externalId = resolver.toExternal(list.docIds[i]);
621
- if (externalId === void 0) continue;
622
- postings.push({
623
- docId: externalId,
624
- termFrequency: list.termFrequencies[i],
625
- fieldName: fieldNameTable.names[list.fieldNameIndices[i]],
626
- positions: list.positions ? list.positions[i] : []
627
- });
628
- }
629
- if (postings.length > 0) {
630
- result[token] = { docFrequency: list.docIdSet.size, postings };
1264
+ const terms = postings.terms;
1265
+ for (let ordinal = 0; ordinal < terms.count; ordinal++) {
1266
+ if (!terms.isLive(ordinal)) continue;
1267
+ const entries = serializeTerm(ordinal, resolver);
1268
+ if (entries.length > 0) {
1269
+ result[terms.tokens[ordinal]] = { docFrequency: terms.docFrequency[ordinal], postings: entries };
631
1270
  }
632
1271
  }
633
1272
  return result;
634
1273
  },
635
1274
  deserialize(data, resolver) {
636
- index.clear();
637
- charBuckets.clear();
1275
+ postings.clear();
638
1276
  for (const token of Object.keys(data)) {
639
- const src = data[token];
640
- const count = src.postings.length;
641
- const docIdSet = /* @__PURE__ */ new Set();
642
- const docIds = new Array(count);
643
- const termFrequencies = new Uint16Array(count);
644
- const fieldNameIndices = new Uint16Array(count);
1277
+ const source = data[token].postings;
645
1278
  let hasPositions = false;
646
- let validCount = 0;
647
- let totalTermFrequency = 0;
648
- for (let i = 0; i < count; i++) {
649
- const p = src.postings[i];
650
- const internalId = resolver.toInternal(p.docId);
651
- if (internalId === void 0) continue;
652
- docIds[validCount] = internalId;
653
- termFrequencies[validCount] = p.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : p.termFrequency;
654
- totalTermFrequency += termFrequencies[validCount];
655
- let fnIndex = fieldNameTable.indexMap.get(p.fieldName);
656
- if (fnIndex === void 0) {
657
- fnIndex = fieldNameTable.names.length;
658
- fieldNameTable.names.push(p.fieldName);
659
- fieldNameTable.indexMap.set(p.fieldName, fnIndex);
660
- }
661
- fieldNameIndices[validCount] = fnIndex;
662
- docIdSet.add(internalId);
663
- if (p.positions.length > 0) hasPositions = true;
664
- validCount++;
665
- }
666
- if (validCount === 0) continue;
667
- docIds.length = validCount;
668
- const finalTF = validCount < count ? termFrequencies.slice(0, validCount) : termFrequencies;
669
- const finalFNI = validCount < count ? fieldNameIndices.slice(0, validCount) : fieldNameIndices;
670
- let positions = null;
671
- if (hasPositions) {
672
- positions = new Array(validCount);
673
- let vi = 0;
674
- for (let i = 0; i < count; i++) {
675
- const internalId = resolver.toInternal(src.postings[i].docId);
676
- if (internalId === void 0) continue;
677
- positions[vi] = src.postings[i].positions;
678
- vi++;
679
- }
1279
+ let valid = 0;
1280
+ for (const posting of source) {
1281
+ if (resolver.toInternal(posting.docId) === void 0) continue;
1282
+ valid++;
1283
+ if (posting.positions.length > 0) hasPositions = true;
680
1284
  }
681
- let ordered = true;
682
- for (let i = 1; i < validCount; i++) {
683
- if (docIds[i] < docIds[i - 1]) {
684
- ordered = false;
685
- break;
686
- }
1285
+ if (valid === 0) continue;
1286
+ postings.reserve(token, valid);
1287
+ for (const posting of source) {
1288
+ const internalId = resolver.toInternal(posting.docId);
1289
+ if (internalId === void 0) continue;
1290
+ const termFrequency = posting.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : posting.termFrequency;
1291
+ const fieldIndex = fieldIndexFor(fieldNameTable, posting.fieldName);
1292
+ postings.insert(token, internalId, termFrequency, fieldIndex, hasPositions ? posting.positions : null);
687
1293
  }
688
- index.set(token, {
689
- length: validCount,
690
- docIds,
691
- termFrequencies: finalTF,
692
- fieldNameIndices: finalFNI,
693
- positions,
694
- docIdSet,
695
- deletedDocs: /* @__PURE__ */ new Set(),
696
- totalTermFrequency,
697
- structureRevision: 0,
698
- ordered
699
- });
700
- trackToken(token);
701
1294
  }
702
1295
  }
703
1296
  };
@@ -866,6 +1459,9 @@ function createSurfaceRegistry() {
866
1459
  else totalsByToken.set(token, next);
867
1460
  }
868
1461
  return {
1462
+ holds(surface) {
1463
+ return entries.has(surface);
1464
+ },
869
1465
  add(surface, token, occurrences) {
870
1466
  if (surface.length === 0 || token.length === 0 || occurrences <= 0) return;
871
1467
  if (surface === token) return;
@@ -2290,7 +2886,7 @@ var BYTES_PER_BOOLEAN_SLOT = 1;
2290
2886
  var BYTES_PER_MIXED_SLOT = 8;
2291
2887
  var BYTES_PER_STRING_CODE_UNIT = 2;
2292
2888
  var STRING_HEADER_BYTES2 = 24;
2293
- function grow(length, wanted) {
2889
+ function grow2(length, wanted) {
2294
2890
  let next = length === 0 ? 64 : length;
2295
2891
  while (next <= wanted) next *= 2;
2296
2892
  return next;
@@ -2325,7 +2921,7 @@ function createValueStore(initialKind) {
2325
2921
  function setNumber(internalId, value) {
2326
2922
  if (numbers === null) return;
2327
2923
  if (internalId >= numbers.length) {
2328
- const grown = new Float64Array(grow(numbers.length, internalId));
2924
+ const grown = new Float64Array(grow2(numbers.length, internalId));
2329
2925
  grown.fill(Number.NaN, numbers.length);
2330
2926
  grown.set(numbers);
2331
2927
  numbers = grown;
@@ -2335,7 +2931,7 @@ function createValueStore(initialKind) {
2335
2931
  function setBoolean(internalId, value) {
2336
2932
  if (booleans === null) return;
2337
2933
  if (internalId >= booleans.length) {
2338
- const grown = new Uint8Array(grow(booleans.length, internalId));
2934
+ const grown = new Uint8Array(grow2(booleans.length, internalId));
2339
2935
  grown.fill(BOOLEAN_MISSING, booleans.length);
2340
2936
  grown.set(booleans);
2341
2937
  booleans = grown;
@@ -2446,10 +3042,9 @@ function createSortColumnSet(docStore) {
2446
3042
  }
2447
3043
  function backfill(key, field, fieldType, mode) {
2448
3044
  const store = createValueStore(kindForFieldType(fieldType));
2449
- for (const [docId, stored] of docStore.all()) {
2450
- const internalId = docStore.getInternalId(docId);
2451
- if (internalId === void 0) continue;
2452
- store.set(internalId, toReducedSortValue(readSortField(stored.fields, field), mode));
3045
+ const read = docStore.fieldReader(field);
3046
+ for (const internalId of docStore.allInternalIds()) {
3047
+ store.set(internalId, toReducedSortValue(read(internalId), mode));
2453
3048
  }
2454
3049
  const entry = {
2455
3050
  field,
@@ -2781,15 +3376,18 @@ function sortedPageOf(state, request) {
2781
3376
  function sortValuesOf(state, docId, fields, fieldTypes, modes) {
2782
3377
  const set = state.sortColumns;
2783
3378
  const internalId = state.docStore.getInternalId(docId);
2784
- const stored = state.docStore.get(docId);
2785
3379
  const key = new Array(fields.length);
2786
3380
  for (let i = 0; i < fields.length; i++) {
2787
3381
  const mode = modes[i] ?? "min";
2788
- if (set !== null && internalId !== void 0 && set.holds(fields[i], fieldTypes[i], mode)) {
3382
+ if (internalId === void 0) {
3383
+ key[i] = null;
3384
+ continue;
3385
+ }
3386
+ if (set?.holds(fields[i], fieldTypes[i], mode)) {
2789
3387
  key[i] = set.column(fields[i], fieldTypes[i], mode).valueOf(internalId);
2790
3388
  continue;
2791
3389
  }
2792
- key[i] = stored === void 0 ? null : toReducedSortValue(readSortField(stored.fields, fields[i]), mode);
3390
+ key[i] = toReducedSortValue(state.docStore.fieldReader(fields[i])(internalId), mode);
2793
3391
  }
2794
3392
  return key;
2795
3393
  }
@@ -2832,7 +3430,15 @@ function buildFilterContext(state, schema, meter) {
2832
3430
  for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
2833
3431
  if (pattern) patternFields.add(fieldPath);
2834
3432
  }
2835
- const getFieldValue = (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath);
3433
+ const fieldReaders = /* @__PURE__ */ new Map();
3434
+ const readerOf = (fieldPath) => {
3435
+ let read = fieldReaders.get(fieldPath);
3436
+ if (read === void 0) {
3437
+ read = state.docStore.fieldReader(fieldPath);
3438
+ fieldReaders.set(fieldPath, read);
3439
+ }
3440
+ return read;
3441
+ };
2836
3442
  let cachedAllDocIdsBitset = null;
2837
3443
  const cachedPresentBitsets = /* @__PURE__ */ new Map();
2838
3444
  return {
@@ -2856,7 +3462,7 @@ function buildFilterContext(state, schema, meter) {
2856
3462
  matches: into,
2857
3463
  meter,
2858
3464
  host: PATTERN_HOST,
2859
- valueOf: (internalId) => getFieldValue(internalId, fieldPath)
3465
+ valueOf: readerOf(fieldPath)
2860
3466
  });
2861
3467
  },
2862
3468
  presentBitset: (fieldPath) => {
@@ -3215,38 +3821,18 @@ function searchFulltextMatches(state, params) {
3215
3821
  // src/core/partition/memory.ts
3216
3822
  var DOC_ENTRY_OVERHEAD_BYTES = 260;
3217
3823
  var LENGTH_COLUMN_SLOT_BYTES = 6;
3218
- var POSTING_ENTRY_BYTES = 40;
3219
- var POSITIONS_ARRAY_OVERHEAD_BYTES = 88;
3220
- var POSITION_BYTES = 8;
3221
- var PER_TERM_OVERHEAD_BYTES = 180;
3824
+ var DOC_FREQUENCY_ENTRY_BYTES = 44;
3222
3825
  var SURFACE_ENTRY_OVERHEAD_BYTES = 140;
3223
3826
  var FIELD_ENTRY_OVERHEAD_BYTES = 42;
3224
3827
  function estimatePartitionBytes(state) {
3225
3828
  const docCount = state.docStore.count();
3226
3829
  if (docCount === 0) return 0;
3227
- const docFreqs = state.stats.docFrequencies;
3228
- let totalPostings = 0;
3229
- let termCount = 0;
3230
- for (const term in docFreqs) {
3231
- totalPostings += docFreqs[term];
3232
- termCount++;
3233
- }
3234
- const fieldLengthTotals = state.stats.totalFieldLengths;
3235
- let totalTokenOccurrences = 0;
3236
- let textFieldCount = 0;
3237
- for (const field in fieldLengthTotals) {
3238
- totalTokenOccurrences += fieldLengthTotals[field];
3239
- textFieldCount++;
3240
- }
3830
+ const textFieldCount = Object.keys(state.stats.totalFieldLengths).length;
3241
3831
  let bytes = state.docStore.contentBytes();
3242
3832
  bytes += docCount * DOC_ENTRY_OVERHEAD_BYTES;
3243
3833
  bytes += textFieldCount * state.docStore.internalIdCapacity() * LENGTH_COLUMN_SLOT_BYTES;
3244
- bytes += totalPostings * POSTING_ENTRY_BYTES;
3245
- if (state.trackPositions) {
3246
- bytes += totalPostings * POSITIONS_ARRAY_OVERHEAD_BYTES;
3247
- bytes += totalTokenOccurrences * POSITION_BYTES;
3248
- }
3249
- bytes += termCount * PER_TERM_OVERHEAD_BYTES;
3834
+ bytes += state.invertedIdx.estimateBytes();
3835
+ bytes += state.invertedIdx.size() * DOC_FREQUENCY_ENTRY_BYTES;
3250
3836
  bytes += state.surfaceRegistry.size() * SURFACE_ENTRY_OVERHEAD_BYTES;
3251
3837
  bytes += docCount * state.numericIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
3252
3838
  bytes += docCount * state.booleanIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
@@ -3282,17 +3868,18 @@ function mergeDocuments(target, source) {
3282
3868
  }
3283
3869
  function mergePostings(target, source, idMapping) {
3284
3870
  for (const token of source.invertedIdx.tokens()) {
3285
- const list = source.invertedIdx.lookup(token);
3871
+ const list = source.invertedIdx.lookupReadOnce?.(token) ?? source.invertedIdx.lookup(token);
3286
3872
  if (list === void 0) continue;
3287
- for (let i = 0; i < list.length; i++) {
3288
- const sourceInternalId = list.docIds[i];
3289
- if (list.deletedDocs.has(sourceInternalId)) continue;
3873
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, positions, length } = list;
3874
+ for (let i = 0; i < length; i++) {
3875
+ const sourceInternalId = docIds[i];
3876
+ if (deletedDocs.has(sourceInternalId)) continue;
3290
3877
  const targetInternalId = idMapping.get(sourceInternalId);
3291
3878
  if (targetInternalId === void 0) continue;
3292
- const fieldName = source.fieldNameTable.names[list.fieldNameIndices[i]];
3879
+ const fieldName = source.fieldNameTable.names[fieldNameIndices[i]];
3293
3880
  const fieldNameIndex2 = getOrCreateFieldNameIndex(target.fieldNameTable, fieldName);
3294
- const positions = list.positions === null ? null : list.positions[i];
3295
- target.invertedIdx.insert(token, targetInternalId, list.termFrequencies[i], fieldNameIndex2, positions ?? null);
3881
+ const entryPositions = positions === null ? null : positions[i];
3882
+ target.invertedIdx.insert(token, targetInternalId, termFrequencies[i], fieldNameIndex2, entryPositions ?? null);
3296
3883
  }
3297
3884
  }
3298
3885
  }
@@ -3404,8 +3991,9 @@ function entriesCovered(bounds) {
3404
3991
  return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
3405
3992
  }
3406
3993
  function scan(list, columns, previous, keepBlocks) {
3994
+ const { docIds, termFrequencies, fieldNameIndices, length } = list;
3407
3995
  const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
3408
- const capacity = keepBlocks + Math.ceil((list.length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
3996
+ const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
3409
3997
  const entryEnd = new Int32Array(capacity);
3410
3998
  const maxTermFrequency = new Int32Array(capacity);
3411
3999
  const minFieldLength = new Float64Array(capacity);
@@ -3427,16 +4015,16 @@ function scan(list, columns, previous, keepBlocks) {
3427
4015
  let blockMaxRun = 0;
3428
4016
  let blockDocs = 0;
3429
4017
  let entry = startEntry;
3430
- while (entry < list.length) {
3431
- const docId = list.docIds[entry];
4018
+ while (entry < length) {
4019
+ const docId = docIds[entry];
3432
4020
  const tombstoned = hasDeleted && deleted.has(docId);
3433
4021
  let run = 0;
3434
- while (entry + run < list.length && list.docIds[entry + run] === docId) {
4022
+ while (entry + run < length && docIds[entry + run] === docId) {
3435
4023
  if (!tombstoned) {
3436
4024
  const position = entry + run;
3437
- const termFrequency = list.termFrequencies[position];
4025
+ const termFrequency = termFrequencies[position];
3438
4026
  if (termFrequency > blockMaxTf) blockMaxTf = termFrequency;
3439
- const fieldLength = fieldLengthBound(columns, list.fieldNameIndices[position], docId);
4027
+ const fieldLength = fieldLengthBound(columns, fieldNameIndices[position], docId);
3440
4028
  if (fieldLength < blockMinLen) blockMinLen = fieldLength;
3441
4029
  }
3442
4030
  run++;
@@ -3446,7 +4034,7 @@ function scan(list, columns, previous, keepBlocks) {
3446
4034
  blockDocs++;
3447
4035
  }
3448
4036
  entry += run;
3449
- if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === list.length) {
4037
+ if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === length) {
3450
4038
  entryEnd[blockIndex] = entry;
3451
4039
  maxTermFrequency[blockIndex] = blockMaxTf;
3452
4040
  minFieldLength[blockIndex] = blockMinLen === Number.POSITIVE_INFINITY ? ABSENT_FIELD_LENGTH_BOUND : blockMinLen;
@@ -4203,10 +4791,11 @@ function countPostings(state, remap) {
4203
4791
  const list = lookupReadOnce(state, token);
4204
4792
  if (list === void 0) continue;
4205
4793
  const listPositions = list.positionCountAt === void 0 ? list.positions : null;
4206
- const tracksPositions = list.positionCountAt === void 0 ? listPositions !== null : state.trackPositions;
4207
- for (let i = 0; i < list.length; i++) {
4208
- const internalId = list.docIds[i];
4209
- if (list.deletedDocs.has(internalId) || remap[internalId] < 0) continue;
4794
+ const tracksPositions = list.holdsPositions ?? (list.positionCountAt === void 0 ? listPositions !== null : state.trackPositions);
4795
+ const { docIds, deletedDocs, length } = list;
4796
+ for (let i = 0; i < length; i++) {
4797
+ const internalId = docIds[i];
4798
+ if (deletedDocs.has(internalId) || remap[internalId] < 0) continue;
4210
4799
  postings++;
4211
4800
  if (!tracksPositions) continue;
4212
4801
  hasPositions = true;
@@ -4257,18 +4846,20 @@ function encodeSegmentState(state) {
4257
4846
  for (const token of state.invertedIdx.tokens()) {
4258
4847
  const list = lookupReadOnce(state, token);
4259
4848
  if (list === void 0) continue;
4260
- const listPositions = positionOffsets === null ? null : list.positions;
4849
+ const copyPositionsAt = list.copyPositionsAt?.bind(list);
4850
+ const listPositions = positionOffsets === null || copyPositionsAt !== void 0 ? null : list.positions;
4261
4851
  const tokenStart = postingCursor;
4262
4852
  const countViaSet = !list.ordered;
4263
4853
  if (countViaSet) unorderedTokenDocs.clear();
4264
4854
  let uniqueDocs = 0;
4265
4855
  let lastCountedDoc = -1;
4266
- for (let i = 0; i < list.length; i++) {
4267
- const internalId = list.docIds[i];
4268
- if (list.deletedDocs.has(internalId) || remap[internalId] < 0) continue;
4856
+ const { docIds: docIds2, termFrequencies, fieldNameIndices, deletedDocs, length } = list;
4857
+ for (let i = 0; i < length; i++) {
4858
+ const internalId = docIds2[i];
4859
+ if (deletedDocs.has(internalId) || remap[internalId] < 0) continue;
4269
4860
  postingDocIds[postingCursor] = remap[internalId];
4270
- postingFrequencies[postingCursor] = list.termFrequencies[i];
4271
- postingFieldIndices[postingCursor] = list.fieldNameIndices[i];
4861
+ postingFrequencies[postingCursor] = termFrequencies[i];
4862
+ postingFieldIndices[postingCursor] = fieldNameIndices[i];
4272
4863
  if (countViaSet) {
4273
4864
  unorderedTokenDocs.add(internalId);
4274
4865
  } else if (internalId !== lastCountedDoc) {
@@ -4278,7 +4869,9 @@ function encodeSegmentState(state) {
4278
4869
  if (positionOffsets !== null && positionValues !== null) {
4279
4870
  positionOffsets[postingCursor] = positionCursor;
4280
4871
  const entryPositions = listPositions === null ? null : listPositions[i];
4281
- if (entryPositions !== null && entryPositions !== void 0) {
4872
+ if (copyPositionsAt !== void 0) {
4873
+ positionCursor += copyPositionsAt(i, positionValues, positionCursor);
4874
+ } else if (entryPositions !== null && entryPositions !== void 0) {
4282
4875
  for (const position of entryPositions) positionValues[positionCursor++] = position;
4283
4876
  }
4284
4877
  }
@@ -4488,17 +5081,18 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
4488
5081
  for (const token of state.invertedIdx.tokens()) {
4489
5082
  const list = state.invertedIdx.lookup(token);
4490
5083
  if (!list) continue;
4491
- const hasDeleted = list.deletedDocs.size > 0;
5084
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, positions, length } = list;
5085
+ const hasDeleted = deletedDocs.size > 0;
4492
5086
  const postings = [];
4493
- for (let i = 0; i < list.length; i++) {
4494
- if (hasDeleted && list.deletedDocs.has(list.docIds[i])) continue;
4495
- const externalId = resolver.toExternal(list.docIds[i]);
5087
+ for (let i = 0; i < length; i++) {
5088
+ if (hasDeleted && deletedDocs.has(docIds[i])) continue;
5089
+ const externalId = resolver.toExternal(docIds[i]);
4496
5090
  if (externalId === void 0) continue;
4497
5091
  postings.push({
4498
5092
  docId: externalId,
4499
- termFrequency: list.termFrequencies[i],
4500
- field: fieldNames[list.fieldNameIndices[i]],
4501
- positions: list.positions ? list.positions[i] : []
5093
+ termFrequency: termFrequencies[i],
5094
+ field: fieldNames[fieldNameIndices[i]],
5095
+ positions: positions === null ? [] : Array.from(positions[i] ?? [])
4502
5096
  });
4503
5097
  }
4504
5098
  if (postings.length > 0) {
@@ -4821,6 +5415,11 @@ function createPartitionIndex(partitionId, trackPositions = true) {
4821
5415
  if (!stored) return void 0;
4822
5416
  return stored.fields;
4823
5417
  },
5418
+ fieldValues(docId, fieldPaths) {
5419
+ const stored = state.docStore.get(docId);
5420
+ if (!stored) return void 0;
5421
+ return fieldPaths.map((fieldPath) => getNestedValue(stored.fields, fieldPath));
5422
+ },
4824
5423
  has(docId) {
4825
5424
  return state.docStore.has(docId);
4826
5425
  },
@@ -4910,7 +5509,7 @@ function frozenSegmentBytes(segment) {
4910
5509
  bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
4911
5510
  }
4912
5511
  for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
4913
- return bytes + segment.partialWords.bytes();
5512
+ return bytes + segment.partialWords.bytes() + segment.removalBytes();
4914
5513
  }
4915
5514
 
4916
5515
  // src/core/partition/composite/compaction.ts
@@ -4948,19 +5547,40 @@ function freezeLiveTailInto(live, liveState, frozen, freeze) {
4948
5547
  frozen.push(segment);
4949
5548
  return segment;
4950
5549
  }
4951
- function swapFrozenSegmentList(frozen, dropSegmentIds, replacement) {
5550
+ function swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings) {
4952
5551
  const dropSet = new Set(dropSegmentIds);
4953
5552
  const dropped = frozen.filter((segment) => dropSet.has(segment.segmentId));
5553
+ const applied = /* @__PURE__ */ new Map();
4954
5554
  for (const segment of dropped) {
4955
5555
  for (const docId of segment.tombstonedDocIds()) {
4956
5556
  if (!replacement.hasDocument(docId)) continue;
4957
5557
  if (dropped.some((other) => other.hasDocument(docId))) continue;
4958
- replacement.tombstoneDocument(docId);
5558
+ const spellings = removedSpellings?.get(docId) ?? replacement.removedSpellingsOf(docId);
5559
+ if (spellings !== void 0) applied.set(docId, spellings);
5560
+ replacement.tombstoneDocument(docId, spellings);
4959
5561
  }
4960
5562
  }
4961
5563
  const kept = frozen.filter((segment) => !dropSet.has(segment.segmentId));
4962
5564
  frozen.length = 0;
4963
5565
  frozen.push(...kept, replacement);
5566
+ return applied;
5567
+ }
5568
+
5569
+ // src/core/partition/composite/documents.ts
5570
+ function compositeDocument(frozen, live, docId, projection) {
5571
+ for (const segment of frozen) {
5572
+ const projected = segment.docStore.projectedDocument(docId, projection);
5573
+ if (projected !== void 0) return projected;
5574
+ }
5575
+ return live.get(docId, projection);
5576
+ }
5577
+ function compositeFieldValues(frozen, live, docId, fieldPaths) {
5578
+ for (const segment of frozen) {
5579
+ const internalId = segment.docStore.getInternalId(docId);
5580
+ if (internalId === void 0) continue;
5581
+ return fieldPaths.map((fieldPath) => segment.docStore.fieldReader(fieldPath)(internalId));
5582
+ }
5583
+ return live.fieldValues(docId, fieldPaths);
4964
5584
  }
4965
5585
 
4966
5586
  // src/core/partition/composite/filters.ts
@@ -5286,7 +5906,7 @@ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
5286
5906
  function isCompositePartition(partition) {
5287
5907
  return "attachFrozenSegment" in partition;
5288
5908
  }
5289
- function createCompositePartition(partitionId, trackPositions = true, existingLive) {
5909
+ function createCompositePartition(partitionId, trackPositions = true, existingLive, textAnalysis) {
5290
5910
  const live = existingLive ?? createPartitionIndex(partitionId, trackPositions);
5291
5911
  const liveState = partitionStateOf(live);
5292
5912
  const frozen = [];
@@ -5294,6 +5914,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5294
5914
  function invalidateDocFrequencies() {
5295
5915
  docFrequenciesCache = null;
5296
5916
  }
5917
+ function bindSegments() {
5918
+ invalidateDocFrequencies();
5919
+ if (textAnalysis === void 0) return;
5920
+ for (const segment of frozen) segment.removals.bind(textAnalysis);
5921
+ }
5297
5922
  function cachedDocFrequencies() {
5298
5923
  if (docFrequenciesCache === null) {
5299
5924
  docFrequenciesCache = mergeDocFrequencies(subs());
@@ -5347,23 +5972,25 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5347
5972
  frozenSegmentsById(segmentIds) {
5348
5973
  return resolveFrozenSegments(frozen, segmentIds, partitionId);
5349
5974
  },
5350
- swapFrozenSegments(dropSegmentIds, replacement) {
5351
- swapFrozenSegmentList(frozen, dropSegmentIds, replacement);
5352
- invalidateDocFrequencies();
5975
+ swapFrozenSegments(dropSegmentIds, replacement, removedSpellings) {
5976
+ if (textAnalysis !== void 0) replacement.removals.bind(textAnalysis);
5977
+ const applied = swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings);
5978
+ bindSegments();
5979
+ return applied;
5353
5980
  },
5354
5981
  freezeLiveTail(freeze) {
5355
5982
  const segment = freezeLiveTailInto(live, liveState, frozen, freeze);
5356
- if (segment !== null) invalidateDocFrequencies();
5983
+ if (segment !== null) bindSegments();
5357
5984
  return segment;
5358
5985
  },
5359
5986
  replaceLiveTail(segment) {
5360
5987
  live.clear();
5361
5988
  frozen.push(segment);
5362
- invalidateDocFrequencies();
5989
+ bindSegments();
5363
5990
  },
5364
5991
  attachFrozenSegment(segment) {
5365
5992
  frozen.push(segment);
5366
- invalidateDocFrequencies();
5993
+ bindSegments();
5367
5994
  },
5368
5995
  mergeSegment(segment) {
5369
5996
  invalidateDocFrequencies();
@@ -5402,7 +6029,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5402
6029
  invalidateDocFrequencies();
5403
6030
  live.insert(docId, document, schema, language, options);
5404
6031
  },
5405
- update(docId, document, schema, language, options) {
6032
+ update(docId, document, schema, language, options, removedSpellings) {
5406
6033
  invalidateDocFrequencies();
5407
6034
  const owner = frozenOwner(docId);
5408
6035
  if (owner === void 0) {
@@ -5410,9 +6037,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5410
6037
  return;
5411
6038
  }
5412
6039
  live.insert(docId, document, schema, language, options);
5413
- owner.tombstoneDocument(docId);
6040
+ owner.tombstoneDocument(docId, removedSpellings);
5414
6041
  },
5415
- remove(docId, schema, language, options) {
6042
+ remove(docId, schema, language, options, removedSpellings) {
5416
6043
  if (live.has(docId)) {
5417
6044
  invalidateDocFrequencies();
5418
6045
  live.remove(docId, schema, language, options);
@@ -5425,7 +6052,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5425
6052
  partitionId
5426
6053
  });
5427
6054
  }
5428
- owner.tombstoneDocument(docId);
6055
+ invalidateDocFrequencies();
6056
+ owner.tombstoneDocument(docId, removedSpellings);
6057
+ },
6058
+ removedSpellingsOf(docId) {
6059
+ return live.has(docId) ? void 0 : frozenOwner(docId)?.removedSpellingsOf(docId);
5429
6060
  },
5430
6061
  beginBatch() {
5431
6062
  live.beginBatch();
@@ -5488,9 +6119,10 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5488
6119
  expandPartialWords(subs(), analysis, []);
5489
6120
  },
5490
6121
  get(docId, projection) {
5491
- const stored = frozenStored(docId);
5492
- if (stored !== void 0) return cloneProjected(stored.fields, projection);
5493
- return live.get(docId, projection);
6122
+ return compositeDocument(frozen, live, docId, projection);
6123
+ },
6124
+ fieldValues(docId, fieldPaths) {
6125
+ return compositeFieldValues(frozen, live, docId, fieldPaths);
5494
6126
  },
5495
6127
  getRef(docId) {
5496
6128
  const stored = frozenStored(docId);
@@ -5509,8 +6141,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5509
6141
  },
5510
6142
  *docIds() {
5511
6143
  for (const segment of frozen) {
5512
- for (const [docId] of segment.docStore.all()) {
5513
- yield docId;
6144
+ for (const internalId of segment.docStore.allInternalIds()) {
6145
+ const docId = segment.docStore.getExternalId(internalId);
6146
+ if (docId !== void 0) yield docId;
5514
6147
  }
5515
6148
  }
5516
6149
  yield* live.docIds();
@@ -5601,6 +6234,18 @@ function createCapacityChecks(source) {
5601
6234
  };
5602
6235
  }
5603
6236
 
6237
+ // src/partitioning/manager/segments.ts
6238
+ function requireNewSegmentDocuments(segment, docPartitionMap) {
6239
+ for (const internalId of segment.docStore.allInternalIds()) {
6240
+ const docId = segment.docStore.getExternalId(internalId);
6241
+ if (docId === void 0 || !docPartitionMap.has(docId)) continue;
6242
+ throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
6243
+ docId,
6244
+ partitionId: docPartitionMap.get(docId)
6245
+ });
6246
+ }
6247
+ }
6248
+
5604
6249
  // src/partitioning/manager/stats.ts
5605
6250
  function aggregateStatsOf(partitions) {
5606
6251
  let totalDocuments = 0;
@@ -5683,6 +6328,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5683
6328
  const count = initialPartitionCount ?? 1;
5684
6329
  const analysis = resolveIndexAnalysis(config);
5685
6330
  const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
6331
+ const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
5686
6332
  const trackPositions = config.trackPositions ?? true;
5687
6333
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
5688
6334
  let partitions = [];
@@ -5711,10 +6357,8 @@ function createPartitionManager(indexName, config, language, router, initialPart
5711
6357
  }
5712
6358
  function asCompositePartition(partitionId) {
5713
6359
  const existing = partitions[partitionId];
5714
- if ("attachFrozenSegment" in existing) {
5715
- return existing;
5716
- }
5717
- const composite = createCompositePartition(partitionId, trackPositions, existing);
6360
+ if (isCompositePartition(existing)) return existing;
6361
+ const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
5718
6362
  partitions[partitionId] = composite;
5719
6363
  return composite;
5720
6364
  }
@@ -5764,6 +6408,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
5764
6408
  get partialWordAnalysis() {
5765
6409
  return partialWordAnalysis;
5766
6410
  },
6411
+ get textAnalysis() {
6412
+ return textAnalysis;
6413
+ },
5767
6414
  getPartition(partitionId) {
5768
6415
  validatePartitionId(partitionId);
5769
6416
  return partitions[partitionId];
@@ -5847,12 +6494,17 @@ function createPartitionManager(indexName, config, language, router, initialPart
5847
6494
  docPartitionMap.set(docId, pid);
5848
6495
  writeLog.documentWritten(docId);
5849
6496
  },
5850
- remove(docId) {
6497
+ removedSpellingsOf(docId) {
6498
+ const pid = locateDocument(docId);
6499
+ const partition = pid === void 0 ? void 0 : partitions[pid];
6500
+ return partition !== void 0 && isCompositePartition(partition) ? partition.removedSpellingsOf(docId) : void 0;
6501
+ },
6502
+ remove(docId, removedSpellings) {
5851
6503
  const pid = locateDocument(docId);
5852
6504
  if (pid === void 0) {
5853
6505
  throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
5854
6506
  }
5855
- partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
6507
+ partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
5856
6508
  docPartitionMap.delete(docId);
5857
6509
  writeLog.documentWritten(docId);
5858
6510
  },
@@ -5866,13 +6518,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
5866
6518
  partitions[i].endBatch();
5867
6519
  }
5868
6520
  },
5869
- update(docId, document, options) {
6521
+ update(docId, document, options, removedSpellings) {
5870
6522
  const pid = locateDocument(docId);
5871
6523
  if (pid === void 0) {
5872
6524
  throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
5873
6525
  }
5874
6526
  const updateOpts = resolveInsertOptions(options);
5875
- partitions[pid].update(docId, document, config.schema, language, updateOpts);
6527
+ partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
5876
6528
  docPartitionMap.set(docId, pid);
5877
6529
  writeLog.documentWritten(docId);
5878
6530
  },
@@ -5880,14 +6532,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5880
6532
  const pid = locateDocument(docId);
5881
6533
  if (pid === void 0) return void 0;
5882
6534
  const doc = partitions[pid].get(docId, projection);
5883
- if (!doc) return void 0;
5884
- for (const [fieldPath, vecIndex] of vecIndexes) {
5885
- if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
5886
- const vector = vecIndex.getVector(docId);
5887
- if (vector) {
5888
- setNestedValue(doc, fieldPath, vector);
5889
- }
5890
- }
6535
+ if (doc) copyVectorsInto(doc, docId, vecIndexes, projection);
5891
6536
  return doc;
5892
6537
  },
5893
6538
  getRef(docId) {
@@ -5895,6 +6540,10 @@ function createPartitionManager(indexName, config, language, router, initialPart
5895
6540
  if (pid === void 0) return void 0;
5896
6541
  return partitions[pid].getRef(docId);
5897
6542
  },
6543
+ fieldValues(docId, fieldPaths) {
6544
+ const pid = locateDocument(docId);
6545
+ return pid === void 0 ? void 0 : partitions[pid].fieldValues(docId, fieldPaths);
6546
+ },
5898
6547
  sortValues(docId, fields, fieldTypes, modes) {
5899
6548
  const pid = locateDocument(docId);
5900
6549
  if (pid === void 0) return fields.map(() => null);
@@ -5930,22 +6579,20 @@ function createPartitionManager(indexName, config, language, router, initialPart
5930
6579
  },
5931
6580
  attachFrozenSegment(partitionId, segment) {
5932
6581
  validatePartitionId(partitionId);
5933
- for (const internalId of segment.docStore.allInternalIds()) {
5934
- const docId = segment.docStore.getExternalId(internalId);
5935
- if (docId === void 0 || !docPartitionMap.has(docId)) continue;
5936
- throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
5937
- docId,
5938
- partitionId: docPartitionMap.get(docId)
5939
- });
5940
- }
6582
+ requireNewSegmentDocuments(segment, docPartitionMap);
5941
6583
  asCompositePartition(partitionId).attachFrozenSegment(segment);
5942
6584
  registerSegmentDocuments(partitionId, segment);
5943
6585
  writeLog.segmentWritten(segment);
5944
6586
  },
5945
- swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
6587
+ swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
5946
6588
  validatePartitionId(partitionId);
5947
- asCompositePartition(partitionId).swapFrozenSegments(dropSegmentIds, replacement);
6589
+ const applied = asCompositePartition(partitionId).swapFrozenSegments(
6590
+ dropSegmentIds,
6591
+ replacement,
6592
+ removedSpellings
6593
+ );
5948
6594
  registerSegmentDocuments(partitionId, replacement);
6595
+ return applied;
5949
6596
  },
5950
6597
  freezeLiveTail(partitionId, freeze) {
5951
6598
  validatePartitionId(partitionId);
@@ -6033,21 +6680,15 @@ function generateId() {
6033
6680
  // src/core/partition/frozen/doc-table.ts
6034
6681
  function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
6035
6682
  const capacity = idTable.count;
6036
- const storedCache = new Array(capacity);
6037
6683
  let sortedIds = null;
6038
6684
  let sortedRevision = -1;
6039
6685
  function storedAt(ordinal) {
6040
- let stored = storedCache[ordinal];
6041
- if (stored === void 0) {
6042
- const fieldLengths = {};
6043
- for (const [fieldName, column] of lengthColumns) {
6044
- const length = column[ordinal];
6045
- if (length > 0) fieldLengths[fieldName] = length;
6046
- }
6047
- stored = { fields: documents.docAt(ordinal), fieldLengths };
6048
- storedCache[ordinal] = stored;
6686
+ const fieldLengths = {};
6687
+ for (const [fieldName, column] of lengthColumns) {
6688
+ const length = column[ordinal];
6689
+ if (length > 0) fieldLengths[fieldName] = length;
6049
6690
  }
6050
- return stored;
6691
+ return { fields: documents.docAt(ordinal), fieldLengths };
6051
6692
  }
6052
6693
  function liveOrdinal(docId) {
6053
6694
  const ordinal = idTable.ordinalOf(docId);
@@ -6116,43 +6757,275 @@ function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
6116
6757
  },
6117
6758
  resolver() {
6118
6759
  return resolver;
6760
+ },
6761
+ fieldReader(fieldPath) {
6762
+ const read = documents.fieldReader(fieldPath);
6763
+ return (internalId) => resolver.isLive(internalId) ? read(internalId) : void 0;
6764
+ },
6765
+ projectedDocument(docId, projection) {
6766
+ const ordinal = liveOrdinal(docId);
6767
+ return ordinal === void 0 ? void 0 : documents.projectedAt(ordinal, projection);
6119
6768
  }
6120
6769
  };
6121
6770
  }
6122
- function removeFieldAtPath(document, fieldPath) {
6123
- const segments = fieldPath.split(".");
6124
- let current = document;
6125
- for (let i = 0; i < segments.length - 1; i += 1) {
6126
- const next = current[segments[i]];
6127
- if (next === null || typeof next !== "object" || Array.isArray(next)) return;
6128
- current = next;
6771
+ var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
6772
+ var keyDecoder = new TextDecoder("utf-8", { fatal: true });
6773
+ function headerAt(view, offset) {
6774
+ if (offset >= view.byteLength) return null;
6775
+ const tag = view.getUint8(offset);
6776
+ if (tag <= 127 || tag >= 224) return { headerBytes: 1, bodyBytes: 0, children: 0 };
6777
+ if (tag <= 143) return { headerBytes: 1, bodyBytes: 0, children: (tag & 15) * 2 };
6778
+ if (tag <= 159) return { headerBytes: 1, bodyBytes: 0, children: tag & 15 };
6779
+ if (tag <= 191) return { headerBytes: 1, bodyBytes: tag & 31, children: 0 };
6780
+ const lengthBytes = LENGTH_BYTES_BY_TAG.get(tag);
6781
+ if (lengthBytes !== void 0) {
6782
+ if (offset + 1 + lengthBytes > view.byteLength) return null;
6783
+ const length = lengthBytes === 1 ? view.getUint8(offset + 1) : lengthBytes === 2 ? view.getUint16(offset + 1, false) : view.getUint32(offset + 1, false);
6784
+ const headerBytes = 1 + lengthBytes;
6785
+ if (ARRAY_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length };
6786
+ if (MAP_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length * 2 };
6787
+ return { headerBytes, bodyBytes: length + (EXT_TAGS.has(tag) ? 1 : 0), children: 0 };
6129
6788
  }
6130
- delete current[segments[segments.length - 1]];
6789
+ const fixedBytes = FIXED_BODY_BYTES_BY_TAG.get(tag);
6790
+ return fixedBytes === void 0 ? null : { headerBytes: 1, bodyBytes: fixedBytes, children: 0 };
6131
6791
  }
6132
- function decodeStoredDocument(bytes, discardedFields) {
6133
- let decoded;
6134
- try {
6135
- decoded = decode(bytes);
6136
- } catch (error) {
6137
- throw new NarsilError(
6138
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
6139
- `A stored document is not MessagePack: ${error instanceof Error ? error.message : String(error)}`
6140
- );
6141
- }
6142
- if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded) || decoded instanceof Uint8Array) {
6143
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "A stored document decodes to something other than a map");
6792
+ var LENGTH_BYTES_BY_TAG = /* @__PURE__ */ new Map([
6793
+ [196, 1],
6794
+ [197, 2],
6795
+ [198, 4],
6796
+ [199, 1],
6797
+ [200, 2],
6798
+ [201, 4],
6799
+ [217, 1],
6800
+ [218, 2],
6801
+ [219, 4],
6802
+ [220, 2],
6803
+ [221, 4],
6804
+ [222, 2],
6805
+ [223, 4]
6806
+ ]);
6807
+ var ARRAY_TAGS = /* @__PURE__ */ new Set([220, 221]);
6808
+ var MAP_TAGS = /* @__PURE__ */ new Set([222, 223]);
6809
+ var EXT_TAGS = /* @__PURE__ */ new Set([199, 200, 201]);
6810
+ var FIXED_BODY_BYTES_BY_TAG = /* @__PURE__ */ new Map([
6811
+ [192, 0],
6812
+ [194, 0],
6813
+ [195, 0],
6814
+ [202, 4],
6815
+ [203, 8],
6816
+ [204, 1],
6817
+ [205, 2],
6818
+ [206, 4],
6819
+ [207, 8],
6820
+ [208, 1],
6821
+ [209, 2],
6822
+ [210, 4],
6823
+ [211, 8],
6824
+ [212, 2],
6825
+ [213, 3],
6826
+ [214, 5],
6827
+ [215, 9],
6828
+ [216, 17]
6829
+ ]);
6830
+ function endOfValue(view, start) {
6831
+ let offset = start;
6832
+ let pending = 1;
6833
+ while (pending > 0) {
6834
+ const header = headerAt(view, offset);
6835
+ if (header === null) return null;
6836
+ offset += header.headerBytes + header.bodyBytes;
6837
+ if (offset > view.byteLength) return null;
6838
+ pending += header.children - 1;
6144
6839
  }
6145
- const document = decoded;
6146
- if (discardedFields !== void 0) {
6147
- for (const fieldPath of discardedFields) removeFieldAtPath(document, fieldPath);
6840
+ return offset;
6841
+ }
6842
+ function stringKeyAt(bytes, view, offset) {
6843
+ const header = headerAt(view, offset);
6844
+ if (header === null || header.children !== 0) return null;
6845
+ const tag = view.getUint8(offset);
6846
+ const isString = tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
6847
+ const end = offset + header.headerBytes + header.bodyBytes;
6848
+ if (!isString || end > bytes.length) return null;
6849
+ try {
6850
+ return { key: keyDecoder.decode(bytes.subarray(offset + header.headerBytes, end)), end };
6851
+ } catch {
6852
+ return null;
6148
6853
  }
6149
- return document;
6150
6854
  }
6151
- function* storedDocumentsOf(data) {
6152
- const count = data.offsets.length - 1;
6153
- for (let ordinal = 0; ordinal < count; ordinal++) {
6154
- yield decodeStoredDocument(
6155
- data.blob.subarray(data.offsets[ordinal], data.offsets[ordinal + 1]),
6855
+ function topLevelMapKeys(bytes) {
6856
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6857
+ const top = headerAt(view, 0);
6858
+ if (top === null || top.bodyBytes !== 0) return null;
6859
+ const tag = view.getUint8(0);
6860
+ if (!(tag >= 128 && tag <= 143 || MAP_TAGS.has(tag))) return null;
6861
+ const keys = /* @__PURE__ */ new Set();
6862
+ let offset = top.headerBytes;
6863
+ for (let pair = 0; pair < top.children / 2; pair += 1) {
6864
+ const key = stringKeyAt(bytes, view, offset);
6865
+ if (key === null) return null;
6866
+ const valueEnd = endOfValue(view, key.end);
6867
+ if (valueEnd === null) return null;
6868
+ keys.add(key.key);
6869
+ offset = valueEnd;
6870
+ }
6871
+ return keys;
6872
+ }
6873
+ function decodeMapWithoutFields(bytes, skippedFields) {
6874
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6875
+ const top = headerAt(view, 0);
6876
+ if (top === null || top.children === 0 || top.bodyBytes !== 0) return null;
6877
+ const tag = view.getUint8(0);
6878
+ const isMap = tag >= 128 && tag <= 143 || MAP_TAGS.has(tag);
6879
+ if (!isMap) return null;
6880
+ const document = {};
6881
+ let offset = top.headerBytes;
6882
+ for (let pair = 0; pair < top.children / 2; pair += 1) {
6883
+ const key = stringKeyAt(bytes, view, offset);
6884
+ if (key === null || FORBIDDEN_KEYS.has(key.key)) return null;
6885
+ const valueEnd = endOfValue(view, key.end);
6886
+ if (valueEnd === null) return null;
6887
+ if (!skippedFields.has(key.key)) {
6888
+ try {
6889
+ document[key.key] = decode(bytes.subarray(key.end, valueEnd));
6890
+ } catch {
6891
+ return null;
6892
+ }
6893
+ }
6894
+ offset = valueEnd;
6895
+ }
6896
+ return offset === bytes.length ? document : null;
6897
+ }
6898
+
6899
+ // src/serialization/msgpack-path.ts
6900
+ var encoder = new TextEncoder();
6901
+ var PROTO_KEY_BYTES = encoder.encode("__proto__");
6902
+ var MISSING = { found: false, value: void 0 };
6903
+ var pathDecoder = new Decoder();
6904
+ function encodePathSegments(segments) {
6905
+ if (segments.length === 0) return null;
6906
+ if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) return null;
6907
+ return segments.map((segment) => encoder.encode(segment));
6908
+ }
6909
+ function isStringTag(tag) {
6910
+ return tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
6911
+ }
6912
+ function isMapTag(tag) {
6913
+ return tag >= 128 && tag <= 143 || tag === 222 || tag === 223;
6914
+ }
6915
+ function isArrayTag(tag) {
6916
+ return tag >= 144 && tag <= 159 || tag === 220 || tag === 221;
6917
+ }
6918
+ function isScalarTag(tag) {
6919
+ if (tag <= 127 || tag >= 224 || isStringTag(tag)) return true;
6920
+ return tag === 192 || tag === 194 || tag === 195 || tag >= 202 && tag <= 211;
6921
+ }
6922
+ function sameBytes(bytes, start, end, target) {
6923
+ if (end - start !== target.length) return false;
6924
+ for (let at = 0; at < target.length; at++) {
6925
+ if (bytes[start + at] !== target[at]) return false;
6926
+ }
6927
+ return true;
6928
+ }
6929
+ function keyRangeAt(view, offset) {
6930
+ if (!isStringTag(view.getUint8(offset))) return null;
6931
+ const header = headerAt(view, offset);
6932
+ if (header === null) return null;
6933
+ const start = offset + header.headerBytes;
6934
+ return { start, end: start + header.bodyBytes };
6935
+ }
6936
+ function readEncodedPath(bytes, segments, arrays) {
6937
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6938
+ let start = 0;
6939
+ let end = bytes.length;
6940
+ for (let depth = 0; depth < segments.length; depth++) {
6941
+ if (start >= end) return null;
6942
+ const tag = view.getUint8(start);
6943
+ if (!isMapTag(tag)) {
6944
+ if (depth === 0) return null;
6945
+ if (isArrayTag(tag)) return arrays === "missing" ? MISSING : null;
6946
+ return isScalarTag(tag) ? MISSING : null;
6947
+ }
6948
+ const header = headerAt(view, start);
6949
+ if (header === null) return null;
6950
+ let offset = start + header.headerBytes;
6951
+ let valueStart = -1;
6952
+ let valueEnd = -1;
6953
+ for (let pair = 0; pair < header.children / 2; pair++) {
6954
+ if (offset >= end) return null;
6955
+ const key = keyRangeAt(view, offset);
6956
+ if (key === null || key.end > end) return null;
6957
+ if (sameBytes(bytes, key.start, key.end, PROTO_KEY_BYTES)) return null;
6958
+ const next = endOfValue(view, key.end);
6959
+ if (next === null || next > end) return null;
6960
+ if (sameBytes(bytes, key.start, key.end, segments[depth])) {
6961
+ valueStart = key.end;
6962
+ valueEnd = next;
6963
+ }
6964
+ offset = next;
6965
+ }
6966
+ if (offset !== end) return null;
6967
+ if (valueStart < 0) return MISSING;
6968
+ start = valueStart;
6969
+ end = valueEnd;
6970
+ }
6971
+ try {
6972
+ return { found: true, value: pathDecoder.decode(bytes.subarray(start, end)) };
6973
+ } catch {
6974
+ return null;
6975
+ }
6976
+ }
6977
+
6978
+ // src/core/partition/frozen/document-source.ts
6979
+ var encodedProjectionPaths = /* @__PURE__ */ new WeakMap();
6980
+ function encodedProjectionPathOf(path) {
6981
+ let segments = encodedProjectionPaths.get(path);
6982
+ if (segments === void 0) {
6983
+ segments = encodePathSegments(path);
6984
+ encodedProjectionPaths.set(path, segments);
6985
+ }
6986
+ return segments;
6987
+ }
6988
+ function sharesPrefix(left, right) {
6989
+ const shared = Math.min(left.length, right.length);
6990
+ for (let at = 0; at < shared; at++) {
6991
+ if (left[at] !== right[at]) return false;
6992
+ }
6993
+ return true;
6994
+ }
6995
+ function removeFieldAtPath(document, fieldPath) {
6996
+ const segments = fieldPath.split(".");
6997
+ let current = document;
6998
+ for (let i = 0; i < segments.length - 1; i += 1) {
6999
+ const next = current[segments[i]];
7000
+ if (next === null || typeof next !== "object" || Array.isArray(next)) return;
7001
+ current = next;
7002
+ }
7003
+ delete current[segments[segments.length - 1]];
7004
+ }
7005
+ function decodeStoredDocument(bytes, discardedFields) {
7006
+ let decoded;
7007
+ try {
7008
+ decoded = decode(bytes);
7009
+ } catch (error) {
7010
+ throw new NarsilError(
7011
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
7012
+ `A stored document is not MessagePack: ${error instanceof Error ? error.message : String(error)}`
7013
+ );
7014
+ }
7015
+ if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded) || decoded instanceof Uint8Array) {
7016
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "A stored document decodes to something other than a map");
7017
+ }
7018
+ const document = decoded;
7019
+ if (discardedFields !== void 0) {
7020
+ for (const fieldPath of discardedFields) removeFieldAtPath(document, fieldPath);
7021
+ }
7022
+ return document;
7023
+ }
7024
+ function* storedDocumentsOf(data) {
7025
+ const count = data.offsets.length - 1;
7026
+ for (let ordinal = 0; ordinal < count; ordinal++) {
7027
+ yield decodeStoredDocument(
7028
+ data.blob.subarray(data.offsets[ordinal], data.offsets[ordinal + 1]),
6156
7029
  data.discardedFields
6157
7030
  );
6158
7031
  }
@@ -6184,25 +7057,40 @@ function encodeDocumentTableData(documents) {
6184
7057
  function wrapEncodedDocumentTable(data) {
6185
7058
  const { blob, offsets } = data;
6186
7059
  const count = offsets.length - 1;
6187
- const decoded = new Array(count);
6188
- const encodedBytes = blob.byteLength + offsets.byteLength;
6189
- let decodedBytes = 0;
7060
+ const discarded = (data.discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
7061
+ const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
7062
+ const bytesAt = (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]);
7063
+ const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
7064
+ function projectedPathAt(ordinal, path) {
7065
+ const segments = encodedProjectionPathOf(path);
7066
+ if (segments === null) return { found: false, value: void 0 };
7067
+ const read = touchesDiscarded(path) ? null : readEncodedPath(bytesAt(ordinal), segments, "missing");
7068
+ if (read === null) return readProjectedPath(docAt(ordinal), path);
7069
+ return read.found ? { found: true, value: valueWithOwnBuffers(read.value) } : read;
7070
+ }
6190
7071
  return {
6191
7072
  count,
6192
- get byteLength() {
6193
- return encodedBytes + decodedBytes;
6194
- },
6195
- docAt(ordinal) {
6196
- let document = decoded[ordinal];
6197
- if (document === void 0) {
6198
- document = decodeStoredDocument(blob.slice(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
6199
- decoded[ordinal] = document;
6200
- decodedBytes += documentValueBytes(document);
6201
- }
6202
- return document;
6203
- },
7073
+ byteLength: blob.byteLength + offsets.byteLength,
7074
+ docAt,
6204
7075
  readAt(ordinal) {
6205
- return decoded[ordinal] ?? decodeStoredDocument(blob.subarray(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
7076
+ return decodeStoredDocument(bytesAt(ordinal), data.discardedFields);
7077
+ },
7078
+ fieldReader(fieldPath) {
7079
+ const path = fieldPath.split(".");
7080
+ const segments = encodePathSegments(path);
7081
+ if (segments === null) return () => void 0;
7082
+ const decodesDocument = touchesDiscarded(path);
7083
+ return (ordinal) => {
7084
+ const read = decodesDocument ? null : readEncodedPath(bytesAt(ordinal), segments, "decode-document");
7085
+ if (read !== null) return valueWithOwnBuffers(read.value);
7086
+ return getNestedValue(docAt(ordinal), fieldPath);
7087
+ };
7088
+ },
7089
+ projectedAt(ordinal, projection) {
7090
+ if (projection === void 0 || projection.kind === "full") return docAt(ordinal);
7091
+ if (projection.kind === "none") return {};
7092
+ if (projection.include === null) return cloneProjected(docAt(ordinal), projection);
7093
+ return projectIncludedPaths(projection.include, projection.exclude, (path) => projectedPathAt(ordinal, path));
6206
7094
  }
6207
7095
  };
6208
7096
  }
@@ -6219,6 +7107,12 @@ function wrapDocumentArray(documents) {
6219
7107
  },
6220
7108
  readAt(ordinal) {
6221
7109
  return documents[ordinal];
7110
+ },
7111
+ fieldReader(fieldPath) {
7112
+ return (ordinal) => getNestedValue(documents[ordinal], fieldPath);
7113
+ },
7114
+ projectedAt(ordinal, projection) {
7115
+ return cloneProjected(documents[ordinal], projection);
6222
7116
  }
6223
7117
  };
6224
7118
  }
@@ -6526,9 +7420,13 @@ function createFrozenEnumReader(entry) {
6526
7420
  }
6527
7421
 
6528
7422
  // src/core/partition/frozen/inverted-reader.ts
6529
- function createFrozenInvertedReader(table, views) {
7423
+ function createFrozenInvertedReader(table, views, counts) {
6530
7424
  function viewAtSorted(sortedIndex) {
6531
- return views.viewAt(table.payloadSlot(sortedIndex), table.documentFrequencyAt(sortedIndex));
7425
+ return views.viewAt(table.payloadSlot(sortedIndex), sortedIndex);
7426
+ }
7427
+ function liveAt(token) {
7428
+ const at = table.find(token);
7429
+ return at < 0 || counts.documentFrequency(at) === 0 ? -1 : at;
6532
7430
  }
6533
7431
  function candidateRange(prefix) {
6534
7432
  if (prefix.length === 0) return { start: 0, end: table.size };
@@ -6536,22 +7434,22 @@ function createFrozenInvertedReader(table, views) {
6536
7434
  }
6537
7435
  return {
6538
7436
  lookup(token) {
6539
- const at = table.find(token);
7437
+ const at = liveAt(token);
6540
7438
  if (at < 0) return void 0;
6541
7439
  return viewAtSorted(at);
6542
7440
  },
6543
7441
  lookupReadOnce(token) {
6544
- const at = table.find(token);
7442
+ const at = liveAt(token);
6545
7443
  if (at < 0) return void 0;
6546
- return views.viewReadOnce(table.payloadSlot(at), table.documentFrequencyAt(at));
7444
+ return views.viewReadOnce(table.payloadSlot(at), at);
6547
7445
  },
6548
7446
  documentFrequency(token) {
6549
7447
  const at = table.find(token);
6550
- return at < 0 ? 0 : table.documentFrequencyAt(at);
7448
+ return at < 0 ? 0 : counts.documentFrequency(at);
6551
7449
  },
6552
7450
  fuzzyLookup(token, tolerance, prefixLength) {
6553
7451
  if (tolerance === 0) {
6554
- const at = table.find(token);
7452
+ const at = liveAt(token);
6555
7453
  return at < 0 ? [] : [{ token, postingList: viewAtSorted(at) }];
6556
7454
  }
6557
7455
  const prefix = fuzzyPrefixOf(token, prefixLength);
@@ -6562,7 +7460,7 @@ function createFrozenInvertedReader(table, views) {
6562
7460
  const candidate = table.tokenAt(at);
6563
7461
  if (!candidate.startsWith(requiredPrefix)) continue;
6564
7462
  const { withinTolerance } = boundedLevenshtein(token, candidate, tolerance);
6565
- if (withinTolerance) {
7463
+ if (withinTolerance && counts.documentFrequency(at) > 0) {
6566
7464
  results.push({ token: candidate, postingList: viewAtSorted(at) });
6567
7465
  }
6568
7466
  }
@@ -6573,14 +7471,15 @@ function createFrozenInvertedReader(table, views) {
6573
7471
  const range = table.prefixRange(prefix);
6574
7472
  const results = [];
6575
7473
  for (let at = range.start; at < range.end; at++) {
6576
- results.push({ term: table.tokenAt(at), documentFrequency: table.documentFrequencyAt(at) });
7474
+ const documentFrequency = counts.documentFrequency(at);
7475
+ if (documentFrequency > 0) results.push({ term: table.tokenAt(at), documentFrequency });
6577
7476
  }
6578
7477
  results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.term, b.term));
6579
7478
  if (results.length > limit) results.length = limit;
6580
7479
  return results;
6581
7480
  },
6582
7481
  has(token) {
6583
- return table.find(token) >= 0;
7482
+ return liveAt(token) >= 0;
6584
7483
  },
6585
7484
  *tokens() {
6586
7485
  for (let at = 0; at < table.size; at++) {
@@ -6592,6 +7491,171 @@ function createFrozenInvertedReader(table, views) {
6592
7491
  }
6593
7492
  };
6594
7493
  }
7494
+ function processLimitBytes() {
7495
+ const hostBytes = totalmem();
7496
+ const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
7497
+ return Number.isFinite(constrained) && constrained > 0 && constrained < hostBytes ? constrained : hostBytes;
7498
+ }
7499
+ function readResidentMemory() {
7500
+ try {
7501
+ const residentBytes = process.memoryUsage.rss();
7502
+ const limitBytes = processLimitBytes();
7503
+ if (!Number.isFinite(residentBytes) || !Number.isFinite(limitBytes) || limitBytes <= 0) return null;
7504
+ return { residentBytes, limitBytes };
7505
+ } catch {
7506
+ return null;
7507
+ }
7508
+ }
7509
+
7510
+ // src/core/partition/frozen/live-counts.ts
7511
+ var UINT32_BYTES = 4;
7512
+ var FLOAT64_BYTES = 8;
7513
+ function removalIndexBytes(columns, terms) {
7514
+ const documentWords = 2 * (columns.documentCount + 1);
7515
+ return (columns.postingDocIds.length + documentWords + 2 * terms) * UINT32_BYTES + terms * FLOAT64_BYTES;
7516
+ }
7517
+ function processHasRoomFor(bytes) {
7518
+ const memory = readResidentMemory();
7519
+ if (memory === null) return true;
7520
+ return memory.residentBytes + bytes <= FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION * memory.limitBytes;
7521
+ }
7522
+ function buildRemovalIndex(columns, table) {
7523
+ const { documentCount, postingDocIds } = columns;
7524
+ const starts = new Uint32Array(documentCount + 1);
7525
+ for (let row = 0; row < postingDocIds.length; row++) starts[postingDocIds[row] + 1]++;
7526
+ for (let ordinal = 0; ordinal < documentCount; ordinal++) starts[ordinal + 1] += starts[ordinal];
7527
+ const cursors = starts.slice(0, documentCount);
7528
+ const rows = new Uint32Array(postingDocIds.length);
7529
+ for (let row = 0; row < postingDocIds.length; row++) rows[cursors[postingDocIds[row]]++] = row;
7530
+ const termOfSlot = new Uint32Array(table.size);
7531
+ for (let sortedIndex = 0; sortedIndex < table.size; sortedIndex++) {
7532
+ termOfSlot[table.payloadSlot(sortedIndex)] = sortedIndex;
7533
+ }
7534
+ return {
7535
+ starts,
7536
+ rows,
7537
+ termOfSlot,
7538
+ documents: new Uint32Array(table.size),
7539
+ occurrences: new Float64Array(table.size)
7540
+ };
7541
+ }
7542
+ function tryBuildRemovalIndex(columns, table) {
7543
+ if (!processHasRoomFor(removalIndexBytes(columns, table.size))) return null;
7544
+ try {
7545
+ return buildRemovalIndex(columns, table);
7546
+ } catch (error) {
7547
+ if (error instanceof RangeError) return null;
7548
+ throw error;
7549
+ }
7550
+ }
7551
+ function slotOfRow(postingOffsets, slots, row) {
7552
+ let low = 0;
7553
+ let high = slots;
7554
+ while (low < high) {
7555
+ const middle = low + high >>> 1;
7556
+ if (postingOffsets[middle + 1] <= row) low = middle + 1;
7557
+ else high = middle;
7558
+ }
7559
+ return low;
7560
+ }
7561
+ function createFrozenLiveCounts(columns, table, tombstones, storedFrequencies) {
7562
+ const { postingOffsets, postingDocIds, postingFrequencies } = columns;
7563
+ let index = null;
7564
+ let lengths = null;
7565
+ let frequencies = null;
7566
+ let frequenciesRevision = -1;
7567
+ function countRemoval(built, ordinal) {
7568
+ const { starts, rows, termOfSlot, documents, occurrences } = built;
7569
+ let previousSlot = -1;
7570
+ for (let at = starts[ordinal]; at < starts[ordinal + 1]; at++) {
7571
+ const row = rows[at];
7572
+ const slot = slotOfRow(postingOffsets, table.size, row);
7573
+ const sortedIndex = termOfSlot[slot];
7574
+ if (slot !== previousSlot) documents[sortedIndex]++;
7575
+ occurrences[sortedIndex] += postingFrequencies[row];
7576
+ previousSlot = slot;
7577
+ }
7578
+ }
7579
+ function removedDocumentsOf(sortedIndex) {
7580
+ const slot = table.payloadSlot(sortedIndex);
7581
+ const end = postingOffsets[slot + 1];
7582
+ let removed = 0;
7583
+ let previous = -1;
7584
+ let ordered = true;
7585
+ for (let row = postingOffsets[slot]; row < end; row++) {
7586
+ const ordinal = postingDocIds[row];
7587
+ if (ordinal < previous) ordered = false;
7588
+ if (ordinal !== previous && tombstones.has(ordinal)) removed++;
7589
+ previous = ordinal;
7590
+ }
7591
+ if (ordered) return removed;
7592
+ const seen = /* @__PURE__ */ new Set();
7593
+ for (let row = postingOffsets[slot]; row < end; row++) {
7594
+ const ordinal = postingDocIds[row];
7595
+ if (tombstones.has(ordinal)) seen.add(ordinal);
7596
+ }
7597
+ return seen.size;
7598
+ }
7599
+ function removedOccurrencesOf(sortedIndex) {
7600
+ const slot = table.payloadSlot(sortedIndex);
7601
+ const end = postingOffsets[slot + 1];
7602
+ let removed = 0;
7603
+ for (let row = postingOffsets[slot]; row < end; row++) {
7604
+ if (tombstones.has(postingDocIds[row])) removed += postingFrequencies[row];
7605
+ }
7606
+ return removed;
7607
+ }
7608
+ function documentFrequency(sortedIndex) {
7609
+ const stored = table.documentFrequencyAt(sortedIndex);
7610
+ if (tombstones.size === 0) return stored;
7611
+ if (index !== null) return stored - index.documents[sortedIndex];
7612
+ return stored - removedDocumentsOf(sortedIndex);
7613
+ }
7614
+ return {
7615
+ totalDocuments() {
7616
+ return columns.documentCount - tombstones.size;
7617
+ },
7618
+ totalFieldLengths() {
7619
+ return lengths ?? columns.totalFieldLengths;
7620
+ },
7621
+ documentFrequency,
7622
+ removedOccurrences(sortedIndex) {
7623
+ if (tombstones.size === 0) return 0;
7624
+ if (index !== null) return index.occurrences[sortedIndex];
7625
+ return removedOccurrencesOf(sortedIndex);
7626
+ },
7627
+ docFrequencies() {
7628
+ if (tombstones.size === 0) return storedFrequencies();
7629
+ if (frequencies !== null && frequenciesRevision === tombstones.revision) return frequencies;
7630
+ const live = /* @__PURE__ */ Object.create(null);
7631
+ for (let at = 0; at < table.size; at++) {
7632
+ const documents = documentFrequency(at);
7633
+ if (documents > 0) live[table.tokenAt(at)] = documents;
7634
+ }
7635
+ frequencies = live;
7636
+ frequenciesRevision = tombstones.revision;
7637
+ return frequencies;
7638
+ },
7639
+ subtractRemoved(ordinal) {
7640
+ lengths ??= { ...columns.totalFieldLengths };
7641
+ for (let field = 0; field < columns.fieldLengthNames.length; field++) {
7642
+ lengths[columns.fieldLengthNames[field]] -= columns.fieldLengthColumns[field][ordinal];
7643
+ }
7644
+ if (index !== null) {
7645
+ countRemoval(index, ordinal);
7646
+ return;
7647
+ }
7648
+ const built = tryBuildRemovalIndex(columns, table);
7649
+ if (built === null) return;
7650
+ for (const removed of tombstones.values()) countRemoval(built, removed);
7651
+ index = built;
7652
+ },
7653
+ bytes() {
7654
+ if (index === null) return 0;
7655
+ return index.starts.byteLength + index.rows.byteLength + index.termOfSlot.byteLength + index.documents.byteLength + index.occurrences.byteLength;
7656
+ }
7657
+ };
7658
+ }
6595
7659
 
6596
7660
  // src/core/partition/frozen/posting-views.ts
6597
7661
  function isOrdered(docIds) {
@@ -6605,16 +7669,16 @@ function sumFrequencies(frequencies) {
6605
7669
  for (let i = 0; i < frequencies.length; i++) total += frequencies[i];
6606
7670
  return total;
6607
7671
  }
6608
- function createFrozenPostingViews(payload, tombstones) {
7672
+ function createFrozenPostingViews(payload, tombstones, counts) {
6609
7673
  const views = /* @__PURE__ */ new Map();
6610
- function buildView(payloadSlot, documentFrequency, keepsPositions) {
7674
+ function buildView(payloadSlot, sortedIndex, keepsPositions) {
6611
7675
  const start = payload.postingOffsets[payloadSlot];
6612
7676
  const end = payload.postingOffsets[payloadSlot + 1];
6613
7677
  const docIds = payload.postingDocIds.subarray(start, end);
6614
7678
  const termFrequencies = payload.postingFrequencies.subarray(start, end);
6615
7679
  const fieldNameIndices = payload.postingFieldIndices.subarray(start, end);
6616
7680
  const ordered = isOrdered(docIds);
6617
- const totalTermFrequency = sumFrequencies(termFrequencies);
7681
+ const storedTermFrequency = sumFrequencies(termFrequencies);
6618
7682
  const { positionOffsets, positionValues } = payload;
6619
7683
  let materializedPositions = null;
6620
7684
  return {
@@ -6636,9 +7700,15 @@ function createFrozenPostingViews(payload, tombstones) {
6636
7700
  if (positionOffsets === null) return 0;
6637
7701
  return positionOffsets[start + row + 1] - positionOffsets[start + row];
6638
7702
  },
6639
- docIdSet: { size: documentFrequency },
7703
+ docIdSet: {
7704
+ get size() {
7705
+ return counts.documentFrequency(sortedIndex);
7706
+ }
7707
+ },
6640
7708
  deletedDocs: tombstones,
6641
- totalTermFrequency,
7709
+ get totalTermFrequency() {
7710
+ return storedTermFrequency - counts.removedOccurrences(sortedIndex);
7711
+ },
6642
7712
  get structureRevision() {
6643
7713
  return tombstones.revision;
6644
7714
  },
@@ -6646,16 +7716,138 @@ function createFrozenPostingViews(payload, tombstones) {
6646
7716
  };
6647
7717
  }
6648
7718
  return {
6649
- viewAt(payloadSlot, documentFrequency) {
7719
+ viewAt(payloadSlot, sortedIndex) {
6650
7720
  let view = views.get(payloadSlot);
6651
7721
  if (view === void 0) {
6652
- view = buildView(payloadSlot, documentFrequency, true);
7722
+ view = buildView(payloadSlot, sortedIndex, true);
6653
7723
  views.set(payloadSlot, view);
6654
7724
  }
6655
7725
  return view;
6656
7726
  },
6657
- viewReadOnce(payloadSlot, documentFrequency) {
6658
- return buildView(payloadSlot, documentFrequency, false);
7727
+ viewReadOnce(payloadSlot, sortedIndex) {
7728
+ return buildView(payloadSlot, sortedIndex, false);
7729
+ }
7730
+ };
7731
+ }
7732
+
7733
+ // src/core/partition/frozen/removal-pass.ts
7734
+ var NO_REMOVED_SPELLINGS = [];
7735
+ function createFrozenRemovalPass(documents, surfaces) {
7736
+ let bound = null;
7737
+ let pending = [];
7738
+ function heldSpellingsOf(ordinal, analysis) {
7739
+ const spellings = [];
7740
+ if (!surfaces.holdsAny()) return spellings;
7741
+ visitDocumentSurfaceSpellings(documents.documentAt(ordinal), analysis, (spelling) => {
7742
+ if (surfaces.holds(spelling)) spellings.push(spelling);
7743
+ });
7744
+ return spellings;
7745
+ }
7746
+ return {
7747
+ get settled() {
7748
+ return pending.length === 0;
7749
+ },
7750
+ bind(analysis) {
7751
+ if (bound !== null) return;
7752
+ const scoped = { ...analysis, fields: documents.textFields };
7753
+ bound = scoped;
7754
+ for (const ordinal of pending) surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, scoped));
7755
+ pending = [];
7756
+ },
7757
+ spellingsOf(ordinal) {
7758
+ return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
7759
+ },
7760
+ afterRemoval(ordinal, spellings) {
7761
+ if (spellings !== void 0) {
7762
+ surfaces.subtractSpellings(ordinal, spellings);
7763
+ } else if (bound !== null) {
7764
+ surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, bound));
7765
+ } else {
7766
+ pending.push(ordinal);
7767
+ }
7768
+ }
7769
+ };
7770
+ }
7771
+
7772
+ // src/core/partition/frozen/surface-slot.ts
7773
+ function applyDocumentSurfaces(target, document, analysis, direction) {
7774
+ if (direction === -1) {
7775
+ visitDocumentSurfaceSpellings(document, analysis, (spelling) => target.subtract(spelling, 1));
7776
+ return;
7777
+ }
7778
+ visitDocumentSpellings(document, analysis, (token, spelling) => {
7779
+ if (spelling !== token) target.add(spelling, token, 1);
7780
+ });
7781
+ }
7782
+ function addForms(target, forms) {
7783
+ for (const surface of Object.keys(forms)) {
7784
+ const value = forms[surface];
7785
+ if (Array.isArray(value)) target.add(surface, value[1], value[0]);
7786
+ }
7787
+ }
7788
+ function createFrozenSurfaceSlot(load, documents) {
7789
+ let registry = null;
7790
+ let uncounted = null;
7791
+ function loaded() {
7792
+ if (registry === null) {
7793
+ const built = createSurfaceRegistry();
7794
+ const forms = load();
7795
+ if (forms !== null) built.deserialize(forms);
7796
+ registry = built;
7797
+ }
7798
+ return registry;
7799
+ }
7800
+ function isCounted(ordinal) {
7801
+ return uncounted === null || uncounted[ordinal] === 0;
7802
+ }
7803
+ return {
7804
+ reader: {
7805
+ candidatesForPrefix: (prefix) => loaded().candidatesForPrefix(prefix),
7806
+ stemChangedTotalFor: (token) => loaded().stemChangedTotalFor(token),
7807
+ size: () => loaded().size(),
7808
+ serialize: () => loaded().serialize()
7809
+ },
7810
+ holdsAny() {
7811
+ return loaded().size() > 0;
7812
+ },
7813
+ holds(spelling) {
7814
+ return loaded().holds(spelling);
7815
+ },
7816
+ subtractSpellings(ordinal, spellings) {
7817
+ uncounted ??= new Uint8Array(documents.count);
7818
+ uncounted[ordinal] = 1;
7819
+ if (spellings.length === 0) return;
7820
+ const target = loaded();
7821
+ for (const spelling of spellings) target.subtract(spelling, 1);
7822
+ },
7823
+ addKeptForms(target, mergedOrdinals, analysis) {
7824
+ const source = loaded();
7825
+ if (analysis === null) {
7826
+ addForms(target, source.serialize());
7827
+ return;
7828
+ }
7829
+ const scoped = { ...analysis, fields: documents.textFields };
7830
+ let kept = 0;
7831
+ let corrections = 0;
7832
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
7833
+ const keeps = mergedOrdinals[ordinal] >= 0;
7834
+ if (keeps) kept++;
7835
+ if (keeps !== isCounted(ordinal)) corrections++;
7836
+ }
7837
+ if (corrections > kept) {
7838
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
7839
+ if (mergedOrdinals[ordinal] >= 0) applyDocumentSurfaces(target, documents.documentAt(ordinal), scoped, 1);
7840
+ }
7841
+ return;
7842
+ }
7843
+ const corrected = createSurfaceRegistry();
7844
+ addForms(corrected, source.serialize());
7845
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
7846
+ const keeps = mergedOrdinals[ordinal] >= 0;
7847
+ if (keeps === isCounted(ordinal)) continue;
7848
+ applyDocumentSurfaces(corrected, documents.documentAt(ordinal), scoped, keeps ? 1 : -1);
7849
+ }
7850
+ addForms(target, corrected.serialize());
6659
7851
  }
6660
7852
  };
6661
7853
  }
@@ -6683,27 +7875,9 @@ function decodeSurfaceTable(table) {
6683
7875
  }
6684
7876
  return forms;
6685
7877
  }
6686
- function createLazySurfaceReader(load) {
6687
- let registry = null;
6688
- function loaded() {
6689
- if (registry === null) {
6690
- const built = createSurfaceRegistry();
6691
- const forms = load();
6692
- if (forms !== null) built.deserialize(forms);
6693
- registry = built;
6694
- }
6695
- return registry;
6696
- }
6697
- return {
6698
- candidatesForPrefix: (prefix) => loaded().candidatesForPrefix(prefix),
6699
- stemChangedTotalFor: (token) => loaded().stemChangedTotalFor(token),
6700
- size: () => loaded().size(),
6701
- serialize: () => loaded().serialize()
6702
- };
6703
- }
6704
7878
 
6705
7879
  // src/core/partition/frozen/token-table.ts
6706
- var encoder = new TextEncoder();
7880
+ var encoder2 = new TextEncoder();
6707
7881
  var decoder3 = new TextDecoder();
6708
7882
  function compareBytes2(blob, start, end, query) {
6709
7883
  const length = end - start;
@@ -6790,7 +7964,7 @@ function wrapFrozenTokenTable(data) {
6790
7964
  return documentFrequencies[sortedIndex];
6791
7965
  },
6792
7966
  prefixRange(prefix) {
6793
- const bytes = encoder.encode(prefix);
7967
+ const bytes = encoder2.encode(prefix);
6794
7968
  const start = lowerBound3(bytes);
6795
7969
  return { start, end: prefixEnd(start, bytes) };
6796
7970
  },
@@ -6808,8 +7982,8 @@ function wrapFrozenTokenTable(data) {
6808
7982
  fromCodePoint = unit;
6809
7983
  toCodePoint = nextCodePoint(unit);
6810
7984
  }
6811
- const from = encoder.encode(String.fromCodePoint(fromCodePoint));
6812
- const to = encoder.encode(String.fromCodePoint(toCodePoint));
7985
+ const from = encoder2.encode(String.fromCodePoint(fromCodePoint));
7986
+ const to = encoder2.encode(String.fromCodePoint(toCodePoint));
6813
7987
  return { start: lowerBound3(from), end: lowerBound3(to) };
6814
7988
  }
6815
7989
  };
@@ -6821,6 +7995,7 @@ function buildFrozenTokenTable(tokens, docFrequencies) {
6821
7995
  // src/core/partition/frozen/tombstones.ts
6822
7996
  function createFrozenTombstones() {
6823
7997
  const removed = /* @__PURE__ */ new Set();
7998
+ const arrivals = [];
6824
7999
  let revision = 0;
6825
8000
  return {
6826
8001
  get size() {
@@ -6835,9 +8010,13 @@ function createFrozenTombstones() {
6835
8010
  add(internalId) {
6836
8011
  if (removed.has(internalId)) return false;
6837
8012
  removed.add(internalId);
8013
+ arrivals.push(internalId);
6838
8014
  revision++;
6839
8015
  return true;
6840
8016
  },
8017
+ at(arrival) {
8018
+ return arrivals[arrival];
8019
+ },
6841
8020
  values() {
6842
8021
  return removed.values();
6843
8022
  }
@@ -7008,26 +8187,36 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis =
7008
8187
  }
7009
8188
 
7010
8189
  // src/core/partition/frozen/index.ts
7011
- function buildStatsView(source) {
7012
- const averageFieldLengths = {};
7013
- if (source.documentCount > 0) {
7014
- for (const [fieldName, total] of Object.entries(source.totalFieldLengths)) {
7015
- averageFieldLengths[fieldName] = total / source.documentCount;
7016
- }
7017
- }
7018
- let materializedFrequencies = null;
8190
+ function buildStatsView(counts) {
7019
8191
  return {
7020
- totalDocuments: source.documentCount,
7021
- totalFieldLengths: source.totalFieldLengths,
7022
- averageFieldLengths,
7023
- get docFrequencies() {
7024
- if (materializedFrequencies === null) {
7025
- materializedFrequencies = source.docFrequencies();
8192
+ get totalDocuments() {
8193
+ return counts.totalDocuments();
8194
+ },
8195
+ get totalFieldLengths() {
8196
+ return counts.totalFieldLengths();
8197
+ },
8198
+ get averageFieldLengths() {
8199
+ const documents = counts.totalDocuments();
8200
+ const averages = {};
8201
+ if (documents > 0) {
8202
+ for (const [fieldName, total] of Object.entries(counts.totalFieldLengths())) {
8203
+ averages[fieldName] = total / documents;
8204
+ }
7026
8205
  }
7027
- return materializedFrequencies;
8206
+ return averages;
8207
+ },
8208
+ get docFrequencies() {
8209
+ return counts.docFrequencies();
7028
8210
  }
7029
8211
  };
7030
8212
  }
8213
+ function storedFrequenciesOf(source) {
8214
+ let materialized = null;
8215
+ return () => {
8216
+ if (materialized === null) materialized = source.docFrequencies();
8217
+ return materialized;
8218
+ };
8219
+ }
7031
8220
  function buildGeoReaders(entries) {
7032
8221
  const readers = /* @__PURE__ */ new Map();
7033
8222
  for (const entry of entries) {
@@ -7039,9 +8228,10 @@ function buildGeoReaders(entries) {
7039
8228
  }
7040
8229
  return readers;
7041
8230
  }
7042
- function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
8231
+ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
7043
8232
  const tombstones = createFrozenTombstones();
7044
- const postingViews = createFrozenPostingViews(source, tombstones);
8233
+ const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
8234
+ const postingViews = createFrozenPostingViews(source, tombstones, counts);
7045
8235
  const lengthColumns = /* @__PURE__ */ new Map();
7046
8236
  for (let i = 0; i < source.fieldLengthNames.length; i++) {
7047
8237
  lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
@@ -7052,6 +8242,14 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
7052
8242
  documentAt: (ordinal) => documentSource.readAt(ordinal),
7053
8243
  isTombstoned: (ordinal) => tombstones.has(ordinal)
7054
8244
  });
8245
+ const textDocuments = {
8246
+ count: documentSource.count,
8247
+ textFields: source.fieldLengthNames,
8248
+ documentAt: (ordinal) => documentSource.readAt(ordinal)
8249
+ };
8250
+ const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
8251
+ const surfaces = createFrozenSurfaceSlot(loadSurfaceForms, textDocuments);
8252
+ const removals = createFrozenRemovalPass(textDocuments, surfaces);
7055
8253
  const numericIndexes = /* @__PURE__ */ new Map();
7056
8254
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
7057
8255
  const sizeIndexes = /* @__PURE__ */ new Map();
@@ -7073,10 +8271,12 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
7073
8271
  documentSource,
7074
8272
  sharedSnapshot,
7075
8273
  arrays: source,
7076
- invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews),
8274
+ invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
7077
8275
  docStore,
7078
- stats: buildStatsView(source),
7079
- surfaceRegistry: createLazySurfaceReader(source.readSurfaceForms),
8276
+ stats: buildStatsView(counts),
8277
+ surfaceRegistry: surfaces.reader,
8278
+ surfaces,
8279
+ removals,
7080
8280
  numericIndexes,
7081
8281
  booleanIndexes,
7082
8282
  enumIndexes,
@@ -7099,15 +8299,21 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
7099
8299
  isTombstoned(ordinal) {
7100
8300
  return tombstones.has(ordinal);
7101
8301
  },
7102
- tombstoneDocument(docId) {
8302
+ removedSpellingsOf(docId) {
8303
+ const ordinal = docStore.getInternalId(docId);
8304
+ if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
8305
+ return removals.spellingsOf(ordinal);
8306
+ },
8307
+ tombstoneDocument(docId, removedSpellings) {
7103
8308
  const ordinal = docStore.getInternalId(docId);
7104
8309
  if (ordinal === void 0) return false;
7105
8310
  const added = tombstones.add(ordinal);
7106
8311
  if (added) {
8312
+ counts.subtractRemoved(ordinal);
7107
8313
  partialWords.afterRemoval(ordinal);
8314
+ removals.afterRemoval(ordinal, removedSpellings);
7108
8315
  docStore.releaseSortedDocIds();
7109
- segment.sortColumns = null;
7110
- segment.scoreBuffer = null;
8316
+ segment.sortColumns?.forget(ordinal);
7111
8317
  }
7112
8318
  return added;
7113
8319
  },
@@ -7115,6 +8321,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
7115
8321
  const docIds = [];
7116
8322
  for (const ordinal of tombstones.values()) docIds.push(source.idTable.idAt(ordinal));
7117
8323
  return docIds;
8324
+ },
8325
+ removalBytes() {
8326
+ return counts.bytes();
7118
8327
  }
7119
8328
  };
7120
8329
  return segment;
@@ -7136,7 +8345,7 @@ function createFrozenSegment(payload, documents, segmentId) {
7136
8345
  payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, tokenTable)
7137
8346
  );
7138
8347
  }
7139
- function createSharedFrozenSegment(snapshot) {
8348
+ function createSharedFrozenSegment(snapshot, survivingSurfaceForms) {
7140
8349
  const tokenTable = wrapFrozenTokenTable(snapshot.tokenTable);
7141
8350
  return assembleFrozenSegment(
7142
8351
  snapshot.segmentId,
@@ -7156,136 +8365,10 @@ function createSharedFrozenSegment(snapshot) {
7156
8365
  },
7157
8366
  wrapEncodedDocumentTable(snapshot.documentTable),
7158
8367
  snapshot,
7159
- snapshot.partialWords
8368
+ snapshot.partialWords,
8369
+ survivingSurfaceForms
7160
8370
  );
7161
8371
  }
7162
- var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
7163
- var keyDecoder = new TextDecoder("utf-8", { fatal: true });
7164
- function headerAt(view, offset) {
7165
- if (offset >= view.byteLength) return null;
7166
- const tag = view.getUint8(offset);
7167
- if (tag <= 127 || tag >= 224) return { headerBytes: 1, bodyBytes: 0, children: 0 };
7168
- if (tag <= 143) return { headerBytes: 1, bodyBytes: 0, children: (tag & 15) * 2 };
7169
- if (tag <= 159) return { headerBytes: 1, bodyBytes: 0, children: tag & 15 };
7170
- if (tag <= 191) return { headerBytes: 1, bodyBytes: tag & 31, children: 0 };
7171
- const lengthBytes = LENGTH_BYTES_BY_TAG.get(tag);
7172
- if (lengthBytes !== void 0) {
7173
- if (offset + 1 + lengthBytes > view.byteLength) return null;
7174
- const length = lengthBytes === 1 ? view.getUint8(offset + 1) : lengthBytes === 2 ? view.getUint16(offset + 1, false) : view.getUint32(offset + 1, false);
7175
- const headerBytes = 1 + lengthBytes;
7176
- if (ARRAY_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length };
7177
- if (MAP_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length * 2 };
7178
- return { headerBytes, bodyBytes: length + (EXT_TAGS.has(tag) ? 1 : 0), children: 0 };
7179
- }
7180
- const fixedBytes = FIXED_BODY_BYTES_BY_TAG.get(tag);
7181
- return fixedBytes === void 0 ? null : { headerBytes: 1, bodyBytes: fixedBytes, children: 0 };
7182
- }
7183
- var LENGTH_BYTES_BY_TAG = /* @__PURE__ */ new Map([
7184
- [196, 1],
7185
- [197, 2],
7186
- [198, 4],
7187
- [199, 1],
7188
- [200, 2],
7189
- [201, 4],
7190
- [217, 1],
7191
- [218, 2],
7192
- [219, 4],
7193
- [220, 2],
7194
- [221, 4],
7195
- [222, 2],
7196
- [223, 4]
7197
- ]);
7198
- var ARRAY_TAGS = /* @__PURE__ */ new Set([220, 221]);
7199
- var MAP_TAGS = /* @__PURE__ */ new Set([222, 223]);
7200
- var EXT_TAGS = /* @__PURE__ */ new Set([199, 200, 201]);
7201
- var FIXED_BODY_BYTES_BY_TAG = /* @__PURE__ */ new Map([
7202
- [192, 0],
7203
- [194, 0],
7204
- [195, 0],
7205
- [202, 4],
7206
- [203, 8],
7207
- [204, 1],
7208
- [205, 2],
7209
- [206, 4],
7210
- [207, 8],
7211
- [208, 1],
7212
- [209, 2],
7213
- [210, 4],
7214
- [211, 8],
7215
- [212, 2],
7216
- [213, 3],
7217
- [214, 5],
7218
- [215, 9],
7219
- [216, 17]
7220
- ]);
7221
- function endOfValue(view, start) {
7222
- let offset = start;
7223
- let pending = 1;
7224
- while (pending > 0) {
7225
- const header = headerAt(view, offset);
7226
- if (header === null) return null;
7227
- offset += header.headerBytes + header.bodyBytes;
7228
- if (offset > view.byteLength) return null;
7229
- pending += header.children - 1;
7230
- }
7231
- return offset;
7232
- }
7233
- function stringKeyAt(bytes, view, offset) {
7234
- const header = headerAt(view, offset);
7235
- if (header === null || header.children !== 0) return null;
7236
- const tag = view.getUint8(offset);
7237
- const isString = tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
7238
- const end = offset + header.headerBytes + header.bodyBytes;
7239
- if (!isString || end > bytes.length) return null;
7240
- try {
7241
- return { key: keyDecoder.decode(bytes.subarray(offset + header.headerBytes, end)), end };
7242
- } catch {
7243
- return null;
7244
- }
7245
- }
7246
- function topLevelMapKeys(bytes) {
7247
- const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
7248
- const top = headerAt(view, 0);
7249
- if (top === null || top.bodyBytes !== 0) return null;
7250
- const tag = view.getUint8(0);
7251
- if (!(tag >= 128 && tag <= 143 || MAP_TAGS.has(tag))) return null;
7252
- const keys = /* @__PURE__ */ new Set();
7253
- let offset = top.headerBytes;
7254
- for (let pair = 0; pair < top.children / 2; pair += 1) {
7255
- const key = stringKeyAt(bytes, view, offset);
7256
- if (key === null) return null;
7257
- const valueEnd = endOfValue(view, key.end);
7258
- if (valueEnd === null) return null;
7259
- keys.add(key.key);
7260
- offset = valueEnd;
7261
- }
7262
- return keys;
7263
- }
7264
- function decodeMapWithoutFields(bytes, skippedFields) {
7265
- const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
7266
- const top = headerAt(view, 0);
7267
- if (top === null || top.children === 0 || top.bodyBytes !== 0) return null;
7268
- const tag = view.getUint8(0);
7269
- const isMap = tag >= 128 && tag <= 143 || MAP_TAGS.has(tag);
7270
- if (!isMap) return null;
7271
- const document = {};
7272
- let offset = top.headerBytes;
7273
- for (let pair = 0; pair < top.children / 2; pair += 1) {
7274
- const key = stringKeyAt(bytes, view, offset);
7275
- if (key === null || FORBIDDEN_KEYS.has(key.key)) return null;
7276
- const valueEnd = endOfValue(view, key.end);
7277
- if (valueEnd === null) return null;
7278
- if (!skippedFields.has(key.key)) {
7279
- try {
7280
- document[key.key] = decode(bytes.subarray(key.end, valueEnd));
7281
- } catch {
7282
- return null;
7283
- }
7284
- }
7285
- offset = valueEnd;
7286
- }
7287
- return offset === bytes.length ? document : null;
7288
- }
7289
8372
 
7290
8373
  // src/core/partition/frozen/merge-fields.ts
7291
8374
  function indexByFieldPath(entries) {
@@ -7677,16 +8760,9 @@ function mergeFieldLengths(inputs, documentCount) {
7677
8760
  }
7678
8761
  return { fieldLengthNames: names, fieldLengthColumns: columns, totalFieldLengths: totals };
7679
8762
  }
7680
- function mergeSurfaceForms2(inputs) {
8763
+ function mergeSurfaceForms2(inputs, analysis) {
7681
8764
  const registry = createSurfaceRegistry();
7682
- for (const input of inputs) {
7683
- const forms = input.segment.arrays.readSurfaceForms();
7684
- if (forms === null) continue;
7685
- for (const surface of Object.keys(forms)) {
7686
- const value = forms[surface];
7687
- if (Array.isArray(value)) registry.add(surface, value[1], value[0]);
7688
- }
7689
- }
8765
+ for (const input of inputs) input.segment.surfaces.addKeptForms(registry, input.remap, analysis);
7690
8766
  return registry.size() === 0 ? null : registry.serialize();
7691
8767
  }
7692
8768
  function holdsADiscardedField(bytes, discardedFields) {
@@ -7756,15 +8832,16 @@ function mergePartialWords(inputs, mergedTokenCount, analysis) {
7756
8832
  for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
7757
8833
  return { spellings: encodePartialSpellingsOf(analysis.fields, counts), reused: null };
7758
8834
  }
7759
- function mergeFrozenSegments(segments, analysis) {
8835
+ function mergeFrozenSegments(segments, analysis, textAnalysis) {
7760
8836
  if (typeof SharedArrayBuffer !== "function") return null;
7761
8837
  const merged = mergeSegmentContents(
7762
8838
  segments.map((segment) => ({ segment })),
7763
- analysis
8839
+ analysis,
8840
+ textAnalysis
7764
8841
  );
7765
8842
  return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis, merged.reusedPartialWords);
7766
8843
  }
7767
- function mergeSegmentContents(selections, analysis = null) {
8844
+ function mergeSegmentContents(selections, analysis = null, textAnalysis = null) {
7768
8845
  const docIds = [];
7769
8846
  const fieldNames = [];
7770
8847
  const inputs = selections.map((selection) => ({
@@ -7791,7 +8868,7 @@ function mergeSegmentContents(selections, analysis = null) {
7791
8868
  fieldLengthColumns: lengths.fieldLengthColumns,
7792
8869
  totalFieldLengths: lengths.totalFieldLengths,
7793
8870
  docFrequencies: postings.docFrequencies,
7794
- surfaceForms: mergeSurfaceForms2(inputs),
8871
+ surfaceForms: mergeSurfaceForms2(inputs, textAnalysis),
7795
8872
  partialSpellings: partialWords.spellings,
7796
8873
  ...mergeFieldIndexes2(inputs)
7797
8874
  };
@@ -8163,22 +9240,29 @@ function runSegmentAction(entry, action) {
8163
9240
  case "attachSegments": {
8164
9241
  for (const segment of action.segments) {
8165
9242
  growPartitionsTo(entry.manager, segment.partitionId);
8166
- const frozen = createSharedFrozenSegment(segment.snapshot);
8167
- for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId);
9243
+ const surfaceForms = segment.survivingSurfaceForms;
9244
+ const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
9245
+ const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
9246
+ for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
8168
9247
  entry.manager.attachFrozenSegment(segment.partitionId, frozen);
8169
9248
  }
8170
9249
  return void 0;
8171
9250
  }
8172
9251
  case "compactSegments": {
8173
9252
  const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
8174
- return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds), entry.manager.partialWordAnalysis);
9253
+ return mergeFrozenSegments(
9254
+ partition.frozenSegmentsById(action.segmentIds),
9255
+ entry.manager.partialWordAnalysis,
9256
+ entry.manager.textAnalysis
9257
+ );
8175
9258
  }
8176
9259
  case "swapSegments": {
8177
9260
  requireComposite(entry, action.indexName, action.partitionId, "swap");
8178
9261
  entry.manager.swapFrozenSegments(
8179
9262
  action.partitionId,
8180
9263
  action.dropSegmentIds,
8181
- createSharedFrozenSegment(action.snapshot)
9264
+ createSharedFrozenSegment(action.snapshot),
9265
+ action.removedSpellings
8182
9266
  );
8183
9267
  return void 0;
8184
9268
  }
@@ -8421,12 +9505,12 @@ function createDirectExecutor(options) {
8421
9505
  }
8422
9506
  case "remove": {
8423
9507
  const entry = requireIndex(action.indexName);
8424
- entry.manager.remove(action.docId);
9508
+ entry.manager.remove(action.docId, action.removedSpellings);
8425
9509
  return void 0;
8426
9510
  }
8427
9511
  case "update": {
8428
9512
  const entry = requireIndex(action.indexName);
8429
- entry.manager.update(action.docId, action.document);
9513
+ entry.manager.update(action.docId, action.document, void 0, action.removedSpellings);
8430
9514
  return void 0;
8431
9515
  }
8432
9516
  case "query": {