@delali/narsil 0.3.1-next.87 → 0.3.1-next.88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
  2. package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
  3. package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
  4. package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
  5. package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
  6. package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
  7. package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
  8. package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
  9. package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
  10. package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
  11. package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
  12. package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
  13. package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
  14. package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
  15. package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
  16. package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
  17. package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
  18. package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
  19. package/dist/distribution.mjs +14 -27
  20. package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
  21. package/dist/index.browser.mjs +1 -1
  22. package/dist/index.mjs +5 -5
  23. package/dist/partial-words.mjs +1 -1
  24. package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
  25. package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
  26. package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
  27. package/dist/workers/entry.mjs +3 -3
  28. package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
  29. package/package.json +13 -8
@@ -1,5 +1,5 @@
1
- import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, toReducedSortValue, readSortField, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, resolveRequestThreadCount, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, defaultSortMode, holdsUnpairedSurrogate, MIN_SWEEP_INTERVAL_MS, MAX_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_INDEX_CONFIGURATION_BYTES, compareComparableValues, exceedsCodePoints, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, compareComparableKeys, sortModeOf, toComparableSortValue, DEFAULT_FUZZY_PREFIX_LENGTH, SORT_VALUE_MAX_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, truncateSortString, MAX_IMPORT_REFUSALS_REPORTED, MIN_OVERSAMPLE, DEFAULT_MAX_PER_GROUP, compareSortValues, isSortMode, SORT_MODES, MIN_DOCUMENTS_FOR_SEGMENTS, SHUTDOWN_TIMEOUT_MS, DEFAULT_PAGE_SIZE, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-FHFFWLEK.mjs';
2
- import { encode, decode } from '@msgpack/msgpack';
1
+ import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, toReducedSortValue, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, resolveRequestThreadCount, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, defaultSortMode, holdsUnpairedSurrogate, MIN_SWEEP_INTERVAL_MS, MAX_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_INDEX_CONFIGURATION_BYTES, compareComparableValues, readSortField, exceedsCodePoints, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, compareComparableKeys, sortModeOf, toComparableSortValue, DEFAULT_FUZZY_PREFIX_LENGTH, SORT_VALUE_MAX_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, truncateSortString, MAX_IMPORT_REFUSALS_REPORTED, MIN_OVERSAMPLE, DEFAULT_MAX_PER_GROUP, compareSortValues, isSortMode, SORT_MODES, MIN_DOCUMENTS_FOR_SEGMENTS, SHUTDOWN_TIMEOUT_MS, DEFAULT_PAGE_SIZE, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-FHFFWLEK.mjs';
2
+ import { Decoder, encode, decode } from '@msgpack/msgpack';
3
3
 
4
4
  // src/analysis/registry.ts
5
5
  var tokenizers = /* @__PURE__ */ new Map();
@@ -896,6 +896,17 @@ function visitTokens(text, language, options, visit) {
896
896
  visit(processed, position, surface === processed ? void 0 : surface, part);
897
897
  }
898
898
  }
899
+ function visitSurfaceSpellings(text, language, visit) {
900
+ const rawParts = splitText(normalizeForSplitting(text), language);
901
+ const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
902
+ const removeDiacritics = language.tokenizer?.normalizeDiacritics ?? false;
903
+ const stripPossessives = language.tokenizer?.stripPossessive ?? false;
904
+ for (const part of rawParts) {
905
+ const candidate = stripPossessives ? stripPossessive(part) : part;
906
+ if (candidate.length < minLength) continue;
907
+ visit(removeDiacritics ? transformToken(candidate, language, false, true) : candidate);
908
+ }
909
+ }
899
910
  function tokenize(text, language, options) {
900
911
  if (options?.customTokenizer) {
901
912
  const customResult = options.customTokenizer.tokenize(text);
@@ -2713,6 +2724,197 @@ function validateVectorPromotion(config) {
2713
2724
  }
2714
2725
  }
2715
2726
 
2727
+ // src/core/partition/utils.ts
2728
+ function getOrCreateFieldNameIndex(table, fieldName) {
2729
+ const existing = table.indexMap.get(fieldName);
2730
+ if (existing !== void 0) return existing;
2731
+ const idx = table.names.length;
2732
+ table.names.push(fieldName);
2733
+ table.indexMap.set(fieldName, idx);
2734
+ return idx;
2735
+ }
2736
+ var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
2737
+ function getNestedValue(obj, path) {
2738
+ const parts = path.split(".");
2739
+ let current = obj;
2740
+ for (let i = 0; i < parts.length; i++) {
2741
+ if (current === null || current === void 0 || typeof current !== "object") return void 0;
2742
+ if (!Object.hasOwn(current, parts[i])) return void 0;
2743
+ current = current[parts[i]];
2744
+ }
2745
+ return current;
2746
+ }
2747
+ function stringArraysEqual(a, b) {
2748
+ if (a === b) return true;
2749
+ if (!a || !b) return false;
2750
+ if (a.length !== b.length) return false;
2751
+ for (let i = 0; i < a.length; i++) {
2752
+ if (a[i] !== b[i]) return false;
2753
+ }
2754
+ return true;
2755
+ }
2756
+ function textFieldsChanged(oldDoc, newDoc, fields) {
2757
+ for (const { path, base } of fields) {
2758
+ if (!isWordIndexedBase(base)) continue;
2759
+ const oldVal = getNestedValue(oldDoc, path);
2760
+ const newVal = getNestedValue(newDoc, path);
2761
+ if (base === "string") {
2762
+ if (oldVal !== newVal) return true;
2763
+ } else {
2764
+ if (!stringArraysEqual(oldVal, newVal)) return true;
2765
+ }
2766
+ }
2767
+ return false;
2768
+ }
2769
+ function parseVectorDimension(fieldType) {
2770
+ const match = VECTOR_PATTERN2.exec(fieldType);
2771
+ return match ? Number.parseInt(match[1], 10) : null;
2772
+ }
2773
+ function tokenizeOptions(options) {
2774
+ return {
2775
+ stem: true,
2776
+ removeStopWords: true,
2777
+ collectSurfaces: options?.collectSurfaces === true,
2778
+ stopWordOverride: options?.stopWordOverride,
2779
+ customTokenizer: options?.customTokenizer
2780
+ };
2781
+ }
2782
+ function getFlatSchema(state, schema) {
2783
+ if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
2784
+ state.flatSchemaCache = flattenSchema(schema);
2785
+ state.lastSchemaRef = schema;
2786
+ return state.flatSchemaCache;
2787
+ }
2788
+
2789
+ // src/core/projection.ts
2790
+ function requireFieldPathList(paths, option) {
2791
+ if (paths === void 0) return;
2792
+ if (!Array.isArray(paths)) {
2793
+ throw new NarsilError(
2794
+ ErrorCodes.SEARCH_INVALID_FIELD,
2795
+ `The "document.${option}" option takes an array of field paths`,
2796
+ { option: `document.${option}`, received: paths === null ? "null" : typeof paths }
2797
+ );
2798
+ }
2799
+ for (let position = 0; position < paths.length; position++) {
2800
+ const entry = paths[position];
2801
+ if (typeof entry === "string") continue;
2802
+ throw new NarsilError(
2803
+ ErrorCodes.SEARCH_INVALID_FIELD,
2804
+ `The "document.${option}" option takes field paths as strings, and entry ${position} is not a string`,
2805
+ { option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
2806
+ );
2807
+ }
2808
+ }
2809
+ function splitPaths(paths) {
2810
+ if (!paths) return [];
2811
+ const split = [];
2812
+ for (const path of paths) {
2813
+ const segments = path.split(".").filter((segment) => segment.length > 0);
2814
+ if (segments.length === 0) continue;
2815
+ if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) continue;
2816
+ split.push(segments);
2817
+ }
2818
+ return split;
2819
+ }
2820
+ function overlaps(a, b) {
2821
+ const shared2 = Math.min(a.length, b.length);
2822
+ for (let i = 0; i < shared2; i++) {
2823
+ if (a[i] !== b[i]) return false;
2824
+ }
2825
+ return true;
2826
+ }
2827
+ function covers(outer, inner) {
2828
+ if (outer.length > inner.length) return false;
2829
+ for (let i = 0; i < outer.length; i++) {
2830
+ if (outer[i] !== inner[i]) return false;
2831
+ }
2832
+ return true;
2833
+ }
2834
+ function readProjectedPath(source, path) {
2835
+ let current = source;
2836
+ for (const segment of path) {
2837
+ if (typeof current !== "object" || current === null || Array.isArray(current))
2838
+ return { found: false, value: void 0 };
2839
+ const record = current;
2840
+ if (!Object.hasOwn(record, segment)) return { found: false, value: void 0 };
2841
+ current = record[segment];
2842
+ }
2843
+ return { found: true, value: current };
2844
+ }
2845
+ function writePath(target, path, value) {
2846
+ let current = target;
2847
+ for (let i = 0; i < path.length - 1; i++) {
2848
+ const segment = path[i];
2849
+ const existing = current[segment];
2850
+ if (typeof existing === "object" && existing !== null && !Array.isArray(existing)) {
2851
+ current = existing;
2852
+ continue;
2853
+ }
2854
+ const created = {};
2855
+ current[segment] = created;
2856
+ current = created;
2857
+ }
2858
+ current[path[path.length - 1]] = value;
2859
+ }
2860
+ function cloneExcept(value, path, exclude) {
2861
+ const reachesDeeper = exclude.some((excluded) => excluded.length > path.length && covers(path, excluded));
2862
+ if (!reachesDeeper) return structuredClone(value);
2863
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return structuredClone(value);
2864
+ const source = value;
2865
+ const kept = {};
2866
+ for (const key of Object.keys(source)) {
2867
+ const childPath = [...path, key];
2868
+ if (exclude.some((excluded) => covers(excluded, childPath))) continue;
2869
+ Object.defineProperty(kept, key, {
2870
+ value: cloneExcept(source[key], childPath, exclude),
2871
+ enumerable: true,
2872
+ writable: true,
2873
+ configurable: true
2874
+ });
2875
+ }
2876
+ return kept;
2877
+ }
2878
+ function resolveProjection(projection) {
2879
+ if (projection === void 0 || projection === true) return { kind: "full" };
2880
+ if (projection === false) return { kind: "none" };
2881
+ if (typeof projection !== "object" || projection === null) return { kind: "full" };
2882
+ requireFieldPathList(projection.include, "include");
2883
+ requireFieldPathList(projection.exclude, "exclude");
2884
+ const askedInclude = Array.isArray(projection.include) && projection.include.length > 0;
2885
+ const askedExclude = Array.isArray(projection.exclude) && projection.exclude.length > 0;
2886
+ if (!askedInclude && !askedExclude) return { kind: "full" };
2887
+ return {
2888
+ kind: "fields",
2889
+ include: askedInclude ? splitPaths(projection.include) : null,
2890
+ exclude: splitPaths(projection.exclude)
2891
+ };
2892
+ }
2893
+ function projectionKeepsField(projection, fieldPath) {
2894
+ if (projection.kind === "full") return true;
2895
+ if (projection.kind === "none") return false;
2896
+ const path = fieldPath.split(".").filter((segment) => segment.length > 0);
2897
+ if (path.length === 0) return false;
2898
+ if (projection.include && !projection.include.some((included) => overlaps(included, path))) return false;
2899
+ return !projection.exclude.some((excluded) => covers(excluded, path));
2900
+ }
2901
+ function cloneProjected(document, projection) {
2902
+ if (projection === void 0 || projection.kind === "full") return structuredClone(document);
2903
+ if (projection.kind === "none") return {};
2904
+ if (projection.include === null) return cloneExcept(document, [], projection.exclude);
2905
+ return projectIncludedPaths(projection.include, projection.exclude, (path) => readProjectedPath(document, path));
2906
+ }
2907
+ function projectIncludedPaths(include, exclude, read) {
2908
+ const projected = {};
2909
+ for (const path of include) {
2910
+ if (exclude.some((excluded) => covers(excluded, path))) continue;
2911
+ const { found, value } = read(path);
2912
+ if (!found) continue;
2913
+ writePath(projected, path, cloneExcept(value, path, exclude));
2914
+ }
2915
+ return projected;
2916
+ }
2917
+
2716
2918
  // src/core/document-store.ts
2717
2919
  function detachedView(view) {
2718
2920
  if (view.byteOffset === 0 && view.byteLength === view.buffer.byteLength) return view;
@@ -2737,7 +2939,7 @@ function sharesABiggerBuffer(value) {
2737
2939
  return false;
2738
2940
  }
2739
2941
  function withOwnBuffers(value) {
2740
- if (value === null || typeof value !== "object") return value;
2942
+ if (value === null || typeof value !== "object" || value instanceof Date) return value;
2741
2943
  if (ArrayBuffer.isView(value)) return detachedView(value);
2742
2944
  if (Array.isArray(value)) {
2743
2945
  const copy2 = new Array(value.length);
@@ -2762,6 +2964,9 @@ function documentWithOwnBuffers(document) {
2762
2964
  if (!sharesABiggerBuffer(document)) return document;
2763
2965
  return withOwnBuffers(document);
2764
2966
  }
2967
+ function valueWithOwnBuffers(value) {
2968
+ return sharesABiggerBuffer(value) ? withOwnBuffers(value) : value;
2969
+ }
2765
2970
  var STRING_HEADER_BYTES = 16;
2766
2971
  var TWO_BYTE_SAMPLE_LIMIT = 16;
2767
2972
  var NUMBER_BYTES = 16;
@@ -2959,12 +3164,22 @@ function createDocumentStore() {
2959
3164
  },
2960
3165
  resolver() {
2961
3166
  return idResolver;
3167
+ },
3168
+ fieldReader(fieldPath) {
3169
+ return (internalId) => {
3170
+ const docId = reverseMap[internalId];
3171
+ const stored = docId === void 0 ? void 0 : docs.get(docId);
3172
+ return stored === void 0 ? void 0 : getNestedValue(stored.fields, fieldPath);
3173
+ };
3174
+ },
3175
+ projectedDocument(docId, projection) {
3176
+ const stored = docs.get(docId);
3177
+ return stored === void 0 ? void 0 : cloneProjected(stored.fields, projection);
2962
3178
  }
2963
3179
  };
2964
3180
  }
2965
3181
 
2966
3182
  // src/core/constants.ts
2967
- var POSTING_LIST_INITIAL_CAPACITY = 8;
2968
3183
  var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
2969
3184
  var MAX_TERM_FREQUENCY = 65535;
2970
3185
 
@@ -3035,433 +3250,912 @@ function bandedDistance(a, b, tolerance) {
3035
3250
  return { distance, withinTolerance: distance <= tolerance, cells };
3036
3251
  }
3037
3252
 
3038
- // src/core/posting-list.ts
3039
- function createPostingList() {
3040
- return {
3041
- length: 0,
3042
- docIds: [],
3043
- termFrequencies: new Uint16Array(POSTING_LIST_INITIAL_CAPACITY),
3044
- fieldNameIndices: new Uint16Array(POSTING_LIST_INITIAL_CAPACITY),
3045
- positions: null,
3046
- docIdSet: /* @__PURE__ */ new Set(),
3047
- deletedDocs: /* @__PURE__ */ new Set(),
3048
- totalTermFrequency: 0,
3049
- structureRevision: 0,
3050
- ordered: true
3051
- };
3052
- }
3053
- function growTypedArrays(list) {
3054
- const newCap = list.termFrequencies.length * 2;
3055
- const newTF = new Uint16Array(newCap);
3056
- newTF.set(list.termFrequencies);
3057
- list.termFrequencies = newTF;
3058
- const newFNI = new Uint16Array(newCap);
3059
- newFNI.set(list.fieldNameIndices);
3060
- list.fieldNameIndices = newFNI;
3061
- }
3062
- function compactList(list) {
3063
- if (list.deletedDocs.size === 0) return;
3064
- list.structureRevision++;
3065
- let writeIdx = 0;
3066
- let ascending = true;
3067
- for (let i = 0; i < list.length; i++) {
3068
- if (!list.deletedDocs.has(list.docIds[i])) {
3069
- if (writeIdx > 0 && list.docIds[i] < list.docIds[writeIdx - 1]) ascending = false;
3070
- if (writeIdx !== i) {
3071
- list.docIds[writeIdx] = list.docIds[i];
3072
- list.termFrequencies[writeIdx] = list.termFrequencies[i];
3073
- list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
3074
- if (list.positions) {
3075
- list.positions[writeIdx] = list.positions[i];
3076
- }
3077
- }
3078
- writeIdx++;
3079
- }
3253
+ // src/core/inverted-index/constants.ts
3254
+ var TERM_TABLE_START_CAPACITY = 1 << 10;
3255
+ var POSTING_POOL_START_ENTRIES = 1 << 12;
3256
+ var POSITION_POOL_START_VALUES = 1 << 12;
3257
+ var RETIRED_DOCUMENTS_START_CAPACITY = 1 << 10;
3258
+ var SLICE_GROWTH_FACTOR = 1.5;
3259
+ var POOL_GROWTH_FACTOR = 1.5;
3260
+ var POOL_VACUUM_FREE_FRACTION = 0.25;
3261
+ var CACHED_VIEW_MIN_ENTRIES = 128;
3262
+ var TERM_VACUUM_MIN_DEAD_TERMS = 1 << 10;
3263
+ var POOL_VACUUM_MIN_FREE_ENTRIES = 1 << 14;
3264
+ var POSITION_VACUUM_MIN_FREE_VALUES = 1 << 14;
3265
+ var TERM_ENTRY_ESTIMATED_BYTES = 64;
3266
+ var TOKEN_STRING_HEADER_BYTES = 16;
3267
+
3268
+ // src/core/inverted-index/pools.ts
3269
+ var NO_SLICE = 255;
3270
+ var EMPTY_POSITIONS = 0;
3271
+ var MAX_POOL_ENTRIES = 2 ** 32 - 1;
3272
+ var SLICE_CAPACITIES = (() => {
3273
+ const capacities = [1];
3274
+ while (capacities[capacities.length - 1] < MAX_POOL_ENTRIES) {
3275
+ const previous = capacities[capacities.length - 1];
3276
+ capacities.push(Math.min(MAX_POOL_ENTRIES, Math.max(previous + 1, Math.ceil(previous * SLICE_GROWTH_FACTOR))));
3277
+ }
3278
+ return Uint32Array.from(capacities);
3279
+ })();
3280
+ function sliceClassFor(entries) {
3281
+ let low = 0;
3282
+ let high = SLICE_CAPACITIES.length - 1;
3283
+ while (low < high) {
3284
+ const middle = low + high >>> 1;
3285
+ if (SLICE_CAPACITIES[middle] < entries) low = middle + 1;
3286
+ else high = middle;
3080
3287
  }
3081
- list.docIds.length = writeIdx;
3082
- if (list.positions) list.positions.length = writeIdx;
3083
- list.length = writeIdx;
3084
- list.deletedDocs.clear();
3085
- list.ordered = ascending;
3288
+ return low;
3086
3289
  }
3087
- function compactDocEntries(list, internalId) {
3088
- list.structureRevision++;
3089
- let writeIdx = 0;
3090
- let ascending = true;
3091
- for (let i = 0; i < list.length; i++) {
3092
- if (list.docIds[i] !== internalId) {
3093
- if (writeIdx > 0 && list.docIds[i] < list.docIds[writeIdx - 1]) ascending = false;
3094
- if (writeIdx !== i) {
3095
- list.docIds[writeIdx] = list.docIds[i];
3096
- list.termFrequencies[writeIdx] = list.termFrequencies[i];
3097
- list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
3098
- if (list.positions) {
3099
- list.positions[writeIdx] = list.positions[i];
3100
- }
3101
- }
3102
- writeIdx++;
3103
- }
3290
+ function sliceCapacity(sliceClass) {
3291
+ return sliceClass === NO_SLICE ? 0 : SLICE_CAPACITIES[sliceClass];
3292
+ }
3293
+ function grownCapacity(current, needed) {
3294
+ return Math.min(MAX_POOL_ENTRIES, Math.max(needed, Math.ceil(current * POOL_GROWTH_FACTOR)));
3295
+ }
3296
+ var PostingPool = class {
3297
+ docIds;
3298
+ frequencies;
3299
+ fieldIndices;
3300
+ positionStarts = null;
3301
+ end = 0;
3302
+ epoch = 0;
3303
+ freeEntries = 0;
3304
+ freeLists = [];
3305
+ constructor(capacity) {
3306
+ this.docIds = new Uint32Array(capacity);
3307
+ this.frequencies = new Uint16Array(capacity);
3308
+ this.fieldIndices = new Uint16Array(capacity);
3309
+ }
3310
+ allocate(sliceClass) {
3311
+ const size = sliceCapacity(sliceClass);
3312
+ const free = this.freeLists[sliceClass];
3313
+ if (free !== void 0) {
3314
+ const reused = free.pop();
3315
+ if (reused !== void 0) {
3316
+ this.freeEntries -= size;
3317
+ return reused;
3318
+ }
3319
+ }
3320
+ const start = this.end;
3321
+ this.reserve(start + size);
3322
+ this.end = start + size;
3323
+ return start;
3324
+ }
3325
+ release(start, sliceClass) {
3326
+ if (sliceClass === NO_SLICE) return;
3327
+ let free = this.freeLists[sliceClass];
3328
+ if (free === void 0) {
3329
+ free = [];
3330
+ this.freeLists[sliceClass] = free;
3331
+ }
3332
+ free.push(start);
3333
+ this.freeEntries += sliceCapacity(sliceClass);
3334
+ }
3335
+ trackPositions() {
3336
+ if (this.positionStarts === null) {
3337
+ this.positionStarts = new Uint32Array(this.docIds.length);
3338
+ this.epoch++;
3339
+ }
3340
+ return this.positionStarts;
3341
+ }
3342
+ move(from, to, count) {
3343
+ this.docIds.copyWithin(to, from, from + count);
3344
+ this.frequencies.copyWithin(to, from, from + count);
3345
+ this.fieldIndices.copyWithin(to, from, from + count);
3346
+ if (this.positionStarts !== null) this.positionStarts.copyWithin(to, from, from + count);
3347
+ }
3348
+ byteLength() {
3349
+ const positionBytes = this.positionStarts === null ? 0 : this.positionStarts.byteLength;
3350
+ return this.docIds.byteLength + this.frequencies.byteLength + this.fieldIndices.byteLength + positionBytes;
3351
+ }
3352
+ reserve(needed) {
3353
+ if (needed <= this.docIds.length) return;
3354
+ const capacity = grownCapacity(this.docIds.length, needed);
3355
+ this.docIds = grownCopy(this.docIds, new Uint32Array(capacity), this.end);
3356
+ this.frequencies = grownCopy(this.frequencies, new Uint16Array(capacity), this.end);
3357
+ this.fieldIndices = grownCopy(this.fieldIndices, new Uint16Array(capacity), this.end);
3358
+ if (this.positionStarts !== null) {
3359
+ this.positionStarts = grownCopy(this.positionStarts, new Uint32Array(capacity), this.end);
3360
+ }
3361
+ this.epoch++;
3104
3362
  }
3105
- list.docIds.length = writeIdx;
3106
- if (list.positions) list.positions.length = writeIdx;
3107
- list.length = writeIdx;
3108
- list.ordered = ascending;
3363
+ };
3364
+ function grownCopy(source, target, used) {
3365
+ target.set(source.subarray(0, used));
3366
+ return target;
3109
3367
  }
3368
+ var PositionPool = class {
3369
+ values;
3370
+ end = 1;
3371
+ freeValues = 0;
3372
+ constructor(capacity) {
3373
+ this.values = new Uint32Array(Math.max(1, capacity));
3374
+ }
3375
+ append(positions) {
3376
+ const count = positions.length;
3377
+ if (count === 0) return EMPTY_POSITIONS;
3378
+ const start = this.end;
3379
+ this.reserve(start + 1 + count);
3380
+ const values = this.values;
3381
+ values[start] = count;
3382
+ for (let i = 0; i < count; i++) values[start + 1 + i] = positions[i];
3383
+ this.end = start + 1 + count;
3384
+ return start;
3385
+ }
3386
+ appendFrom(source, start) {
3387
+ if (start === EMPTY_POSITIONS) return EMPTY_POSITIONS;
3388
+ const count = source.values[start];
3389
+ const target = this.end;
3390
+ this.reserve(target + 1 + count);
3391
+ this.values.set(source.values.subarray(start, start + 1 + count), target);
3392
+ this.end = target + 1 + count;
3393
+ return target;
3394
+ }
3395
+ countAt(start) {
3396
+ return this.values[start];
3397
+ }
3398
+ read(start) {
3399
+ const count = this.values[start];
3400
+ const out = new Array(count);
3401
+ for (let i = 0; i < count; i++) out[i] = this.values[start + 1 + i];
3402
+ return out;
3403
+ }
3404
+ copyInto(start, target, offset) {
3405
+ const count = this.values[start];
3406
+ if (count > 0) target.set(this.values.subarray(start + 1, start + 1 + count), offset);
3407
+ return count;
3408
+ }
3409
+ release(start) {
3410
+ if (start === EMPTY_POSITIONS) return;
3411
+ this.freeValues += this.values[start] + 1;
3412
+ }
3413
+ reserve(needed) {
3414
+ if (needed <= this.values.length) return;
3415
+ const capacity = grownCapacity(this.values.length, needed);
3416
+ const grown = new Uint32Array(capacity);
3417
+ grown.set(this.values.subarray(0, this.end));
3418
+ this.values = grown;
3419
+ }
3420
+ };
3110
3421
 
3111
- // src/core/inverted-index.ts
3112
- function createInvertedIndex(fieldNameTable) {
3113
- const index = /* @__PURE__ */ new Map();
3114
- const charBuckets = /* @__PURE__ */ new Map();
3115
- let batchMode = false;
3116
- const batchDirtyTokens = /* @__PURE__ */ new Set();
3117
- function trackToken(token) {
3118
- if (token.length === 0) return;
3119
- const ch = token[0];
3120
- let bucket = charBuckets.get(ch);
3121
- if (!bucket) {
3122
- bucket = /* @__PURE__ */ new Set();
3123
- charBuckets.set(ch, bucket);
3422
+ // src/core/inverted-index/terms.ts
3423
+ var TERM_LIVE = 1;
3424
+ var TERM_ORDERED = 2;
3425
+ var TERM_POSITIONS = 4;
3426
+ function grow(source, target) {
3427
+ target.set(source);
3428
+ return target;
3429
+ }
3430
+ var TermColumns = class _TermColumns {
3431
+ constructor(startCapacity) {
3432
+ this.startCapacity = startCapacity;
3433
+ this.sliceStart = new Uint32Array(startCapacity);
3434
+ this.sliceClass = new Uint8Array(startCapacity);
3435
+ this.lengths = new Uint32Array(startCapacity);
3436
+ this.docFrequency = new Uint32Array(startCapacity);
3437
+ this.deadDocuments = new Uint32Array(startCapacity);
3438
+ this.totalTermFrequency = new Float64Array(startCapacity);
3439
+ this.revision = new Uint32Array(startCapacity);
3440
+ this.flags = new Uint8Array(startCapacity);
3441
+ }
3442
+ ids = /* @__PURE__ */ new Map();
3443
+ tokens = [];
3444
+ sliceStart;
3445
+ sliceClass;
3446
+ lengths;
3447
+ docFrequency;
3448
+ deadDocuments;
3449
+ totalTermFrequency;
3450
+ revision;
3451
+ flags;
3452
+ count = 0;
3453
+ liveCount = 0;
3454
+ generation = 0;
3455
+ tokenCodeUnits = 0;
3456
+ buckets = /* @__PURE__ */ new Map();
3457
+ get deadCount() {
3458
+ return this.count - this.liveCount;
3459
+ }
3460
+ add(token) {
3461
+ const ordinal = this.count;
3462
+ if (ordinal === this.flags.length) this.growTo(ordinal * 2);
3463
+ this.count = ordinal + 1;
3464
+ this.liveCount++;
3465
+ this.tokenCodeUnits += token.length;
3466
+ this.ids.set(token, ordinal);
3467
+ this.tokens.push(token);
3468
+ this.sliceClass[ordinal] = NO_SLICE;
3469
+ this.flags[ordinal] = TERM_LIVE | TERM_ORDERED;
3470
+ this.track(token, ordinal);
3471
+ return ordinal;
3472
+ }
3473
+ kill(ordinal) {
3474
+ const token = this.tokens[ordinal];
3475
+ this.ids.delete(token);
3476
+ const bucket = token.length > 0 ? this.buckets.get(token[0]) : void 0;
3477
+ if (bucket !== void 0) bucket.dead++;
3478
+ this.tokens[ordinal] = "";
3479
+ this.flags[ordinal] = 0;
3480
+ this.lengths[ordinal] = 0;
3481
+ this.docFrequency[ordinal] = 0;
3482
+ this.deadDocuments[ordinal] = 0;
3483
+ this.totalTermFrequency[ordinal] = 0;
3484
+ this.sliceClass[ordinal] = NO_SLICE;
3485
+ this.liveCount--;
3486
+ this.tokenCodeUnits -= token.length;
3487
+ }
3488
+ isLive(ordinal) {
3489
+ return (this.flags[ordinal] & TERM_LIVE) !== 0;
3490
+ }
3491
+ forEachLive(visit) {
3492
+ const flags = this.flags;
3493
+ for (let ordinal = 0; ordinal < this.count; ordinal++) {
3494
+ if ((flags[ordinal] & TERM_LIVE) !== 0) visit(ordinal);
3495
+ }
3496
+ }
3497
+ forEachStartingWith(prefix, visit) {
3498
+ if (prefix.length === 0) {
3499
+ this.forEachLive(visit);
3500
+ return;
3124
3501
  }
3125
- bucket.add(token);
3502
+ const bucket = this.buckets.get(prefix[0]);
3503
+ if (bucket === void 0) return;
3504
+ this.compactBucket(prefix[0], bucket);
3505
+ const { ordinals, length } = bucket;
3506
+ const flags = this.flags;
3507
+ const tokens = this.tokens;
3508
+ const checkPrefix = prefix.length > 1;
3509
+ for (let i = 0; i < length; i++) {
3510
+ const ordinal = ordinals[i];
3511
+ if ((flags[ordinal] & TERM_LIVE) === 0) continue;
3512
+ if (checkPrefix && !tokens[ordinal].startsWith(prefix)) continue;
3513
+ visit(ordinal);
3514
+ }
3515
+ }
3516
+ renumber() {
3517
+ const remap = new Int32Array(this.count).fill(-1);
3518
+ const capacity = Math.max(this.startCapacity, this.liveCount);
3519
+ const next = new _TermColumns(capacity);
3520
+ for (let ordinal = 0; ordinal < this.count; ordinal++) {
3521
+ if ((this.flags[ordinal] & TERM_LIVE) === 0) continue;
3522
+ const target = next.add(this.tokens[ordinal]);
3523
+ remap[ordinal] = target;
3524
+ next.sliceStart[target] = this.sliceStart[ordinal];
3525
+ next.sliceClass[target] = this.sliceClass[ordinal];
3526
+ next.lengths[target] = this.lengths[ordinal];
3527
+ next.docFrequency[target] = this.docFrequency[ordinal];
3528
+ next.deadDocuments[target] = this.deadDocuments[ordinal];
3529
+ next.totalTermFrequency[target] = this.totalTermFrequency[ordinal];
3530
+ next.revision[target] = this.revision[ordinal];
3531
+ next.flags[target] = this.flags[ordinal];
3532
+ }
3533
+ this.ids = next.ids;
3534
+ this.tokens = next.tokens;
3535
+ this.sliceStart = next.sliceStart;
3536
+ this.sliceClass = next.sliceClass;
3537
+ this.lengths = next.lengths;
3538
+ this.docFrequency = next.docFrequency;
3539
+ this.deadDocuments = next.deadDocuments;
3540
+ this.totalTermFrequency = next.totalTermFrequency;
3541
+ this.revision = next.revision;
3542
+ this.flags = next.flags;
3543
+ this.buckets = next.buckets;
3544
+ this.count = next.count;
3545
+ this.liveCount = next.liveCount;
3546
+ this.generation++;
3547
+ return remap;
3126
3548
  }
3127
- function untrackToken(token) {
3549
+ clear() {
3550
+ this.ids = /* @__PURE__ */ new Map();
3551
+ this.tokens = [];
3552
+ this.buckets = /* @__PURE__ */ new Map();
3553
+ this.count = 0;
3554
+ this.liveCount = 0;
3555
+ this.tokenCodeUnits = 0;
3556
+ this.generation++;
3557
+ this.sliceStart = new Uint32Array(this.startCapacity);
3558
+ this.sliceClass = new Uint8Array(this.startCapacity);
3559
+ this.lengths = new Uint32Array(this.startCapacity);
3560
+ this.docFrequency = new Uint32Array(this.startCapacity);
3561
+ this.deadDocuments = new Uint32Array(this.startCapacity);
3562
+ this.totalTermFrequency = new Float64Array(this.startCapacity);
3563
+ this.revision = new Uint32Array(this.startCapacity);
3564
+ this.flags = new Uint8Array(this.startCapacity);
3565
+ }
3566
+ byteLength() {
3567
+ let bucketBytes = 0;
3568
+ for (const bucket of this.buckets.values()) bucketBytes += bucket.ordinals.byteLength;
3569
+ return this.sliceStart.byteLength + this.sliceClass.byteLength + this.lengths.byteLength + this.docFrequency.byteLength + this.deadDocuments.byteLength + this.totalTermFrequency.byteLength + this.revision.byteLength + this.flags.byteLength + bucketBytes;
3570
+ }
3571
+ track(token, ordinal) {
3128
3572
  if (token.length === 0) return;
3129
- const ch = token[0];
3130
- const bucket = charBuckets.get(ch);
3131
- if (bucket) {
3132
- bucket.delete(token);
3133
- if (bucket.size === 0) charBuckets.delete(ch);
3134
- }
3573
+ const first = token[0];
3574
+ let bucket = this.buckets.get(first);
3575
+ if (bucket === void 0) {
3576
+ bucket = { ordinals: new Uint32Array(4), length: 0, dead: 0 };
3577
+ this.buckets.set(first, bucket);
3578
+ }
3579
+ if (bucket.length === bucket.ordinals.length) {
3580
+ bucket.ordinals = grow(bucket.ordinals, new Uint32Array(bucket.ordinals.length * 2));
3581
+ }
3582
+ bucket.ordinals[bucket.length++] = ordinal;
3583
+ }
3584
+ compactBucket(first, bucket) {
3585
+ if (bucket.dead * 2 <= bucket.length) return;
3586
+ const { ordinals } = bucket;
3587
+ let kept = 0;
3588
+ for (let i = 0; i < bucket.length; i++) {
3589
+ const ordinal = ordinals[i];
3590
+ if ((this.flags[ordinal] & TERM_LIVE) !== 0) ordinals[kept++] = ordinal;
3591
+ }
3592
+ bucket.length = kept;
3593
+ bucket.dead = 0;
3594
+ if (kept === 0) this.buckets.delete(first);
3595
+ }
3596
+ growTo(capacity) {
3597
+ this.sliceStart = grow(this.sliceStart, new Uint32Array(capacity));
3598
+ this.sliceClass = grow(this.sliceClass, new Uint8Array(capacity));
3599
+ this.lengths = grow(this.lengths, new Uint32Array(capacity));
3600
+ this.docFrequency = grow(this.docFrequency, new Uint32Array(capacity));
3601
+ this.deadDocuments = grow(this.deadDocuments, new Uint32Array(capacity));
3602
+ this.totalTermFrequency = grow(this.totalTermFrequency, new Float64Array(capacity));
3603
+ this.revision = grow(this.revision, new Uint32Array(capacity));
3604
+ this.flags = grow(this.flags, new Uint8Array(capacity));
3135
3605
  }
3136
- function tokensStartingWith(prefix) {
3137
- if (prefix.length === 0) return index.keys();
3138
- const bucket = charBuckets.get(prefix[0]);
3139
- if (!bucket) return [];
3140
- if (prefix.length === 1) return bucket;
3141
- const filtered = [];
3142
- for (const t of bucket) {
3143
- if (t.startsWith(prefix)) {
3144
- filtered.push(t);
3606
+ };
3607
+
3608
+ // src/core/inverted-index/view.ts
3609
+ var NO_TOMBSTONES = { size: 0, has: () => false };
3610
+ var NO_DOCUMENT_IDS = new Uint32Array(0);
3611
+ var NO_FREQUENCIES = new Uint16Array(0);
3612
+ var DETACHED = -1;
3613
+ var LivePostingView = class {
3614
+ constructor(store, token, ordinal, keepsPositions) {
3615
+ this.store = store;
3616
+ this.token = token;
3617
+ this.keepsPositions = keepsPositions;
3618
+ this.ordinal = ordinal;
3619
+ this.generation = store.terms.generation;
3620
+ const view = this;
3621
+ this.docIdSet = {
3622
+ get size() {
3623
+ const live = view.sync();
3624
+ return live === DETACHED ? 0 : view.store.terms.docFrequency[live];
3145
3625
  }
3626
+ };
3627
+ this.retiredRows = {
3628
+ get size() {
3629
+ const live = view.sync();
3630
+ return live === DETACHED ? 0 : view.store.terms.deadDocuments[live];
3631
+ },
3632
+ has: (internalId) => store.isRetired(internalId)
3633
+ };
3634
+ }
3635
+ docIdSet;
3636
+ retiredRows;
3637
+ ordinal;
3638
+ generation;
3639
+ syncedEpoch = -1;
3640
+ syncedStart = -1;
3641
+ syncedLength = 0;
3642
+ syncedRevision = -1;
3643
+ ids = NO_DOCUMENT_IDS;
3644
+ frequencies = NO_FREQUENCIES;
3645
+ fields = NO_FREQUENCIES;
3646
+ materialized = null;
3647
+ get length() {
3648
+ this.sync();
3649
+ return this.syncedLength;
3650
+ }
3651
+ get docIds() {
3652
+ this.sync();
3653
+ return this.ids;
3654
+ }
3655
+ get termFrequencies() {
3656
+ this.sync();
3657
+ return this.frequencies;
3658
+ }
3659
+ get fieldNameIndices() {
3660
+ this.sync();
3661
+ return this.fields;
3662
+ }
3663
+ get holdsPositions() {
3664
+ const live = this.sync();
3665
+ return live !== DETACHED && (this.store.terms.flags[live] & TERM_POSITIONS) !== 0;
3666
+ }
3667
+ get positions() {
3668
+ if (!this.holdsPositions) return null;
3669
+ if (this.materialized !== null) return this.materialized;
3670
+ const starts = this.store.pool.positionStarts;
3671
+ const rows = new Array(this.syncedLength);
3672
+ for (let row = 0; row < this.syncedLength; row++) {
3673
+ rows[row] = starts === null ? [] : this.store.positions.read(starts[this.syncedStart + row]);
3674
+ }
3675
+ if (this.keepsPositions) this.materialized = rows;
3676
+ return rows;
3677
+ }
3678
+ positionCountAt(row) {
3679
+ if (!this.holdsPositions) return 0;
3680
+ const starts = this.store.pool.positionStarts;
3681
+ return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
3682
+ }
3683
+ copyPositionsAt(row, target, offset) {
3684
+ if (!this.holdsPositions) return 0;
3685
+ const starts = this.store.pool.positionStarts;
3686
+ return starts === null ? 0 : this.store.positions.copyInto(starts[this.syncedStart + row], target, offset);
3687
+ }
3688
+ get deletedDocs() {
3689
+ const live = this.sync();
3690
+ if (live === DETACHED || this.store.terms.deadDocuments[live] === 0) return NO_TOMBSTONES;
3691
+ return this.retiredRows;
3692
+ }
3693
+ get totalTermFrequency() {
3694
+ const live = this.sync();
3695
+ return live === DETACHED ? 0 : this.store.terms.totalTermFrequency[live];
3696
+ }
3697
+ get structureRevision() {
3698
+ const live = this.sync();
3699
+ return live === DETACHED ? this.syncedRevision : this.store.terms.revision[live];
3700
+ }
3701
+ get ordered() {
3702
+ const live = this.sync();
3703
+ return live === DETACHED || (this.store.terms.flags[live] & TERM_ORDERED) !== 0;
3704
+ }
3705
+ sync() {
3706
+ const terms = this.store.terms;
3707
+ if (this.generation !== terms.generation) {
3708
+ this.ordinal = terms.ids.get(this.token) ?? DETACHED;
3709
+ this.generation = terms.generation;
3710
+ }
3711
+ const ordinal = this.ordinal;
3712
+ if (ordinal === DETACHED || !terms.isLive(ordinal)) {
3713
+ this.detach();
3714
+ return DETACHED;
3715
+ }
3716
+ const pool = this.store.pool;
3717
+ const start = terms.sliceStart[ordinal];
3718
+ const length = terms.lengths[ordinal];
3719
+ const revision = terms.revision[ordinal];
3720
+ if (pool.epoch !== this.syncedEpoch || start !== this.syncedStart || length !== this.syncedLength || revision !== this.syncedRevision) {
3721
+ this.ids = pool.docIds.subarray(start, start + length);
3722
+ this.frequencies = pool.frequencies.subarray(start, start + length);
3723
+ this.fields = pool.fieldIndices.subarray(start, start + length);
3724
+ this.materialized = null;
3725
+ this.syncedEpoch = pool.epoch;
3726
+ this.syncedStart = start;
3727
+ this.syncedLength = length;
3728
+ this.syncedRevision = revision;
3146
3729
  }
3147
- return filtered;
3730
+ return ordinal;
3731
+ }
3732
+ detach() {
3733
+ if (this.syncedStart === DETACHED && this.syncedLength === 0) return;
3734
+ this.ids = NO_DOCUMENT_IDS;
3735
+ this.frequencies = NO_FREQUENCIES;
3736
+ this.fields = NO_FREQUENCIES;
3737
+ this.materialized = null;
3738
+ this.syncedStart = DETACHED;
3739
+ this.syncedLength = 0;
3740
+ this.syncedEpoch = -1;
3148
3741
  }
3149
- function getOrCreateList(token) {
3150
- let list = index.get(token);
3151
- if (!list) {
3152
- list = createPostingList();
3153
- index.set(token, list);
3154
- trackToken(token);
3742
+ };
3743
+
3744
+ // src/core/inverted-index/postings.ts
3745
+ function holdsDocument(docIds, start, length, internalId, ordered) {
3746
+ if (!ordered) {
3747
+ for (let i = start; i < start + length; i++) {
3748
+ if (docIds[i] === internalId) return true;
3155
3749
  }
3156
- return list;
3750
+ return false;
3157
3751
  }
3158
- return {
3159
- insert(token, internalId, termFrequency, fieldNameIndex2, positions) {
3160
- const list = getOrCreateList(token);
3161
- if (list.deletedDocs.size > 0 && list.deletedDocs.has(internalId)) {
3162
- compactDocEntries(list, internalId);
3163
- list.deletedDocs.delete(internalId);
3164
- }
3165
- if (list.length >= list.termFrequencies.length) {
3166
- growTypedArrays(list);
3167
- }
3168
- const idx = list.length;
3169
- if (idx > 0 && internalId < list.docIds[idx - 1]) list.ordered = false;
3170
- list.docIds.push(internalId);
3171
- list.termFrequencies[idx] = termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : termFrequency;
3172
- list.fieldNameIndices[idx] = fieldNameIndex2;
3173
- list.totalTermFrequency += list.termFrequencies[idx];
3174
- if (positions !== null) {
3175
- if (!list.positions) {
3176
- list.positions = [];
3177
- for (let j = 0; j < idx; j++) list.positions.push([]);
3178
- }
3179
- list.positions.push(positions);
3180
- } else if (list.positions !== null) {
3181
- list.positions.push([]);
3182
- }
3183
- if (!list.docIdSet.has(internalId)) {
3184
- list.docIdSet.add(internalId);
3185
- }
3186
- list.length++;
3187
- },
3188
- remove(token, internalId, occurrences) {
3189
- const list = index.get(token);
3190
- if (!list || !list.docIdSet.has(internalId)) return;
3191
- list.structureRevision++;
3192
- list.docIdSet.delete(internalId);
3193
- list.deletedDocs.add(internalId);
3194
- list.totalTermFrequency = Math.max(0, list.totalTermFrequency - occurrences);
3195
- if (list.docIdSet.size === 0) {
3196
- index.delete(token);
3197
- untrackToken(token);
3198
- if (batchMode) batchDirtyTokens.delete(token);
3199
- return;
3752
+ let low = start;
3753
+ let high = start + length;
3754
+ while (low < high) {
3755
+ const middle = low + high >>> 1;
3756
+ if (docIds[middle] < internalId) low = middle + 1;
3757
+ else high = middle;
3758
+ }
3759
+ return low < start + length && docIds[low] === internalId;
3760
+ }
3761
+ var LivePostings = class {
3762
+ terms = new TermColumns(TERM_TABLE_START_CAPACITY);
3763
+ pool = new PostingPool(POSTING_POOL_START_ENTRIES);
3764
+ positions = new PositionPool(POSITION_POOL_START_VALUES);
3765
+ retired = new Uint8Array(RETIRED_DOCUMENTS_START_CAPACITY);
3766
+ batchMode = false;
3767
+ dirty = /* @__PURE__ */ new Set();
3768
+ cachedViews = /* @__PURE__ */ new Map();
3769
+ compactedSlack = 0;
3770
+ view(ordinal, cached) {
3771
+ if (!cached || this.terms.lengths[ordinal] < CACHED_VIEW_MIN_ENTRIES) {
3772
+ return new LivePostingView(this, this.terms.tokens[ordinal], ordinal, cached);
3773
+ }
3774
+ let view = this.cachedViews.get(ordinal);
3775
+ if (view === void 0) {
3776
+ view = new LivePostingView(this, this.terms.tokens[ordinal], ordinal, true);
3777
+ this.cachedViews.set(ordinal, view);
3778
+ }
3779
+ return view;
3780
+ }
3781
+ reserve(token, entries) {
3782
+ const terms = this.terms;
3783
+ const ordinal = terms.ids.get(token) ?? terms.add(token);
3784
+ if (sliceCapacity(terms.sliceClass[ordinal]) < terms.lengths[ordinal] + entries) {
3785
+ this.relocate(ordinal, terms.lengths[ordinal] + entries);
3786
+ }
3787
+ }
3788
+ insert(token, internalId, termFrequency, fieldIndex, positions) {
3789
+ const terms = this.terms;
3790
+ if (this.isRetired(internalId)) this.purgeRetired();
3791
+ const ordinal = terms.ids.get(token) ?? terms.add(token);
3792
+ const length = terms.lengths[ordinal];
3793
+ if (length >= sliceCapacity(terms.sliceClass[ordinal])) this.relocate(ordinal, length + 1);
3794
+ const pool = this.pool;
3795
+ const start = terms.sliceStart[ordinal];
3796
+ const at = start + length;
3797
+ const flags = terms.flags[ordinal];
3798
+ let newDocument = true;
3799
+ if (length > 0) {
3800
+ const last = pool.docIds[at - 1];
3801
+ const ordered = (flags & TERM_ORDERED) !== 0;
3802
+ if (internalId === last) {
3803
+ newDocument = false;
3804
+ } else if (!ordered || internalId < last) {
3805
+ newDocument = !holdsDocument(pool.docIds, start, length, internalId, ordered);
3806
+ if (internalId < last) terms.flags[ordinal] = flags & ~TERM_ORDERED;
3807
+ }
3808
+ }
3809
+ const stored = termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : termFrequency;
3810
+ pool.docIds[at] = internalId;
3811
+ pool.frequencies[at] = stored;
3812
+ pool.fieldIndices[at] = fieldIndex;
3813
+ terms.totalTermFrequency[ordinal] += stored;
3814
+ if (positions !== null) {
3815
+ const starts = pool.trackPositions();
3816
+ if ((flags & TERM_POSITIONS) === 0) {
3817
+ terms.flags[ordinal] |= TERM_POSITIONS;
3818
+ starts.fill(EMPTY_POSITIONS, start, at);
3819
+ }
3820
+ starts[at] = this.positions.append(positions);
3821
+ } else if ((flags & TERM_POSITIONS) !== 0) {
3822
+ pool.trackPositions()[at] = EMPTY_POSITIONS;
3823
+ }
3824
+ if (newDocument) terms.docFrequency[ordinal]++;
3825
+ terms.lengths[ordinal] = length + 1;
3826
+ this.repackIfSparse();
3827
+ }
3828
+ remove(token, internalId, occurrences) {
3829
+ const terms = this.terms;
3830
+ const ordinal = terms.ids.get(token);
3831
+ if (ordinal === void 0) return;
3832
+ const length = terms.lengths[ordinal];
3833
+ const ordered = (terms.flags[ordinal] & TERM_ORDERED) !== 0;
3834
+ if (!holdsDocument(this.pool.docIds, terms.sliceStart[ordinal], length, internalId, ordered)) return;
3835
+ terms.revision[ordinal]++;
3836
+ terms.docFrequency[ordinal]--;
3837
+ terms.deadDocuments[ordinal]++;
3838
+ this.retire(internalId);
3839
+ terms.totalTermFrequency[ordinal] = Math.max(0, terms.totalTermFrequency[ordinal] - occurrences);
3840
+ if (terms.docFrequency[ordinal] === 0) {
3841
+ this.dropTerm(ordinal);
3842
+ return;
3843
+ }
3844
+ if (this.batchMode) {
3845
+ this.dirty.add(ordinal);
3846
+ return;
3847
+ }
3848
+ if (terms.deadDocuments[ordinal] / length > POSTING_LIST_COMPACTION_THRESHOLD) this.compact(ordinal);
3849
+ this.repackIfSparse();
3850
+ }
3851
+ isRetired(internalId) {
3852
+ return internalId < this.retired.length && this.retired[internalId] === 1;
3853
+ }
3854
+ beginBatch() {
3855
+ this.batchMode = true;
3856
+ this.dirty.clear();
3857
+ }
3858
+ endBatch() {
3859
+ this.batchMode = false;
3860
+ const terms = this.terms;
3861
+ for (const ordinal of this.dirty) {
3862
+ if (!terms.isLive(ordinal)) continue;
3863
+ if (terms.deadDocuments[ordinal] / terms.lengths[ordinal] > POSTING_LIST_COMPACTION_THRESHOLD) {
3864
+ this.compact(ordinal);
3200
3865
  }
3201
- if (batchMode) {
3202
- batchDirtyTokens.add(token);
3203
- return;
3866
+ }
3867
+ this.dirty.clear();
3868
+ this.repackIfSparse();
3869
+ }
3870
+ clear() {
3871
+ const epoch = this.pool.epoch + 1;
3872
+ this.terms.clear();
3873
+ this.pool = new PostingPool(POSTING_POOL_START_ENTRIES);
3874
+ this.pool.epoch = epoch;
3875
+ this.positions = new PositionPool(POSITION_POOL_START_VALUES);
3876
+ this.retired = new Uint8Array(RETIRED_DOCUMENTS_START_CAPACITY);
3877
+ this.dirty.clear();
3878
+ this.cachedViews.clear();
3879
+ this.compactedSlack = 0;
3880
+ }
3881
+ byteLength() {
3882
+ return this.terms.byteLength() + this.pool.byteLength() + this.positions.values.byteLength + this.retired.byteLength;
3883
+ }
3884
+ retire(internalId) {
3885
+ if (internalId >= this.retired.length) {
3886
+ const grown = new Uint8Array(Math.max(internalId + 1, this.retired.length * 2));
3887
+ grown.set(this.retired);
3888
+ this.retired = grown;
3889
+ }
3890
+ this.retired[internalId] = 1;
3891
+ }
3892
+ purgeRetired() {
3893
+ const terms = this.terms;
3894
+ for (let ordinal = 0; ordinal < terms.count; ordinal++) {
3895
+ if (terms.isLive(ordinal) && terms.deadDocuments[ordinal] > 0) this.compact(ordinal);
3896
+ }
3897
+ this.retired.fill(0);
3898
+ }
3899
+ compact(ordinal) {
3900
+ if (this.terms.deadDocuments[ordinal] === 0) return;
3901
+ const retired = this.retired;
3902
+ this.filterEntries(ordinal, (id) => id < retired.length && retired[id] === 1);
3903
+ this.terms.deadDocuments[ordinal] = 0;
3904
+ }
3905
+ filterEntries(ordinal, drop) {
3906
+ const terms = this.terms;
3907
+ const pool = this.pool;
3908
+ const start = terms.sliceStart[ordinal];
3909
+ const length = terms.lengths[ordinal];
3910
+ const starts = (terms.flags[ordinal] & TERM_POSITIONS) !== 0 ? pool.positionStarts : null;
3911
+ const { docIds, frequencies, fieldIndices } = pool;
3912
+ terms.revision[ordinal]++;
3913
+ let kept = 0;
3914
+ let ascending = true;
3915
+ for (let i = start; i < start + length; i++) {
3916
+ const internalId = docIds[i];
3917
+ if (drop(internalId)) {
3918
+ if (starts !== null) this.positions.release(starts[i]);
3919
+ continue;
3204
3920
  }
3205
- if (list.deletedDocs.size / list.length > POSTING_LIST_COMPACTION_THRESHOLD) {
3206
- compactList(list);
3921
+ const target = start + kept;
3922
+ if (kept > 0 && internalId < docIds[target - 1]) ascending = false;
3923
+ if (target !== i) {
3924
+ docIds[target] = internalId;
3925
+ frequencies[target] = frequencies[i];
3926
+ fieldIndices[target] = fieldIndices[i];
3927
+ if (starts !== null) starts[target] = starts[i];
3928
+ }
3929
+ kept++;
3930
+ }
3931
+ terms.lengths[ordinal] = kept;
3932
+ terms.flags[ordinal] = ascending ? terms.flags[ordinal] | TERM_ORDERED : terms.flags[ordinal] & ~TERM_ORDERED;
3933
+ this.compactedSlack += length - kept;
3934
+ }
3935
+ relocate(ordinal, needed) {
3936
+ const terms = this.terms;
3937
+ const previousClass = terms.sliceClass[ordinal];
3938
+ const previousStart = terms.sliceStart[ordinal];
3939
+ const length = terms.lengths[ordinal];
3940
+ const grownClass = previousClass === NO_SLICE ? 0 : previousClass + 1;
3941
+ const nextClass = Math.max(grownClass, sliceClassFor(needed));
3942
+ const nextStart = this.pool.allocate(nextClass);
3943
+ if (length > 0) this.pool.move(previousStart, nextStart, length);
3944
+ this.pool.release(previousStart, previousClass);
3945
+ terms.sliceStart[ordinal] = nextStart;
3946
+ terms.sliceClass[ordinal] = nextClass;
3947
+ }
3948
+ dropTerm(ordinal) {
3949
+ const terms = this.terms;
3950
+ const pool = this.pool;
3951
+ if ((terms.flags[ordinal] & TERM_POSITIONS) !== 0 && pool.positionStarts !== null) {
3952
+ const start = terms.sliceStart[ordinal];
3953
+ for (let i = start; i < start + terms.lengths[ordinal]; i++) this.positions.release(pool.positionStarts[i]);
3954
+ }
3955
+ pool.release(terms.sliceStart[ordinal], terms.sliceClass[ordinal]);
3956
+ this.dirty.delete(ordinal);
3957
+ this.cachedViews.delete(ordinal);
3958
+ terms.kill(ordinal);
3959
+ if (terms.deadCount >= TERM_VACUUM_MIN_DEAD_TERMS && terms.deadCount > terms.liveCount) this.renumberTerms();
3960
+ }
3961
+ renumberTerms() {
3962
+ const remap = this.terms.renumber();
3963
+ const dirty = /* @__PURE__ */ new Set();
3964
+ for (const ordinal of this.dirty) {
3965
+ if (remap[ordinal] >= 0) dirty.add(remap[ordinal]);
3966
+ }
3967
+ this.dirty = dirty;
3968
+ this.cachedViews.clear();
3969
+ }
3970
+ repackIfSparse() {
3971
+ const pool = this.pool;
3972
+ const freeEntries = pool.freeEntries + this.compactedSlack;
3973
+ const sparsePostings = freeEntries >= POOL_VACUUM_MIN_FREE_ENTRIES && freeEntries > pool.end * POOL_VACUUM_FREE_FRACTION;
3974
+ const positions = this.positions;
3975
+ const sparsePositions = positions.freeValues >= POSITION_VACUUM_MIN_FREE_VALUES && positions.freeValues > positions.end * POOL_VACUUM_FREE_FRACTION;
3976
+ if (sparsePostings || sparsePositions) this.repack(sparsePositions);
3977
+ }
3978
+ repack(rebuildPositions) {
3979
+ const terms = this.terms;
3980
+ const pool = this.pool;
3981
+ const positions = this.positions;
3982
+ let capacity = 0;
3983
+ for (let ordinal = 0; ordinal < terms.count; ordinal++) {
3984
+ if (terms.isLive(ordinal)) capacity += sliceCapacity(sliceClassFor(terms.lengths[ordinal]));
3985
+ }
3986
+ const nextPool = new PostingPool(Math.max(1, capacity));
3987
+ nextPool.epoch = pool.epoch + 1;
3988
+ const sourceStarts = pool.positionStarts;
3989
+ const targetStarts = sourceStarts === null ? null : nextPool.trackPositions();
3990
+ const nextPositions = rebuildPositions ? new PositionPool(positions.end - positions.freeValues) : positions;
3991
+ for (let ordinal = 0; ordinal < terms.count; ordinal++) {
3992
+ if (!terms.isLive(ordinal)) continue;
3993
+ const start = terms.sliceStart[ordinal];
3994
+ const length = terms.lengths[ordinal];
3995
+ const sliceClass = sliceClassFor(length);
3996
+ const target = nextPool.allocate(sliceClass);
3997
+ nextPool.docIds.set(pool.docIds.subarray(start, start + length), target);
3998
+ nextPool.frequencies.set(pool.frequencies.subarray(start, start + length), target);
3999
+ nextPool.fieldIndices.set(pool.fieldIndices.subarray(start, start + length), target);
4000
+ if ((terms.flags[ordinal] & TERM_POSITIONS) !== 0 && sourceStarts !== null && targetStarts !== null) {
4001
+ if (rebuildPositions) {
4002
+ for (let i = 0; i < length; i++)
4003
+ targetStarts[target + i] = nextPositions.appendFrom(positions, sourceStarts[start + i]);
4004
+ } else {
4005
+ targetStarts.set(sourceStarts.subarray(start, start + length), target);
4006
+ }
3207
4007
  }
4008
+ terms.sliceStart[ordinal] = target;
4009
+ terms.sliceClass[ordinal] = sliceClass;
4010
+ }
4011
+ this.pool = nextPool;
4012
+ this.positions = nextPositions;
4013
+ this.compactedSlack = 0;
4014
+ }
4015
+ };
4016
+
4017
+ // src/core/inverted-index/index.ts
4018
+ function fieldIndexFor(fieldNameTable, fieldName) {
4019
+ let index = fieldNameTable.indexMap.get(fieldName);
4020
+ if (index === void 0) {
4021
+ index = fieldNameTable.names.length;
4022
+ fieldNameTable.names.push(fieldName);
4023
+ fieldNameTable.indexMap.set(fieldName, index);
4024
+ }
4025
+ return index;
4026
+ }
4027
+ function createInvertedIndex(fieldNameTable) {
4028
+ const postings = new LivePostings();
4029
+ function* liveTokens() {
4030
+ const terms = postings.terms;
4031
+ const generation = terms.generation;
4032
+ for (let ordinal = 0; ordinal < terms.count && terms.generation === generation; ordinal++) {
4033
+ if (terms.isLive(ordinal)) yield terms.tokens[ordinal];
4034
+ }
4035
+ }
4036
+ function serializeTerm(ordinal, resolver) {
4037
+ const list = postings.view(ordinal, false);
4038
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, length } = list;
4039
+ const positions = list.positions;
4040
+ const entries = [];
4041
+ for (let i = 0; i < length; i++) {
4042
+ if (deletedDocs.size > 0 && deletedDocs.has(docIds[i])) continue;
4043
+ const externalId = resolver.toExternal(docIds[i]);
4044
+ if (externalId === void 0) continue;
4045
+ entries.push({
4046
+ docId: externalId,
4047
+ termFrequency: termFrequencies[i],
4048
+ fieldName: fieldNameTable.names[fieldNameIndices[i]],
4049
+ positions: positions === null ? [] : [...positions[i]]
4050
+ });
4051
+ }
4052
+ return entries;
4053
+ }
4054
+ return {
4055
+ insert(token, internalId, termFrequency, fieldNameIndex2, positions) {
4056
+ postings.insert(token, internalId, termFrequency, fieldNameIndex2, positions);
4057
+ },
4058
+ remove(token, internalId, occurrences) {
4059
+ postings.remove(token, internalId, occurrences);
3208
4060
  },
3209
4061
  beginBatch() {
3210
- batchMode = true;
3211
- batchDirtyTokens.clear();
4062
+ postings.beginBatch();
3212
4063
  },
3213
4064
  endBatch() {
3214
- batchMode = false;
3215
- for (const token of batchDirtyTokens) {
3216
- const list = index.get(token);
3217
- if (!list) continue;
3218
- if (list.deletedDocs.size / list.length > POSTING_LIST_COMPACTION_THRESHOLD) {
3219
- compactList(list);
3220
- }
3221
- }
3222
- batchDirtyTokens.clear();
4065
+ postings.endBatch();
3223
4066
  },
3224
4067
  lookup(token) {
3225
- return index.get(token);
4068
+ const ordinal = postings.terms.ids.get(token);
4069
+ return ordinal === void 0 ? void 0 : postings.view(ordinal, true);
4070
+ },
4071
+ lookupReadOnce(token) {
4072
+ const ordinal = postings.terms.ids.get(token);
4073
+ return ordinal === void 0 ? void 0 : postings.view(ordinal, false);
3226
4074
  },
3227
4075
  documentFrequency(token) {
3228
- return index.get(token)?.docIdSet.size ?? 0;
4076
+ const ordinal = postings.terms.ids.get(token);
4077
+ return ordinal === void 0 ? 0 : postings.terms.docFrequency[ordinal];
3229
4078
  },
3230
4079
  fuzzyLookup(token, tolerance, prefixLength) {
4080
+ const terms = postings.terms;
3231
4081
  if (tolerance === 0) {
3232
- const exact = index.get(token);
3233
- return exact ? [{ token, postingList: exact }] : [];
4082
+ const exact = terms.ids.get(token);
4083
+ return exact === void 0 ? [] : [{ token, postingList: postings.view(exact, true) }];
3234
4084
  }
3235
4085
  const results = [];
3236
- const candidates = tokensStartingWith(fuzzyPrefixOf(token, prefixLength));
3237
- for (const candidate of candidates) {
3238
- const { withinTolerance } = boundedLevenshtein(token, candidate, tolerance);
3239
- if (withinTolerance) {
3240
- const postingList = index.get(candidate);
3241
- if (postingList) results.push({ token: candidate, postingList });
4086
+ terms.forEachStartingWith(fuzzyPrefixOf(token, prefixLength), (ordinal) => {
4087
+ const candidate = terms.tokens[ordinal];
4088
+ if (boundedLevenshtein(token, candidate, tolerance).withinTolerance) {
4089
+ results.push({ token: candidate, postingList: postings.view(ordinal, true) });
3242
4090
  }
3243
- }
4091
+ });
3244
4092
  return results;
3245
4093
  },
3246
4094
  prefixSearch(prefix, limit) {
3247
4095
  if (prefix.length === 0 || limit <= 0) return [];
3248
- const candidates = tokensStartingWith(prefix);
4096
+ const terms = postings.terms;
3249
4097
  const results = [];
3250
- for (const term of candidates) {
3251
- if (!term.startsWith(prefix)) continue;
3252
- const list = index.get(term);
3253
- if (!list) continue;
3254
- results.push({ term, documentFrequency: list.docIdSet.size });
3255
- }
4098
+ terms.forEachStartingWith(prefix, (ordinal) => {
4099
+ results.push({ term: terms.tokens[ordinal], documentFrequency: terms.docFrequency[ordinal] });
4100
+ });
3256
4101
  results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.term, b.term));
3257
4102
  if (results.length > limit) results.length = limit;
3258
4103
  return results;
3259
4104
  },
3260
4105
  has(token) {
3261
- return index.has(token);
4106
+ return postings.terms.ids.has(token);
3262
4107
  },
3263
4108
  tokens() {
3264
- return index.keys();
4109
+ return liveTokens();
3265
4110
  },
3266
4111
  size() {
3267
- return index.size;
4112
+ return postings.terms.liveCount;
3268
4113
  },
3269
4114
  clear() {
3270
- index.clear();
3271
- charBuckets.clear();
4115
+ postings.clear();
4116
+ },
4117
+ estimateBytes() {
4118
+ const terms = postings.terms;
4119
+ const perTerm = TERM_ENTRY_ESTIMATED_BYTES + TOKEN_STRING_HEADER_BYTES;
4120
+ return postings.byteLength() + terms.liveCount * perTerm + terms.tokenCodeUnits * 2;
3272
4121
  },
3273
4122
  serialize(resolver) {
3274
4123
  const result = /* @__PURE__ */ Object.create(null);
3275
- for (const [token, list] of index) {
3276
- const postings = [];
3277
- for (let i = 0; i < list.length; i++) {
3278
- if (list.deletedDocs.size > 0 && list.deletedDocs.has(list.docIds[i])) continue;
3279
- const externalId = resolver.toExternal(list.docIds[i]);
3280
- if (externalId === void 0) continue;
3281
- postings.push({
3282
- docId: externalId,
3283
- termFrequency: list.termFrequencies[i],
3284
- fieldName: fieldNameTable.names[list.fieldNameIndices[i]],
3285
- positions: list.positions ? list.positions[i] : []
3286
- });
3287
- }
3288
- if (postings.length > 0) {
3289
- result[token] = { docFrequency: list.docIdSet.size, postings };
4124
+ const terms = postings.terms;
4125
+ for (let ordinal = 0; ordinal < terms.count; ordinal++) {
4126
+ if (!terms.isLive(ordinal)) continue;
4127
+ const entries = serializeTerm(ordinal, resolver);
4128
+ if (entries.length > 0) {
4129
+ result[terms.tokens[ordinal]] = { docFrequency: terms.docFrequency[ordinal], postings: entries };
3290
4130
  }
3291
4131
  }
3292
4132
  return result;
3293
4133
  },
3294
4134
  deserialize(data, resolver) {
3295
- index.clear();
3296
- charBuckets.clear();
4135
+ postings.clear();
3297
4136
  for (const token of Object.keys(data)) {
3298
- const src = data[token];
3299
- const count = src.postings.length;
3300
- const docIdSet = /* @__PURE__ */ new Set();
3301
- const docIds = new Array(count);
3302
- const termFrequencies = new Uint16Array(count);
3303
- const fieldNameIndices = new Uint16Array(count);
4137
+ const source = data[token].postings;
3304
4138
  let hasPositions = false;
3305
- let validCount = 0;
3306
- let totalTermFrequency = 0;
3307
- for (let i = 0; i < count; i++) {
3308
- const p = src.postings[i];
3309
- const internalId = resolver.toInternal(p.docId);
3310
- if (internalId === void 0) continue;
3311
- docIds[validCount] = internalId;
3312
- termFrequencies[validCount] = p.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : p.termFrequency;
3313
- totalTermFrequency += termFrequencies[validCount];
3314
- let fnIndex = fieldNameTable.indexMap.get(p.fieldName);
3315
- if (fnIndex === void 0) {
3316
- fnIndex = fieldNameTable.names.length;
3317
- fieldNameTable.names.push(p.fieldName);
3318
- fieldNameTable.indexMap.set(p.fieldName, fnIndex);
3319
- }
3320
- fieldNameIndices[validCount] = fnIndex;
3321
- docIdSet.add(internalId);
3322
- if (p.positions.length > 0) hasPositions = true;
3323
- validCount++;
4139
+ let valid = 0;
4140
+ for (const posting of source) {
4141
+ if (resolver.toInternal(posting.docId) === void 0) continue;
4142
+ valid++;
4143
+ if (posting.positions.length > 0) hasPositions = true;
3324
4144
  }
3325
- if (validCount === 0) continue;
3326
- docIds.length = validCount;
3327
- const finalTF = validCount < count ? termFrequencies.slice(0, validCount) : termFrequencies;
3328
- const finalFNI = validCount < count ? fieldNameIndices.slice(0, validCount) : fieldNameIndices;
3329
- let positions = null;
3330
- if (hasPositions) {
3331
- positions = new Array(validCount);
3332
- let vi = 0;
3333
- for (let i = 0; i < count; i++) {
3334
- const internalId = resolver.toInternal(src.postings[i].docId);
3335
- if (internalId === void 0) continue;
3336
- positions[vi] = src.postings[i].positions;
3337
- vi++;
3338
- }
3339
- }
3340
- let ordered = true;
3341
- for (let i = 1; i < validCount; i++) {
3342
- if (docIds[i] < docIds[i - 1]) {
3343
- ordered = false;
3344
- break;
3345
- }
4145
+ if (valid === 0) continue;
4146
+ postings.reserve(token, valid);
4147
+ for (const posting of source) {
4148
+ const internalId = resolver.toInternal(posting.docId);
4149
+ if (internalId === void 0) continue;
4150
+ const termFrequency = posting.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : posting.termFrequency;
4151
+ const fieldIndex = fieldIndexFor(fieldNameTable, posting.fieldName);
4152
+ postings.insert(token, internalId, termFrequency, fieldIndex, hasPositions ? posting.positions : null);
3346
4153
  }
3347
- index.set(token, {
3348
- length: validCount,
3349
- docIds,
3350
- termFrequencies: finalTF,
3351
- fieldNameIndices: finalFNI,
3352
- positions,
3353
- docIdSet,
3354
- deletedDocs: /* @__PURE__ */ new Set(),
3355
- totalTermFrequency,
3356
- structureRevision: 0,
3357
- ordered
3358
- });
3359
- trackToken(token);
3360
4154
  }
3361
4155
  }
3362
4156
  };
3363
4157
  }
3364
4158
 
3365
- // src/core/projection.ts
3366
- function splitPaths(paths) {
3367
- if (!paths) return [];
3368
- const split = [];
3369
- for (const path of paths) {
3370
- const segments = path.split(".").filter((segment) => segment.length > 0);
3371
- if (segments.length === 0) continue;
3372
- if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) continue;
3373
- split.push(segments);
3374
- }
3375
- return split;
3376
- }
3377
- function overlaps(a, b) {
3378
- const shared2 = Math.min(a.length, b.length);
3379
- for (let i = 0; i < shared2; i++) {
3380
- if (a[i] !== b[i]) return false;
3381
- }
3382
- return true;
3383
- }
3384
- function covers(outer, inner) {
3385
- if (outer.length > inner.length) return false;
3386
- for (let i = 0; i < outer.length; i++) {
3387
- if (outer[i] !== inner[i]) return false;
3388
- }
3389
- return true;
3390
- }
3391
- function readPath(source, path) {
3392
- let current = source;
3393
- for (const segment of path) {
3394
- if (typeof current !== "object" || current === null || Array.isArray(current))
3395
- return { found: false, value: void 0 };
3396
- const record = current;
3397
- if (!Object.hasOwn(record, segment)) return { found: false, value: void 0 };
3398
- current = record[segment];
3399
- }
3400
- return { found: true, value: current };
3401
- }
3402
- function writePath(target, path, value) {
3403
- let current = target;
3404
- for (let i = 0; i < path.length - 1; i++) {
3405
- const segment = path[i];
3406
- const existing = current[segment];
3407
- if (typeof existing === "object" && existing !== null && !Array.isArray(existing)) {
3408
- current = existing;
3409
- continue;
3410
- }
3411
- const created = {};
3412
- current[segment] = created;
3413
- current = created;
3414
- }
3415
- current[path[path.length - 1]] = value;
3416
- }
3417
- function cloneExcept(value, path, exclude) {
3418
- const reachesDeeper = exclude.some((excluded) => excluded.length > path.length && covers(path, excluded));
3419
- if (!reachesDeeper) return structuredClone(value);
3420
- if (typeof value !== "object" || value === null || Array.isArray(value)) return structuredClone(value);
3421
- const source = value;
3422
- const kept = {};
3423
- for (const key of Object.keys(source)) {
3424
- const childPath = [...path, key];
3425
- if (exclude.some((excluded) => covers(excluded, childPath))) continue;
3426
- kept[key] = cloneExcept(source[key], childPath, exclude);
3427
- }
3428
- return kept;
3429
- }
3430
- function resolveProjection(projection) {
3431
- if (projection === void 0 || projection === true) return { kind: "full" };
3432
- if (projection === false) return { kind: "none" };
3433
- if (typeof projection !== "object" || projection === null) return { kind: "full" };
3434
- const askedInclude = Array.isArray(projection.include) && projection.include.length > 0;
3435
- const askedExclude = Array.isArray(projection.exclude) && projection.exclude.length > 0;
3436
- if (!askedInclude && !askedExclude) return { kind: "full" };
3437
- return {
3438
- kind: "fields",
3439
- include: askedInclude ? splitPaths(projection.include) : null,
3440
- exclude: splitPaths(projection.exclude)
3441
- };
3442
- }
3443
- function projectionKeepsField(projection, fieldPath) {
3444
- if (projection.kind === "full") return true;
3445
- if (projection.kind === "none") return false;
3446
- const path = fieldPath.split(".").filter((segment) => segment.length > 0);
3447
- if (path.length === 0) return false;
3448
- if (projection.include && !projection.include.some((included) => overlaps(included, path))) return false;
3449
- return !projection.exclude.some((excluded) => covers(excluded, path));
3450
- }
3451
- function cloneProjected(document, projection) {
3452
- if (projection === void 0 || projection.kind === "full") return structuredClone(document);
3453
- if (projection.kind === "none") return {};
3454
- if (projection.include === null) return cloneExcept(document, [], projection.exclude);
3455
- const projected = {};
3456
- for (const path of projection.include) {
3457
- if (projection.exclude.some((excluded) => covers(excluded, path))) continue;
3458
- const { found, value } = readPath(document, path);
3459
- if (!found) continue;
3460
- writePath(projected, path, cloneExcept(value, path, projection.exclude));
3461
- }
3462
- return projected;
3463
- }
3464
-
3465
4159
  // src/core/statistics.ts
3466
4160
  function applyUniqueTokens(tokens, docFrequencies, increment) {
3467
4161
  const fields = Object.keys(tokens);
@@ -3625,6 +4319,9 @@ function createSurfaceRegistry() {
3625
4319
  else totalsByToken.set(token, next);
3626
4320
  }
3627
4321
  return {
4322
+ holds(surface) {
4323
+ return entries.has(surface);
4324
+ },
3628
4325
  add(surface, token, occurrences) {
3629
4326
  if (surface.length === 0 || token.length === 0 || occurrences <= 0) return;
3630
4327
  if (surface === token) return;
@@ -3747,78 +4444,6 @@ function bitsetFromSet(set, capacity) {
3747
4444
  return bs;
3748
4445
  }
3749
4446
 
3750
- // src/core/partition/utils.ts
3751
- function getOrCreateFieldNameIndex(table, fieldName) {
3752
- const existing = table.indexMap.get(fieldName);
3753
- if (existing !== void 0) return existing;
3754
- const idx = table.names.length;
3755
- table.names.push(fieldName);
3756
- table.indexMap.set(fieldName, idx);
3757
- return idx;
3758
- }
3759
- var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
3760
- function getNestedValue(obj, path) {
3761
- const parts = path.split(".");
3762
- let current = obj;
3763
- for (let i = 0; i < parts.length; i++) {
3764
- if (current === null || current === void 0 || typeof current !== "object") return void 0;
3765
- if (!Object.hasOwn(current, parts[i])) return void 0;
3766
- current = current[parts[i]];
3767
- }
3768
- return current;
3769
- }
3770
- function stringArraysEqual(a, b) {
3771
- if (a === b) return true;
3772
- if (!a || !b) return false;
3773
- if (a.length !== b.length) return false;
3774
- for (let i = 0; i < a.length; i++) {
3775
- if (a[i] !== b[i]) return false;
3776
- }
3777
- return true;
3778
- }
3779
- function textFieldsChanged(oldDoc, newDoc, fields) {
3780
- for (const { path, base } of fields) {
3781
- if (!isWordIndexedBase(base)) continue;
3782
- const oldVal = getNestedValue(oldDoc, path);
3783
- const newVal = getNestedValue(newDoc, path);
3784
- if (base === "string") {
3785
- if (oldVal !== newVal) return true;
3786
- } else {
3787
- if (!stringArraysEqual(oldVal, newVal)) return true;
3788
- }
3789
- }
3790
- return false;
3791
- }
3792
- function parseVectorDimension(fieldType) {
3793
- const match = VECTOR_PATTERN2.exec(fieldType);
3794
- return match ? Number.parseInt(match[1], 10) : null;
3795
- }
3796
- function tokenizeOptions(options) {
3797
- return {
3798
- stem: true,
3799
- removeStopWords: true,
3800
- collectSurfaces: options?.collectSurfaces === true,
3801
- stopWordOverride: options?.stopWordOverride,
3802
- customTokenizer: options?.customTokenizer
3803
- };
3804
- }
3805
- function getFlatSchema(state, schema) {
3806
- if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
3807
- state.flatSchemaCache = flattenSchema(schema);
3808
- state.lastSchemaRef = schema;
3809
- return state.flatSchemaCache;
3810
- }
3811
- function getFieldValueForDoc(docStore, docId, fieldPath) {
3812
- const stored = docStore.get(docId);
3813
- if (!stored) return void 0;
3814
- return getNestedValue(stored.fields, fieldPath);
3815
- }
3816
- function getFieldValueByInternalId(docStore, internalId, fieldPath) {
3817
- const externalId = docStore.getExternalId(internalId);
3818
- if (externalId === void 0) return void 0;
3819
- return getFieldValueForDoc(docStore, externalId, fieldPath);
3820
- }
3821
-
3822
4447
  // src/core/partition/facets.ts
3823
4448
  var UNDECLARED_FIELD_TYPE = "";
3824
4449
  function keptFacetValues(limit) {
@@ -3834,6 +4459,7 @@ function ordinalBitsetOf(state, matched) {
3834
4459
  return bitset;
3835
4460
  }
3836
4461
  function* matchedFieldValues(state, fieldPath, matched) {
4462
+ const read = state.docStore.fieldReader(fieldPath);
3837
4463
  if ("ordinalBitset" in matched) {
3838
4464
  const bits = matched.ordinalBitset;
3839
4465
  for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
@@ -3842,14 +4468,15 @@ function* matchedFieldValues(state, fieldPath, matched) {
3842
4468
  const base = wordIndex << 5;
3843
4469
  while (word !== 0) {
3844
4470
  const trailingZeros = Math.clz32(word & -word) ^ 31;
3845
- yield getFieldValueByInternalId(state.docStore, base + trailingZeros, fieldPath);
4471
+ yield read(base + trailingZeros);
3846
4472
  word &= word - 1;
3847
4473
  }
3848
4474
  }
3849
4475
  return;
3850
4476
  }
3851
4477
  for (const docId of matched) {
3852
- yield getFieldValueForDoc(state.docStore, docId, fieldPath);
4478
+ const internalId = state.docStore.getInternalId(docId);
4479
+ yield internalId === void 0 ? void 0 : read(internalId);
3853
4480
  }
3854
4481
  }
3855
4482
  function countValuesFromDocuments(state, fieldPath, matched) {
@@ -6071,7 +6698,7 @@ var BYTES_PER_BOOLEAN_SLOT = 1;
6071
6698
  var BYTES_PER_MIXED_SLOT = 8;
6072
6699
  var BYTES_PER_STRING_CODE_UNIT = 2;
6073
6700
  var STRING_HEADER_BYTES2 = 24;
6074
- function grow(length, wanted) {
6701
+ function grow2(length, wanted) {
6075
6702
  let next = length === 0 ? 64 : length;
6076
6703
  while (next <= wanted) next *= 2;
6077
6704
  return next;
@@ -6106,7 +6733,7 @@ function createValueStore(initialKind) {
6106
6733
  function setNumber(internalId, value) {
6107
6734
  if (numbers === null) return;
6108
6735
  if (internalId >= numbers.length) {
6109
- const grown = new Float64Array(grow(numbers.length, internalId));
6736
+ const grown = new Float64Array(grow2(numbers.length, internalId));
6110
6737
  grown.fill(Number.NaN, numbers.length);
6111
6738
  grown.set(numbers);
6112
6739
  numbers = grown;
@@ -6116,7 +6743,7 @@ function createValueStore(initialKind) {
6116
6743
  function setBoolean(internalId, value) {
6117
6744
  if (booleans === null) return;
6118
6745
  if (internalId >= booleans.length) {
6119
- const grown = new Uint8Array(grow(booleans.length, internalId));
6746
+ const grown = new Uint8Array(grow2(booleans.length, internalId));
6120
6747
  grown.fill(BOOLEAN_MISSING, booleans.length);
6121
6748
  grown.set(booleans);
6122
6749
  booleans = grown;
@@ -6227,10 +6854,9 @@ function createSortColumnSet(docStore) {
6227
6854
  }
6228
6855
  function backfill(key, field, fieldType, mode) {
6229
6856
  const store = createValueStore(kindForFieldType(fieldType));
6230
- for (const [docId, stored] of docStore.all()) {
6231
- const internalId = docStore.getInternalId(docId);
6232
- if (internalId === void 0) continue;
6233
- store.set(internalId, toReducedSortValue(readSortField(stored.fields, field), mode));
6857
+ const read = docStore.fieldReader(field);
6858
+ for (const internalId of docStore.allInternalIds()) {
6859
+ store.set(internalId, toReducedSortValue(read(internalId), mode));
6234
6860
  }
6235
6861
  const entry = {
6236
6862
  field,
@@ -6562,15 +7188,18 @@ function sortedPageOf(state, request) {
6562
7188
  function sortValuesOf(state, docId, fields, fieldTypes, modes) {
6563
7189
  const set = state.sortColumns;
6564
7190
  const internalId = state.docStore.getInternalId(docId);
6565
- const stored = state.docStore.get(docId);
6566
7191
  const key = new Array(fields.length);
6567
7192
  for (let i = 0; i < fields.length; i++) {
6568
7193
  const mode = modes[i] ?? "min";
6569
- if (set !== null && internalId !== void 0 && set.holds(fields[i], fieldTypes[i], mode)) {
7194
+ if (internalId === void 0) {
7195
+ key[i] = null;
7196
+ continue;
7197
+ }
7198
+ if (set?.holds(fields[i], fieldTypes[i], mode)) {
6570
7199
  key[i] = set.column(fields[i], fieldTypes[i], mode).valueOf(internalId);
6571
7200
  continue;
6572
7201
  }
6573
- key[i] = stored === void 0 ? null : toReducedSortValue(readSortField(stored.fields, fields[i]), mode);
7202
+ key[i] = toReducedSortValue(state.docStore.fieldReader(fields[i])(internalId), mode);
6574
7203
  }
6575
7204
  return key;
6576
7205
  }
@@ -6613,7 +7242,15 @@ function buildFilterContext(state, schema, meter) {
6613
7242
  for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
6614
7243
  if (pattern) patternFields.add(fieldPath);
6615
7244
  }
6616
- const getFieldValue = (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath);
7245
+ const fieldReaders = /* @__PURE__ */ new Map();
7246
+ const readerOf = (fieldPath) => {
7247
+ let read = fieldReaders.get(fieldPath);
7248
+ if (read === void 0) {
7249
+ read = state.docStore.fieldReader(fieldPath);
7250
+ fieldReaders.set(fieldPath, read);
7251
+ }
7252
+ return read;
7253
+ };
6617
7254
  let cachedAllDocIdsBitset = null;
6618
7255
  const cachedPresentBitsets = /* @__PURE__ */ new Map();
6619
7256
  return {
@@ -6637,7 +7274,7 @@ function buildFilterContext(state, schema, meter) {
6637
7274
  matches: into,
6638
7275
  meter,
6639
7276
  host: PATTERN_HOST,
6640
- valueOf: (internalId) => getFieldValue(internalId, fieldPath)
7277
+ valueOf: readerOf(fieldPath)
6641
7278
  });
6642
7279
  },
6643
7280
  presentBitset: (fieldPath) => {
@@ -6996,38 +7633,18 @@ function searchFulltextMatches(state, params) {
6996
7633
  // src/core/partition/memory.ts
6997
7634
  var DOC_ENTRY_OVERHEAD_BYTES = 260;
6998
7635
  var LENGTH_COLUMN_SLOT_BYTES = 6;
6999
- var POSTING_ENTRY_BYTES = 40;
7000
- var POSITIONS_ARRAY_OVERHEAD_BYTES = 88;
7001
- var POSITION_BYTES = 8;
7002
- var PER_TERM_OVERHEAD_BYTES = 180;
7636
+ var DOC_FREQUENCY_ENTRY_BYTES = 44;
7003
7637
  var SURFACE_ENTRY_OVERHEAD_BYTES = 140;
7004
7638
  var FIELD_ENTRY_OVERHEAD_BYTES = 42;
7005
7639
  function estimatePartitionBytes(state) {
7006
7640
  const docCount = state.docStore.count();
7007
7641
  if (docCount === 0) return 0;
7008
- const docFreqs = state.stats.docFrequencies;
7009
- let totalPostings = 0;
7010
- let termCount = 0;
7011
- for (const term in docFreqs) {
7012
- totalPostings += docFreqs[term];
7013
- termCount++;
7014
- }
7015
- const fieldLengthTotals = state.stats.totalFieldLengths;
7016
- let totalTokenOccurrences = 0;
7017
- let textFieldCount = 0;
7018
- for (const field in fieldLengthTotals) {
7019
- totalTokenOccurrences += fieldLengthTotals[field];
7020
- textFieldCount++;
7021
- }
7642
+ const textFieldCount = Object.keys(state.stats.totalFieldLengths).length;
7022
7643
  let bytes = state.docStore.contentBytes();
7023
7644
  bytes += docCount * DOC_ENTRY_OVERHEAD_BYTES;
7024
7645
  bytes += textFieldCount * state.docStore.internalIdCapacity() * LENGTH_COLUMN_SLOT_BYTES;
7025
- bytes += totalPostings * POSTING_ENTRY_BYTES;
7026
- if (state.trackPositions) {
7027
- bytes += totalPostings * POSITIONS_ARRAY_OVERHEAD_BYTES;
7028
- bytes += totalTokenOccurrences * POSITION_BYTES;
7029
- }
7030
- bytes += termCount * PER_TERM_OVERHEAD_BYTES;
7646
+ bytes += state.invertedIdx.estimateBytes();
7647
+ bytes += state.invertedIdx.size() * DOC_FREQUENCY_ENTRY_BYTES;
7031
7648
  bytes += state.surfaceRegistry.size() * SURFACE_ENTRY_OVERHEAD_BYTES;
7032
7649
  bytes += docCount * state.numericIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
7033
7650
  bytes += docCount * state.booleanIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
@@ -7063,17 +7680,18 @@ function mergeDocuments(target, source) {
7063
7680
  }
7064
7681
  function mergePostings(target, source, idMapping) {
7065
7682
  for (const token of source.invertedIdx.tokens()) {
7066
- const list = source.invertedIdx.lookup(token);
7683
+ const list = source.invertedIdx.lookupReadOnce?.(token) ?? source.invertedIdx.lookup(token);
7067
7684
  if (list === void 0) continue;
7068
- for (let i = 0; i < list.length; i++) {
7069
- const sourceInternalId = list.docIds[i];
7070
- if (list.deletedDocs.has(sourceInternalId)) continue;
7685
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, positions, length } = list;
7686
+ for (let i = 0; i < length; i++) {
7687
+ const sourceInternalId = docIds[i];
7688
+ if (deletedDocs.has(sourceInternalId)) continue;
7071
7689
  const targetInternalId = idMapping.get(sourceInternalId);
7072
7690
  if (targetInternalId === void 0) continue;
7073
- const fieldName = source.fieldNameTable.names[list.fieldNameIndices[i]];
7691
+ const fieldName = source.fieldNameTable.names[fieldNameIndices[i]];
7074
7692
  const fieldNameIndex2 = getOrCreateFieldNameIndex(target.fieldNameTable, fieldName);
7075
- const positions = list.positions === null ? null : list.positions[i];
7076
- target.invertedIdx.insert(token, targetInternalId, list.termFrequencies[i], fieldNameIndex2, positions ?? null);
7693
+ const entryPositions = positions === null ? null : positions[i];
7694
+ target.invertedIdx.insert(token, targetInternalId, termFrequencies[i], fieldNameIndex2, entryPositions ?? null);
7077
7695
  }
7078
7696
  }
7079
7697
  }
@@ -7185,8 +7803,9 @@ function entriesCovered(bounds) {
7185
7803
  return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
7186
7804
  }
7187
7805
  function scan(list, columns, previous, keepBlocks) {
7806
+ const { docIds, termFrequencies, fieldNameIndices, length } = list;
7188
7807
  const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
7189
- const capacity = keepBlocks + Math.ceil((list.length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
7808
+ const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
7190
7809
  const entryEnd = new Int32Array(capacity);
7191
7810
  const maxTermFrequency = new Int32Array(capacity);
7192
7811
  const minFieldLength = new Float64Array(capacity);
@@ -7208,16 +7827,16 @@ function scan(list, columns, previous, keepBlocks) {
7208
7827
  let blockMaxRun = 0;
7209
7828
  let blockDocs = 0;
7210
7829
  let entry = startEntry;
7211
- while (entry < list.length) {
7212
- const docId = list.docIds[entry];
7830
+ while (entry < length) {
7831
+ const docId = docIds[entry];
7213
7832
  const tombstoned = hasDeleted && deleted.has(docId);
7214
7833
  let run = 0;
7215
- while (entry + run < list.length && list.docIds[entry + run] === docId) {
7834
+ while (entry + run < length && docIds[entry + run] === docId) {
7216
7835
  if (!tombstoned) {
7217
7836
  const position = entry + run;
7218
- const termFrequency = list.termFrequencies[position];
7837
+ const termFrequency = termFrequencies[position];
7219
7838
  if (termFrequency > blockMaxTf) blockMaxTf = termFrequency;
7220
- const fieldLength = fieldLengthBound(columns, list.fieldNameIndices[position], docId);
7839
+ const fieldLength = fieldLengthBound(columns, fieldNameIndices[position], docId);
7221
7840
  if (fieldLength < blockMinLen) blockMinLen = fieldLength;
7222
7841
  }
7223
7842
  run++;
@@ -7227,7 +7846,7 @@ function scan(list, columns, previous, keepBlocks) {
7227
7846
  blockDocs++;
7228
7847
  }
7229
7848
  entry += run;
7230
- if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === list.length) {
7849
+ if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === length) {
7231
7850
  entryEnd[blockIndex] = entry;
7232
7851
  maxTermFrequency[blockIndex] = blockMaxTf;
7233
7852
  minFieldLength[blockIndex] = blockMinLen === Number.POSITIVE_INFINITY ? ABSENT_FIELD_LENGTH_BOUND : blockMinLen;
@@ -7984,10 +8603,11 @@ function countPostings(state, remap) {
7984
8603
  const list = lookupReadOnce(state, token);
7985
8604
  if (list === void 0) continue;
7986
8605
  const listPositions = list.positionCountAt === void 0 ? list.positions : null;
7987
- const tracksPositions = list.positionCountAt === void 0 ? listPositions !== null : state.trackPositions;
7988
- for (let i = 0; i < list.length; i++) {
7989
- const internalId = list.docIds[i];
7990
- if (list.deletedDocs.has(internalId) || remap[internalId] < 0) continue;
8606
+ const tracksPositions = list.holdsPositions ?? (list.positionCountAt === void 0 ? listPositions !== null : state.trackPositions);
8607
+ const { docIds, deletedDocs, length } = list;
8608
+ for (let i = 0; i < length; i++) {
8609
+ const internalId = docIds[i];
8610
+ if (deletedDocs.has(internalId) || remap[internalId] < 0) continue;
7991
8611
  postings++;
7992
8612
  if (!tracksPositions) continue;
7993
8613
  hasPositions = true;
@@ -8038,18 +8658,20 @@ function encodeSegmentState(state) {
8038
8658
  for (const token of state.invertedIdx.tokens()) {
8039
8659
  const list = lookupReadOnce(state, token);
8040
8660
  if (list === void 0) continue;
8041
- const listPositions = positionOffsets === null ? null : list.positions;
8661
+ const copyPositionsAt = list.copyPositionsAt?.bind(list);
8662
+ const listPositions = positionOffsets === null || copyPositionsAt !== void 0 ? null : list.positions;
8042
8663
  const tokenStart = postingCursor;
8043
8664
  const countViaSet = !list.ordered;
8044
8665
  if (countViaSet) unorderedTokenDocs.clear();
8045
8666
  let uniqueDocs = 0;
8046
8667
  let lastCountedDoc = -1;
8047
- for (let i = 0; i < list.length; i++) {
8048
- const internalId = list.docIds[i];
8049
- if (list.deletedDocs.has(internalId) || remap[internalId] < 0) continue;
8668
+ const { docIds: docIds2, termFrequencies, fieldNameIndices, deletedDocs, length } = list;
8669
+ for (let i = 0; i < length; i++) {
8670
+ const internalId = docIds2[i];
8671
+ if (deletedDocs.has(internalId) || remap[internalId] < 0) continue;
8050
8672
  postingDocIds[postingCursor] = remap[internalId];
8051
- postingFrequencies[postingCursor] = list.termFrequencies[i];
8052
- postingFieldIndices[postingCursor] = list.fieldNameIndices[i];
8673
+ postingFrequencies[postingCursor] = termFrequencies[i];
8674
+ postingFieldIndices[postingCursor] = fieldNameIndices[i];
8053
8675
  if (countViaSet) {
8054
8676
  unorderedTokenDocs.add(internalId);
8055
8677
  } else if (internalId !== lastCountedDoc) {
@@ -8059,7 +8681,9 @@ function encodeSegmentState(state) {
8059
8681
  if (positionOffsets !== null && positionValues !== null) {
8060
8682
  positionOffsets[postingCursor] = positionCursor;
8061
8683
  const entryPositions = listPositions === null ? null : listPositions[i];
8062
- if (entryPositions !== null && entryPositions !== void 0) {
8684
+ if (copyPositionsAt !== void 0) {
8685
+ positionCursor += copyPositionsAt(i, positionValues, positionCursor);
8686
+ } else if (entryPositions !== null && entryPositions !== void 0) {
8063
8687
  for (const position of entryPositions) positionValues[positionCursor++] = position;
8064
8688
  }
8065
8689
  }
@@ -8269,17 +8893,18 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
8269
8893
  for (const token of state.invertedIdx.tokens()) {
8270
8894
  const list = state.invertedIdx.lookup(token);
8271
8895
  if (!list) continue;
8272
- const hasDeleted = list.deletedDocs.size > 0;
8896
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, positions, length } = list;
8897
+ const hasDeleted = deletedDocs.size > 0;
8273
8898
  const postings = [];
8274
- for (let i = 0; i < list.length; i++) {
8275
- if (hasDeleted && list.deletedDocs.has(list.docIds[i])) continue;
8276
- const externalId = resolver.toExternal(list.docIds[i]);
8899
+ for (let i = 0; i < length; i++) {
8900
+ if (hasDeleted && deletedDocs.has(docIds[i])) continue;
8901
+ const externalId = resolver.toExternal(docIds[i]);
8277
8902
  if (externalId === void 0) continue;
8278
8903
  postings.push({
8279
8904
  docId: externalId,
8280
- termFrequency: list.termFrequencies[i],
8281
- field: fieldNames[list.fieldNameIndices[i]],
8282
- positions: list.positions ? list.positions[i] : []
8905
+ termFrequency: termFrequencies[i],
8906
+ field: fieldNames[fieldNameIndices[i]],
8907
+ positions: positions === null ? [] : Array.from(positions[i] ?? [])
8283
8908
  });
8284
8909
  }
8285
8910
  if (postings.length > 0) {
@@ -8674,6 +9299,11 @@ function createPartitionIndex(partitionId, trackPositions = true) {
8674
9299
  if (!stored) return void 0;
8675
9300
  return stored.fields;
8676
9301
  },
9302
+ fieldValues(docId, fieldPaths) {
9303
+ const stored = state.docStore.get(docId);
9304
+ if (!stored) return void 0;
9305
+ return fieldPaths.map((fieldPath) => getNestedValue(stored.fields, fieldPath));
9306
+ },
8677
9307
  has(docId) {
8678
9308
  return state.docStore.has(docId);
8679
9309
  },
@@ -10177,7 +10807,7 @@ function frozenSegmentBytes(segment) {
10177
10807
  bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
10178
10808
  }
10179
10809
  for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
10180
- return bytes + segment.partialWords.bytes();
10810
+ return bytes + segment.partialWords.bytes() + segment.removalBytes();
10181
10811
  }
10182
10812
 
10183
10813
  // src/core/partition/composite/compaction.ts
@@ -10215,19 +10845,40 @@ function freezeLiveTailInto(live, liveState, frozen, freeze) {
10215
10845
  frozen.push(segment);
10216
10846
  return segment;
10217
10847
  }
10218
- function swapFrozenSegmentList(frozen, dropSegmentIds, replacement) {
10848
+ function swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings) {
10219
10849
  const dropSet = new Set(dropSegmentIds);
10220
10850
  const dropped = frozen.filter((segment) => dropSet.has(segment.segmentId));
10851
+ const applied = /* @__PURE__ */ new Map();
10221
10852
  for (const segment of dropped) {
10222
10853
  for (const docId of segment.tombstonedDocIds()) {
10223
10854
  if (!replacement.hasDocument(docId)) continue;
10224
10855
  if (dropped.some((other) => other.hasDocument(docId))) continue;
10225
- replacement.tombstoneDocument(docId);
10856
+ const spellings = removedSpellings?.get(docId) ?? replacement.removedSpellingsOf(docId);
10857
+ if (spellings !== void 0) applied.set(docId, spellings);
10858
+ replacement.tombstoneDocument(docId, spellings);
10226
10859
  }
10227
10860
  }
10228
10861
  const kept = frozen.filter((segment) => !dropSet.has(segment.segmentId));
10229
10862
  frozen.length = 0;
10230
10863
  frozen.push(...kept, replacement);
10864
+ return applied;
10865
+ }
10866
+
10867
+ // src/core/partition/composite/documents.ts
10868
+ function compositeDocument(frozen, live, docId, projection) {
10869
+ for (const segment of frozen) {
10870
+ const projected = segment.docStore.projectedDocument(docId, projection);
10871
+ if (projected !== void 0) return projected;
10872
+ }
10873
+ return live.get(docId, projection);
10874
+ }
10875
+ function compositeFieldValues(frozen, live, docId, fieldPaths) {
10876
+ for (const segment of frozen) {
10877
+ const internalId = segment.docStore.getInternalId(docId);
10878
+ if (internalId === void 0) continue;
10879
+ return fieldPaths.map((fieldPath) => segment.docStore.fieldReader(fieldPath)(internalId));
10880
+ }
10881
+ return live.fieldValues(docId, fieldPaths);
10231
10882
  }
10232
10883
 
10233
10884
  // src/core/partition/composite/filters.ts
@@ -10553,7 +11204,7 @@ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
10553
11204
  function isCompositePartition(partition) {
10554
11205
  return "attachFrozenSegment" in partition;
10555
11206
  }
10556
- function createCompositePartition(partitionId, trackPositions = true, existingLive) {
11207
+ function createCompositePartition(partitionId, trackPositions = true, existingLive, textAnalysis) {
10557
11208
  const live = existingLive ?? createPartitionIndex(partitionId, trackPositions);
10558
11209
  const liveState = partitionStateOf(live);
10559
11210
  const frozen = [];
@@ -10561,6 +11212,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10561
11212
  function invalidateDocFrequencies() {
10562
11213
  docFrequenciesCache = null;
10563
11214
  }
11215
+ function bindSegments() {
11216
+ invalidateDocFrequencies();
11217
+ if (textAnalysis === void 0) return;
11218
+ for (const segment of frozen) segment.removals.bind(textAnalysis);
11219
+ }
10564
11220
  function cachedDocFrequencies() {
10565
11221
  if (docFrequenciesCache === null) {
10566
11222
  docFrequenciesCache = mergeDocFrequencies(subs());
@@ -10614,23 +11270,25 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10614
11270
  frozenSegmentsById(segmentIds) {
10615
11271
  return resolveFrozenSegments(frozen, segmentIds, partitionId);
10616
11272
  },
10617
- swapFrozenSegments(dropSegmentIds, replacement) {
10618
- swapFrozenSegmentList(frozen, dropSegmentIds, replacement);
10619
- invalidateDocFrequencies();
11273
+ swapFrozenSegments(dropSegmentIds, replacement, removedSpellings) {
11274
+ if (textAnalysis !== void 0) replacement.removals.bind(textAnalysis);
11275
+ const applied = swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings);
11276
+ bindSegments();
11277
+ return applied;
10620
11278
  },
10621
11279
  freezeLiveTail(freeze) {
10622
11280
  const segment = freezeLiveTailInto(live, liveState, frozen, freeze);
10623
- if (segment !== null) invalidateDocFrequencies();
11281
+ if (segment !== null) bindSegments();
10624
11282
  return segment;
10625
11283
  },
10626
11284
  replaceLiveTail(segment) {
10627
11285
  live.clear();
10628
11286
  frozen.push(segment);
10629
- invalidateDocFrequencies();
11287
+ bindSegments();
10630
11288
  },
10631
11289
  attachFrozenSegment(segment) {
10632
11290
  frozen.push(segment);
10633
- invalidateDocFrequencies();
11291
+ bindSegments();
10634
11292
  },
10635
11293
  mergeSegment(segment) {
10636
11294
  invalidateDocFrequencies();
@@ -10669,7 +11327,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10669
11327
  invalidateDocFrequencies();
10670
11328
  live.insert(docId, document, schema, language, options);
10671
11329
  },
10672
- update(docId, document, schema, language, options) {
11330
+ update(docId, document, schema, language, options, removedSpellings) {
10673
11331
  invalidateDocFrequencies();
10674
11332
  const owner = frozenOwner(docId);
10675
11333
  if (owner === void 0) {
@@ -10677,9 +11335,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10677
11335
  return;
10678
11336
  }
10679
11337
  live.insert(docId, document, schema, language, options);
10680
- owner.tombstoneDocument(docId);
11338
+ owner.tombstoneDocument(docId, removedSpellings);
10681
11339
  },
10682
- remove(docId, schema, language, options) {
11340
+ remove(docId, schema, language, options, removedSpellings) {
10683
11341
  if (live.has(docId)) {
10684
11342
  invalidateDocFrequencies();
10685
11343
  live.remove(docId, schema, language, options);
@@ -10692,7 +11350,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10692
11350
  partitionId
10693
11351
  });
10694
11352
  }
10695
- owner.tombstoneDocument(docId);
11353
+ invalidateDocFrequencies();
11354
+ owner.tombstoneDocument(docId, removedSpellings);
11355
+ },
11356
+ removedSpellingsOf(docId) {
11357
+ return live.has(docId) ? void 0 : frozenOwner(docId)?.removedSpellingsOf(docId);
10696
11358
  },
10697
11359
  beginBatch() {
10698
11360
  live.beginBatch();
@@ -10755,9 +11417,10 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10755
11417
  expandPartialWords(subs(), analysis, []);
10756
11418
  },
10757
11419
  get(docId, projection) {
10758
- const stored = frozenStored(docId);
10759
- if (stored !== void 0) return cloneProjected(stored.fields, projection);
10760
- return live.get(docId, projection);
11420
+ return compositeDocument(frozen, live, docId, projection);
11421
+ },
11422
+ fieldValues(docId, fieldPaths) {
11423
+ return compositeFieldValues(frozen, live, docId, fieldPaths);
10761
11424
  },
10762
11425
  getRef(docId) {
10763
11426
  const stored = frozenStored(docId);
@@ -10776,8 +11439,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10776
11439
  },
10777
11440
  *docIds() {
10778
11441
  for (const segment of frozen) {
10779
- for (const [docId] of segment.docStore.all()) {
10780
- yield docId;
11442
+ for (const internalId of segment.docStore.allInternalIds()) {
11443
+ const docId = segment.docStore.getExternalId(internalId);
11444
+ if (docId !== void 0) yield docId;
10781
11445
  }
10782
11446
  }
10783
11447
  yield* live.docIds();
@@ -10886,6 +11550,25 @@ function setNestedValue(obj, path, value) {
10886
11550
  }
10887
11551
  current[segments[segments.length - 1]] = value;
10888
11552
  }
11553
+ function copyVectorsInto(document, docId, vectorIndexes, projection) {
11554
+ for (const [fieldPath, vectorIndex] of vectorIndexes) {
11555
+ if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
11556
+ const vector = vectorIndex.getVector(docId);
11557
+ if (vector) setNestedValue(document, fieldPath, vector);
11558
+ }
11559
+ }
11560
+
11561
+ // src/partitioning/manager/segments.ts
11562
+ function requireNewSegmentDocuments(segment, docPartitionMap) {
11563
+ for (const internalId of segment.docStore.allInternalIds()) {
11564
+ const docId = segment.docStore.getExternalId(internalId);
11565
+ if (docId === void 0 || !docPartitionMap.has(docId)) continue;
11566
+ throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
11567
+ docId,
11568
+ partitionId: docPartitionMap.get(docId)
11569
+ });
11570
+ }
11571
+ }
10889
11572
 
10890
11573
  // src/partitioning/manager/stats.ts
10891
11574
  function aggregateStatsOf(partitions) {
@@ -10969,6 +11652,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
10969
11652
  const count = initialPartitionCount ?? 1;
10970
11653
  const analysis = resolveIndexAnalysis(config);
10971
11654
  const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
11655
+ const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
10972
11656
  const trackPositions = config.trackPositions ?? true;
10973
11657
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
10974
11658
  let partitions = [];
@@ -10997,10 +11681,8 @@ function createPartitionManager(indexName, config, language, router, initialPart
10997
11681
  }
10998
11682
  function asCompositePartition(partitionId) {
10999
11683
  const existing = partitions[partitionId];
11000
- if ("attachFrozenSegment" in existing) {
11001
- return existing;
11002
- }
11003
- const composite = createCompositePartition(partitionId, trackPositions, existing);
11684
+ if (isCompositePartition(existing)) return existing;
11685
+ const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
11004
11686
  partitions[partitionId] = composite;
11005
11687
  return composite;
11006
11688
  }
@@ -11050,6 +11732,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
11050
11732
  get partialWordAnalysis() {
11051
11733
  return partialWordAnalysis;
11052
11734
  },
11735
+ get textAnalysis() {
11736
+ return textAnalysis;
11737
+ },
11053
11738
  getPartition(partitionId) {
11054
11739
  validatePartitionId(partitionId);
11055
11740
  return partitions[partitionId];
@@ -11133,12 +11818,17 @@ function createPartitionManager(indexName, config, language, router, initialPart
11133
11818
  docPartitionMap.set(docId, pid);
11134
11819
  writeLog.documentWritten(docId);
11135
11820
  },
11136
- remove(docId) {
11821
+ removedSpellingsOf(docId) {
11822
+ const pid = locateDocument(docId);
11823
+ const partition = pid === void 0 ? void 0 : partitions[pid];
11824
+ return partition !== void 0 && isCompositePartition(partition) ? partition.removedSpellingsOf(docId) : void 0;
11825
+ },
11826
+ remove(docId, removedSpellings) {
11137
11827
  const pid = locateDocument(docId);
11138
11828
  if (pid === void 0) {
11139
11829
  throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
11140
11830
  }
11141
- partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
11831
+ partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
11142
11832
  docPartitionMap.delete(docId);
11143
11833
  writeLog.documentWritten(docId);
11144
11834
  },
@@ -11152,13 +11842,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
11152
11842
  partitions[i].endBatch();
11153
11843
  }
11154
11844
  },
11155
- update(docId, document, options) {
11845
+ update(docId, document, options, removedSpellings) {
11156
11846
  const pid = locateDocument(docId);
11157
11847
  if (pid === void 0) {
11158
11848
  throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
11159
11849
  }
11160
11850
  const updateOpts = resolveInsertOptions(options);
11161
- partitions[pid].update(docId, document, config.schema, language, updateOpts);
11851
+ partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
11162
11852
  docPartitionMap.set(docId, pid);
11163
11853
  writeLog.documentWritten(docId);
11164
11854
  },
@@ -11166,14 +11856,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11166
11856
  const pid = locateDocument(docId);
11167
11857
  if (pid === void 0) return void 0;
11168
11858
  const doc = partitions[pid].get(docId, projection);
11169
- if (!doc) return void 0;
11170
- for (const [fieldPath, vecIndex] of vecIndexes) {
11171
- if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
11172
- const vector = vecIndex.getVector(docId);
11173
- if (vector) {
11174
- setNestedValue(doc, fieldPath, vector);
11175
- }
11176
- }
11859
+ if (doc) copyVectorsInto(doc, docId, vecIndexes, projection);
11177
11860
  return doc;
11178
11861
  },
11179
11862
  getRef(docId) {
@@ -11181,6 +11864,10 @@ function createPartitionManager(indexName, config, language, router, initialPart
11181
11864
  if (pid === void 0) return void 0;
11182
11865
  return partitions[pid].getRef(docId);
11183
11866
  },
11867
+ fieldValues(docId, fieldPaths) {
11868
+ const pid = locateDocument(docId);
11869
+ return pid === void 0 ? void 0 : partitions[pid].fieldValues(docId, fieldPaths);
11870
+ },
11184
11871
  sortValues(docId, fields, fieldTypes, modes) {
11185
11872
  const pid = locateDocument(docId);
11186
11873
  if (pid === void 0) return fields.map(() => null);
@@ -11216,22 +11903,20 @@ function createPartitionManager(indexName, config, language, router, initialPart
11216
11903
  },
11217
11904
  attachFrozenSegment(partitionId, segment) {
11218
11905
  validatePartitionId(partitionId);
11219
- for (const internalId of segment.docStore.allInternalIds()) {
11220
- const docId = segment.docStore.getExternalId(internalId);
11221
- if (docId === void 0 || !docPartitionMap.has(docId)) continue;
11222
- throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
11223
- docId,
11224
- partitionId: docPartitionMap.get(docId)
11225
- });
11226
- }
11906
+ requireNewSegmentDocuments(segment, docPartitionMap);
11227
11907
  asCompositePartition(partitionId).attachFrozenSegment(segment);
11228
11908
  registerSegmentDocuments(partitionId, segment);
11229
11909
  writeLog.segmentWritten(segment);
11230
11910
  },
11231
- swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
11911
+ swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
11232
11912
  validatePartitionId(partitionId);
11233
- asCompositePartition(partitionId).swapFrozenSegments(dropSegmentIds, replacement);
11913
+ const applied = asCompositePartition(partitionId).swapFrozenSegments(
11914
+ dropSegmentIds,
11915
+ replacement,
11916
+ removedSpellings
11917
+ );
11234
11918
  registerSegmentDocuments(partitionId, replacement);
11919
+ return applied;
11235
11920
  },
11236
11921
  freezeLiveTail(partitionId, freeze) {
11237
11922
  validatePartitionId(partitionId);
@@ -11291,21 +11976,15 @@ function fanOutMatchCount(manager, params, language, schema, options) {
11291
11976
  // src/core/partition/frozen/doc-table.ts
11292
11977
  function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
11293
11978
  const capacity = idTable.count;
11294
- const storedCache = new Array(capacity);
11295
11979
  let sortedIds = null;
11296
11980
  let sortedRevision = -1;
11297
11981
  function storedAt(ordinal) {
11298
- let stored = storedCache[ordinal];
11299
- if (stored === void 0) {
11300
- const fieldLengths = {};
11301
- for (const [fieldName, column] of lengthColumns) {
11302
- const length = column[ordinal];
11303
- if (length > 0) fieldLengths[fieldName] = length;
11304
- }
11305
- stored = { fields: documents.docAt(ordinal), fieldLengths };
11306
- storedCache[ordinal] = stored;
11982
+ const fieldLengths = {};
11983
+ for (const [fieldName, column] of lengthColumns) {
11984
+ const length = column[ordinal];
11985
+ if (length > 0) fieldLengths[fieldName] = length;
11307
11986
  }
11308
- return stored;
11987
+ return { fields: documents.docAt(ordinal), fieldLengths };
11309
11988
  }
11310
11989
  function liveOrdinal(docId) {
11311
11990
  const ordinal = idTable.ordinalOf(docId);
@@ -11374,9 +12053,241 @@ function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
11374
12053
  },
11375
12054
  resolver() {
11376
12055
  return resolver;
12056
+ },
12057
+ fieldReader(fieldPath) {
12058
+ const read = documents.fieldReader(fieldPath);
12059
+ return (internalId) => resolver.isLive(internalId) ? read(internalId) : void 0;
12060
+ },
12061
+ projectedDocument(docId, projection) {
12062
+ const ordinal = liveOrdinal(docId);
12063
+ return ordinal === void 0 ? void 0 : documents.projectedAt(ordinal, projection);
11377
12064
  }
11378
12065
  };
11379
12066
  }
12067
+ var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
12068
+ var keyDecoder = new TextDecoder("utf-8", { fatal: true });
12069
+ function headerAt(view, offset) {
12070
+ if (offset >= view.byteLength) return null;
12071
+ const tag = view.getUint8(offset);
12072
+ if (tag <= 127 || tag >= 224) return { headerBytes: 1, bodyBytes: 0, children: 0 };
12073
+ if (tag <= 143) return { headerBytes: 1, bodyBytes: 0, children: (tag & 15) * 2 };
12074
+ if (tag <= 159) return { headerBytes: 1, bodyBytes: 0, children: tag & 15 };
12075
+ if (tag <= 191) return { headerBytes: 1, bodyBytes: tag & 31, children: 0 };
12076
+ const lengthBytes = LENGTH_BYTES_BY_TAG.get(tag);
12077
+ if (lengthBytes !== void 0) {
12078
+ if (offset + 1 + lengthBytes > view.byteLength) return null;
12079
+ const length = lengthBytes === 1 ? view.getUint8(offset + 1) : lengthBytes === 2 ? view.getUint16(offset + 1, false) : view.getUint32(offset + 1, false);
12080
+ const headerBytes = 1 + lengthBytes;
12081
+ if (ARRAY_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length };
12082
+ if (MAP_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length * 2 };
12083
+ return { headerBytes, bodyBytes: length + (EXT_TAGS.has(tag) ? 1 : 0), children: 0 };
12084
+ }
12085
+ const fixedBytes = FIXED_BODY_BYTES_BY_TAG.get(tag);
12086
+ return fixedBytes === void 0 ? null : { headerBytes: 1, bodyBytes: fixedBytes, children: 0 };
12087
+ }
12088
+ var LENGTH_BYTES_BY_TAG = /* @__PURE__ */ new Map([
12089
+ [196, 1],
12090
+ [197, 2],
12091
+ [198, 4],
12092
+ [199, 1],
12093
+ [200, 2],
12094
+ [201, 4],
12095
+ [217, 1],
12096
+ [218, 2],
12097
+ [219, 4],
12098
+ [220, 2],
12099
+ [221, 4],
12100
+ [222, 2],
12101
+ [223, 4]
12102
+ ]);
12103
+ var ARRAY_TAGS = /* @__PURE__ */ new Set([220, 221]);
12104
+ var MAP_TAGS = /* @__PURE__ */ new Set([222, 223]);
12105
+ var EXT_TAGS = /* @__PURE__ */ new Set([199, 200, 201]);
12106
+ var FIXED_BODY_BYTES_BY_TAG = /* @__PURE__ */ new Map([
12107
+ [192, 0],
12108
+ [194, 0],
12109
+ [195, 0],
12110
+ [202, 4],
12111
+ [203, 8],
12112
+ [204, 1],
12113
+ [205, 2],
12114
+ [206, 4],
12115
+ [207, 8],
12116
+ [208, 1],
12117
+ [209, 2],
12118
+ [210, 4],
12119
+ [211, 8],
12120
+ [212, 2],
12121
+ [213, 3],
12122
+ [214, 5],
12123
+ [215, 9],
12124
+ [216, 17]
12125
+ ]);
12126
+ function endOfValue(view, start) {
12127
+ let offset = start;
12128
+ let pending = 1;
12129
+ while (pending > 0) {
12130
+ const header = headerAt(view, offset);
12131
+ if (header === null) return null;
12132
+ offset += header.headerBytes + header.bodyBytes;
12133
+ if (offset > view.byteLength) return null;
12134
+ pending += header.children - 1;
12135
+ }
12136
+ return offset;
12137
+ }
12138
+ function stringKeyAt(bytes, view, offset) {
12139
+ const header = headerAt(view, offset);
12140
+ if (header === null || header.children !== 0) return null;
12141
+ const tag = view.getUint8(offset);
12142
+ const isString = tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
12143
+ const end = offset + header.headerBytes + header.bodyBytes;
12144
+ if (!isString || end > bytes.length) return null;
12145
+ try {
12146
+ return { key: keyDecoder.decode(bytes.subarray(offset + header.headerBytes, end)), end };
12147
+ } catch {
12148
+ return null;
12149
+ }
12150
+ }
12151
+ function topLevelMapKeys(bytes) {
12152
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
12153
+ const top = headerAt(view, 0);
12154
+ if (top === null || top.bodyBytes !== 0) return null;
12155
+ const tag = view.getUint8(0);
12156
+ if (!(tag >= 128 && tag <= 143 || MAP_TAGS.has(tag))) return null;
12157
+ const keys = /* @__PURE__ */ new Set();
12158
+ let offset = top.headerBytes;
12159
+ for (let pair = 0; pair < top.children / 2; pair += 1) {
12160
+ const key = stringKeyAt(bytes, view, offset);
12161
+ if (key === null) return null;
12162
+ const valueEnd = endOfValue(view, key.end);
12163
+ if (valueEnd === null) return null;
12164
+ keys.add(key.key);
12165
+ offset = valueEnd;
12166
+ }
12167
+ return keys;
12168
+ }
12169
+ function decodeMapWithoutFields(bytes, skippedFields) {
12170
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
12171
+ const top = headerAt(view, 0);
12172
+ if (top === null || top.children === 0 || top.bodyBytes !== 0) return null;
12173
+ const tag = view.getUint8(0);
12174
+ const isMap = tag >= 128 && tag <= 143 || MAP_TAGS.has(tag);
12175
+ if (!isMap) return null;
12176
+ const document = {};
12177
+ let offset = top.headerBytes;
12178
+ for (let pair = 0; pair < top.children / 2; pair += 1) {
12179
+ const key = stringKeyAt(bytes, view, offset);
12180
+ if (key === null || FORBIDDEN_KEYS.has(key.key)) return null;
12181
+ const valueEnd = endOfValue(view, key.end);
12182
+ if (valueEnd === null) return null;
12183
+ if (!skippedFields.has(key.key)) {
12184
+ try {
12185
+ document[key.key] = decode(bytes.subarray(key.end, valueEnd));
12186
+ } catch {
12187
+ return null;
12188
+ }
12189
+ }
12190
+ offset = valueEnd;
12191
+ }
12192
+ return offset === bytes.length ? document : null;
12193
+ }
12194
+
12195
+ // src/serialization/msgpack-path.ts
12196
+ var encoder2 = new TextEncoder();
12197
+ var PROTO_KEY_BYTES = encoder2.encode("__proto__");
12198
+ var MISSING = { found: false, value: void 0 };
12199
+ var pathDecoder = new Decoder();
12200
+ function encodePathSegments(segments) {
12201
+ if (segments.length === 0) return null;
12202
+ if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) return null;
12203
+ return segments.map((segment) => encoder2.encode(segment));
12204
+ }
12205
+ function isStringTag(tag) {
12206
+ return tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
12207
+ }
12208
+ function isMapTag(tag) {
12209
+ return tag >= 128 && tag <= 143 || tag === 222 || tag === 223;
12210
+ }
12211
+ function isArrayTag(tag) {
12212
+ return tag >= 144 && tag <= 159 || tag === 220 || tag === 221;
12213
+ }
12214
+ function isScalarTag(tag) {
12215
+ if (tag <= 127 || tag >= 224 || isStringTag(tag)) return true;
12216
+ return tag === 192 || tag === 194 || tag === 195 || tag >= 202 && tag <= 211;
12217
+ }
12218
+ function sameBytes(bytes, start, end, target) {
12219
+ if (end - start !== target.length) return false;
12220
+ for (let at = 0; at < target.length; at++) {
12221
+ if (bytes[start + at] !== target[at]) return false;
12222
+ }
12223
+ return true;
12224
+ }
12225
+ function keyRangeAt(view, offset) {
12226
+ if (!isStringTag(view.getUint8(offset))) return null;
12227
+ const header = headerAt(view, offset);
12228
+ if (header === null) return null;
12229
+ const start = offset + header.headerBytes;
12230
+ return { start, end: start + header.bodyBytes };
12231
+ }
12232
+ function readEncodedPath(bytes, segments, arrays) {
12233
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
12234
+ let start = 0;
12235
+ let end = bytes.length;
12236
+ for (let depth = 0; depth < segments.length; depth++) {
12237
+ if (start >= end) return null;
12238
+ const tag = view.getUint8(start);
12239
+ if (!isMapTag(tag)) {
12240
+ if (depth === 0) return null;
12241
+ if (isArrayTag(tag)) return arrays === "missing" ? MISSING : null;
12242
+ return isScalarTag(tag) ? MISSING : null;
12243
+ }
12244
+ const header = headerAt(view, start);
12245
+ if (header === null) return null;
12246
+ let offset = start + header.headerBytes;
12247
+ let valueStart = -1;
12248
+ let valueEnd = -1;
12249
+ for (let pair = 0; pair < header.children / 2; pair++) {
12250
+ if (offset >= end) return null;
12251
+ const key = keyRangeAt(view, offset);
12252
+ if (key === null || key.end > end) return null;
12253
+ if (sameBytes(bytes, key.start, key.end, PROTO_KEY_BYTES)) return null;
12254
+ const next = endOfValue(view, key.end);
12255
+ if (next === null || next > end) return null;
12256
+ if (sameBytes(bytes, key.start, key.end, segments[depth])) {
12257
+ valueStart = key.end;
12258
+ valueEnd = next;
12259
+ }
12260
+ offset = next;
12261
+ }
12262
+ if (offset !== end) return null;
12263
+ if (valueStart < 0) return MISSING;
12264
+ start = valueStart;
12265
+ end = valueEnd;
12266
+ }
12267
+ try {
12268
+ return { found: true, value: pathDecoder.decode(bytes.subarray(start, end)) };
12269
+ } catch {
12270
+ return null;
12271
+ }
12272
+ }
12273
+
12274
+ // src/core/partition/frozen/document-source.ts
12275
+ var encodedProjectionPaths = /* @__PURE__ */ new WeakMap();
12276
+ function encodedProjectionPathOf(path) {
12277
+ let segments = encodedProjectionPaths.get(path);
12278
+ if (segments === void 0) {
12279
+ segments = encodePathSegments(path);
12280
+ encodedProjectionPaths.set(path, segments);
12281
+ }
12282
+ return segments;
12283
+ }
12284
+ function sharesPrefix(left, right) {
12285
+ const shared2 = Math.min(left.length, right.length);
12286
+ for (let at = 0; at < shared2; at++) {
12287
+ if (left[at] !== right[at]) return false;
12288
+ }
12289
+ return true;
12290
+ }
11380
12291
  function removeFieldAtPath(document, fieldPath) {
11381
12292
  const segments = fieldPath.split(".");
11382
12293
  let current = document;
@@ -11442,25 +12353,40 @@ function encodeDocumentTableData(documents) {
11442
12353
  function wrapEncodedDocumentTable(data) {
11443
12354
  const { blob, offsets } = data;
11444
12355
  const count = offsets.length - 1;
11445
- const decoded = new Array(count);
11446
- const encodedBytes = blob.byteLength + offsets.byteLength;
11447
- let decodedBytes = 0;
12356
+ const discarded = (data.discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
12357
+ const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
12358
+ const bytesAt = (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]);
12359
+ const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
12360
+ function projectedPathAt(ordinal, path) {
12361
+ const segments = encodedProjectionPathOf(path);
12362
+ if (segments === null) return { found: false, value: void 0 };
12363
+ const read = touchesDiscarded(path) ? null : readEncodedPath(bytesAt(ordinal), segments, "missing");
12364
+ if (read === null) return readProjectedPath(docAt(ordinal), path);
12365
+ return read.found ? { found: true, value: valueWithOwnBuffers(read.value) } : read;
12366
+ }
11448
12367
  return {
11449
12368
  count,
11450
- get byteLength() {
11451
- return encodedBytes + decodedBytes;
11452
- },
11453
- docAt(ordinal) {
11454
- let document = decoded[ordinal];
11455
- if (document === void 0) {
11456
- document = decodeStoredDocument(blob.slice(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
11457
- decoded[ordinal] = document;
11458
- decodedBytes += documentValueBytes(document);
11459
- }
11460
- return document;
11461
- },
12369
+ byteLength: blob.byteLength + offsets.byteLength,
12370
+ docAt,
11462
12371
  readAt(ordinal) {
11463
- return decoded[ordinal] ?? decodeStoredDocument(blob.subarray(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
12372
+ return decodeStoredDocument(bytesAt(ordinal), data.discardedFields);
12373
+ },
12374
+ fieldReader(fieldPath) {
12375
+ const path = fieldPath.split(".");
12376
+ const segments = encodePathSegments(path);
12377
+ if (segments === null) return () => void 0;
12378
+ const decodesDocument = touchesDiscarded(path);
12379
+ return (ordinal) => {
12380
+ const read = decodesDocument ? null : readEncodedPath(bytesAt(ordinal), segments, "decode-document");
12381
+ if (read !== null) return valueWithOwnBuffers(read.value);
12382
+ return getNestedValue(docAt(ordinal), fieldPath);
12383
+ };
12384
+ },
12385
+ projectedAt(ordinal, projection) {
12386
+ if (projection === void 0 || projection.kind === "full") return docAt(ordinal);
12387
+ if (projection.kind === "none") return {};
12388
+ if (projection.include === null) return cloneProjected(docAt(ordinal), projection);
12389
+ return projectIncludedPaths(projection.include, projection.exclude, (path) => projectedPathAt(ordinal, path));
11464
12390
  }
11465
12391
  };
11466
12392
  }
@@ -11477,6 +12403,12 @@ function wrapDocumentArray(documents) {
11477
12403
  },
11478
12404
  readAt(ordinal) {
11479
12405
  return documents[ordinal];
12406
+ },
12407
+ fieldReader(fieldPath) {
12408
+ return (ordinal) => getNestedValue(documents[ordinal], fieldPath);
12409
+ },
12410
+ projectedAt(ordinal, projection) {
12411
+ return cloneProjected(documents[ordinal], projection);
11480
12412
  }
11481
12413
  };
11482
12414
  }
@@ -11484,18 +12416,18 @@ function wrapDocumentArray(documents) {
11484
12416
  // src/core/partition/frozen/string-blob.ts
11485
12417
  var SURROGATE = /[\ud800-\udfff]/;
11486
12418
  var UTF8_BYTES_PER_UTF16_UNIT = 3;
11487
- var encoder2 = new TextEncoder();
12419
+ var encoder3 = new TextEncoder();
11488
12420
  var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
11489
12421
  function encodeLookupKey(text) {
11490
- if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder2.encode(text);
11491
- const { written } = encoder2.encodeInto(text, lookupKeyBytes);
12422
+ if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder3.encode(text);
12423
+ const { written } = encoder3.encodeInto(text, lookupKeyBytes);
11492
12424
  return lookupKeyBytes.subarray(0, written);
11493
12425
  }
11494
12426
  function encodeOneByOne(strings) {
11495
12427
  const encoded = new Array(strings.length);
11496
12428
  let blobLength = 0;
11497
12429
  for (let i = 0; i < strings.length; i++) {
11498
- encoded[i] = encoder2.encode(strings[i]);
12430
+ encoded[i] = encoder3.encode(strings[i]);
11499
12431
  blobLength += encoded[i].length;
11500
12432
  }
11501
12433
  const blob = new Uint8Array(blobLength);
@@ -11519,7 +12451,7 @@ function basicPlaneUtf8Bytes(text) {
11519
12451
  function encodeStringBlob(strings) {
11520
12452
  const joined2 = strings.join("");
11521
12453
  if (SURROGATE.test(joined2)) return encodeOneByOne(strings);
11522
- const blob = encoder2.encode(joined2);
12454
+ const blob = encoder3.encode(joined2);
11523
12455
  const offsets = new Uint32Array(strings.length + 1);
11524
12456
  let cursor = 0;
11525
12457
  for (let i = 0; i < strings.length; i++) {
@@ -11854,9 +12786,13 @@ function createFrozenEnumReader(entry) {
11854
12786
  }
11855
12787
 
11856
12788
  // src/core/partition/frozen/inverted-reader.ts
11857
- function createFrozenInvertedReader(table, views) {
12789
+ function createFrozenInvertedReader(table, views, counts) {
11858
12790
  function viewAtSorted(sortedIndex) {
11859
- return views.viewAt(table.payloadSlot(sortedIndex), table.documentFrequencyAt(sortedIndex));
12791
+ return views.viewAt(table.payloadSlot(sortedIndex), sortedIndex);
12792
+ }
12793
+ function liveAt(token) {
12794
+ const at = table.find(token);
12795
+ return at < 0 || counts.documentFrequency(at) === 0 ? -1 : at;
11860
12796
  }
11861
12797
  function candidateRange(prefix) {
11862
12798
  if (prefix.length === 0) return { start: 0, end: table.size };
@@ -11864,22 +12800,22 @@ function createFrozenInvertedReader(table, views) {
11864
12800
  }
11865
12801
  return {
11866
12802
  lookup(token) {
11867
- const at = table.find(token);
12803
+ const at = liveAt(token);
11868
12804
  if (at < 0) return void 0;
11869
12805
  return viewAtSorted(at);
11870
12806
  },
11871
12807
  lookupReadOnce(token) {
11872
- const at = table.find(token);
12808
+ const at = liveAt(token);
11873
12809
  if (at < 0) return void 0;
11874
- return views.viewReadOnce(table.payloadSlot(at), table.documentFrequencyAt(at));
12810
+ return views.viewReadOnce(table.payloadSlot(at), at);
11875
12811
  },
11876
12812
  documentFrequency(token) {
11877
12813
  const at = table.find(token);
11878
- return at < 0 ? 0 : table.documentFrequencyAt(at);
12814
+ return at < 0 ? 0 : counts.documentFrequency(at);
11879
12815
  },
11880
12816
  fuzzyLookup(token, tolerance, prefixLength) {
11881
12817
  if (tolerance === 0) {
11882
- const at = table.find(token);
12818
+ const at = liveAt(token);
11883
12819
  return at < 0 ? [] : [{ token, postingList: viewAtSorted(at) }];
11884
12820
  }
11885
12821
  const prefix = fuzzyPrefixOf(token, prefixLength);
@@ -11890,7 +12826,7 @@ function createFrozenInvertedReader(table, views) {
11890
12826
  const candidate = table.tokenAt(at);
11891
12827
  if (!candidate.startsWith(requiredPrefix)) continue;
11892
12828
  const { withinTolerance } = boundedLevenshtein(token, candidate, tolerance);
11893
- if (withinTolerance) {
12829
+ if (withinTolerance && counts.documentFrequency(at) > 0) {
11894
12830
  results.push({ token: candidate, postingList: viewAtSorted(at) });
11895
12831
  }
11896
12832
  }
@@ -11901,14 +12837,15 @@ function createFrozenInvertedReader(table, views) {
11901
12837
  const range = table.prefixRange(prefix);
11902
12838
  const results = [];
11903
12839
  for (let at = range.start; at < range.end; at++) {
11904
- results.push({ term: table.tokenAt(at), documentFrequency: table.documentFrequencyAt(at) });
12840
+ const documentFrequency = counts.documentFrequency(at);
12841
+ if (documentFrequency > 0) results.push({ term: table.tokenAt(at), documentFrequency });
11905
12842
  }
11906
12843
  results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.term, b.term));
11907
12844
  if (results.length > limit) results.length = limit;
11908
12845
  return results;
11909
12846
  },
11910
12847
  has(token) {
11911
- return table.find(token) >= 0;
12848
+ return liveAt(token) >= 0;
11912
12849
  },
11913
12850
  *tokens() {
11914
12851
  for (let at = 0; at < table.size; at++) {
@@ -11921,6 +12858,154 @@ function createFrozenInvertedReader(table, views) {
11921
12858
  };
11922
12859
  }
11923
12860
 
12861
+ // src/core/partition/frozen/live-counts.ts
12862
+ var UINT32_BYTES = 4;
12863
+ var FLOAT64_BYTES = 8;
12864
+ function removalIndexBytes(columns, terms) {
12865
+ const documentWords = 2 * (columns.documentCount + 1);
12866
+ return (columns.postingDocIds.length + documentWords + 2 * terms) * UINT32_BYTES + terms * FLOAT64_BYTES;
12867
+ }
12868
+ function processHasRoomFor(bytes) {
12869
+ return true;
12870
+ }
12871
+ function buildRemovalIndex(columns, table) {
12872
+ const { documentCount, postingDocIds } = columns;
12873
+ const starts = new Uint32Array(documentCount + 1);
12874
+ for (let row = 0; row < postingDocIds.length; row++) starts[postingDocIds[row] + 1]++;
12875
+ for (let ordinal = 0; ordinal < documentCount; ordinal++) starts[ordinal + 1] += starts[ordinal];
12876
+ const cursors = starts.slice(0, documentCount);
12877
+ const rows = new Uint32Array(postingDocIds.length);
12878
+ for (let row = 0; row < postingDocIds.length; row++) rows[cursors[postingDocIds[row]]++] = row;
12879
+ const termOfSlot = new Uint32Array(table.size);
12880
+ for (let sortedIndex = 0; sortedIndex < table.size; sortedIndex++) {
12881
+ termOfSlot[table.payloadSlot(sortedIndex)] = sortedIndex;
12882
+ }
12883
+ return {
12884
+ starts,
12885
+ rows,
12886
+ termOfSlot,
12887
+ documents: new Uint32Array(table.size),
12888
+ occurrences: new Float64Array(table.size)
12889
+ };
12890
+ }
12891
+ function tryBuildRemovalIndex(columns, table) {
12892
+ if (!processHasRoomFor(removalIndexBytes(columns, table.size))) return null;
12893
+ try {
12894
+ return buildRemovalIndex(columns, table);
12895
+ } catch (error) {
12896
+ if (error instanceof RangeError) return null;
12897
+ throw error;
12898
+ }
12899
+ }
12900
+ function slotOfRow(postingOffsets, slots, row) {
12901
+ let low = 0;
12902
+ let high = slots;
12903
+ while (low < high) {
12904
+ const middle = low + high >>> 1;
12905
+ if (postingOffsets[middle + 1] <= row) low = middle + 1;
12906
+ else high = middle;
12907
+ }
12908
+ return low;
12909
+ }
12910
+ function createFrozenLiveCounts(columns, table, tombstones, storedFrequencies) {
12911
+ const { postingOffsets, postingDocIds, postingFrequencies } = columns;
12912
+ let index = null;
12913
+ let lengths = null;
12914
+ let frequencies = null;
12915
+ let frequenciesRevision = -1;
12916
+ function countRemoval(built, ordinal) {
12917
+ const { starts, rows, termOfSlot, documents, occurrences } = built;
12918
+ let previousSlot = -1;
12919
+ for (let at = starts[ordinal]; at < starts[ordinal + 1]; at++) {
12920
+ const row = rows[at];
12921
+ const slot = slotOfRow(postingOffsets, table.size, row);
12922
+ const sortedIndex = termOfSlot[slot];
12923
+ if (slot !== previousSlot) documents[sortedIndex]++;
12924
+ occurrences[sortedIndex] += postingFrequencies[row];
12925
+ previousSlot = slot;
12926
+ }
12927
+ }
12928
+ function removedDocumentsOf(sortedIndex) {
12929
+ const slot = table.payloadSlot(sortedIndex);
12930
+ const end = postingOffsets[slot + 1];
12931
+ let removed = 0;
12932
+ let previous = -1;
12933
+ let ordered = true;
12934
+ for (let row = postingOffsets[slot]; row < end; row++) {
12935
+ const ordinal = postingDocIds[row];
12936
+ if (ordinal < previous) ordered = false;
12937
+ if (ordinal !== previous && tombstones.has(ordinal)) removed++;
12938
+ previous = ordinal;
12939
+ }
12940
+ if (ordered) return removed;
12941
+ const seen = /* @__PURE__ */ new Set();
12942
+ for (let row = postingOffsets[slot]; row < end; row++) {
12943
+ const ordinal = postingDocIds[row];
12944
+ if (tombstones.has(ordinal)) seen.add(ordinal);
12945
+ }
12946
+ return seen.size;
12947
+ }
12948
+ function removedOccurrencesOf(sortedIndex) {
12949
+ const slot = table.payloadSlot(sortedIndex);
12950
+ const end = postingOffsets[slot + 1];
12951
+ let removed = 0;
12952
+ for (let row = postingOffsets[slot]; row < end; row++) {
12953
+ if (tombstones.has(postingDocIds[row])) removed += postingFrequencies[row];
12954
+ }
12955
+ return removed;
12956
+ }
12957
+ function documentFrequency(sortedIndex) {
12958
+ const stored = table.documentFrequencyAt(sortedIndex);
12959
+ if (tombstones.size === 0) return stored;
12960
+ if (index !== null) return stored - index.documents[sortedIndex];
12961
+ return stored - removedDocumentsOf(sortedIndex);
12962
+ }
12963
+ return {
12964
+ totalDocuments() {
12965
+ return columns.documentCount - tombstones.size;
12966
+ },
12967
+ totalFieldLengths() {
12968
+ return lengths ?? columns.totalFieldLengths;
12969
+ },
12970
+ documentFrequency,
12971
+ removedOccurrences(sortedIndex) {
12972
+ if (tombstones.size === 0) return 0;
12973
+ if (index !== null) return index.occurrences[sortedIndex];
12974
+ return removedOccurrencesOf(sortedIndex);
12975
+ },
12976
+ docFrequencies() {
12977
+ if (tombstones.size === 0) return storedFrequencies();
12978
+ if (frequencies !== null && frequenciesRevision === tombstones.revision) return frequencies;
12979
+ const live = /* @__PURE__ */ Object.create(null);
12980
+ for (let at = 0; at < table.size; at++) {
12981
+ const documents = documentFrequency(at);
12982
+ if (documents > 0) live[table.tokenAt(at)] = documents;
12983
+ }
12984
+ frequencies = live;
12985
+ frequenciesRevision = tombstones.revision;
12986
+ return frequencies;
12987
+ },
12988
+ subtractRemoved(ordinal) {
12989
+ lengths ??= { ...columns.totalFieldLengths };
12990
+ for (let field = 0; field < columns.fieldLengthNames.length; field++) {
12991
+ lengths[columns.fieldLengthNames[field]] -= columns.fieldLengthColumns[field][ordinal];
12992
+ }
12993
+ if (index !== null) {
12994
+ countRemoval(index, ordinal);
12995
+ return;
12996
+ }
12997
+ const built = tryBuildRemovalIndex(columns, table);
12998
+ if (built === null) return;
12999
+ for (const removed of tombstones.values()) countRemoval(built, removed);
13000
+ index = built;
13001
+ },
13002
+ bytes() {
13003
+ if (index === null) return 0;
13004
+ return index.starts.byteLength + index.rows.byteLength + index.termOfSlot.byteLength + index.documents.byteLength + index.occurrences.byteLength;
13005
+ }
13006
+ };
13007
+ }
13008
+
11924
13009
  // src/core/partition/frozen/posting-views.ts
11925
13010
  function isOrdered(docIds) {
11926
13011
  for (let i = 1; i < docIds.length; i++) {
@@ -11933,16 +13018,16 @@ function sumFrequencies(frequencies) {
11933
13018
  for (let i = 0; i < frequencies.length; i++) total += frequencies[i];
11934
13019
  return total;
11935
13020
  }
11936
- function createFrozenPostingViews(payload, tombstones) {
13021
+ function createFrozenPostingViews(payload, tombstones, counts) {
11937
13022
  const views = /* @__PURE__ */ new Map();
11938
- function buildView(payloadSlot, documentFrequency, keepsPositions) {
13023
+ function buildView(payloadSlot, sortedIndex, keepsPositions) {
11939
13024
  const start = payload.postingOffsets[payloadSlot];
11940
13025
  const end = payload.postingOffsets[payloadSlot + 1];
11941
13026
  const docIds = payload.postingDocIds.subarray(start, end);
11942
13027
  const termFrequencies = payload.postingFrequencies.subarray(start, end);
11943
13028
  const fieldNameIndices = payload.postingFieldIndices.subarray(start, end);
11944
13029
  const ordered = isOrdered(docIds);
11945
- const totalTermFrequency = sumFrequencies(termFrequencies);
13030
+ const storedTermFrequency = sumFrequencies(termFrequencies);
11946
13031
  const { positionOffsets, positionValues } = payload;
11947
13032
  let materializedPositions = null;
11948
13033
  return {
@@ -11964,9 +13049,15 @@ function createFrozenPostingViews(payload, tombstones) {
11964
13049
  if (positionOffsets === null) return 0;
11965
13050
  return positionOffsets[start + row + 1] - positionOffsets[start + row];
11966
13051
  },
11967
- docIdSet: { size: documentFrequency },
13052
+ docIdSet: {
13053
+ get size() {
13054
+ return counts.documentFrequency(sortedIndex);
13055
+ }
13056
+ },
11968
13057
  deletedDocs: tombstones,
11969
- totalTermFrequency,
13058
+ get totalTermFrequency() {
13059
+ return storedTermFrequency - counts.removedOccurrences(sortedIndex);
13060
+ },
11970
13061
  get structureRevision() {
11971
13062
  return tombstones.revision;
11972
13063
  },
@@ -11974,16 +13065,168 @@ function createFrozenPostingViews(payload, tombstones) {
11974
13065
  };
11975
13066
  }
11976
13067
  return {
11977
- viewAt(payloadSlot, documentFrequency) {
11978
- let view = views.get(payloadSlot);
11979
- if (view === void 0) {
11980
- view = buildView(payloadSlot, documentFrequency, true);
11981
- views.set(payloadSlot, view);
13068
+ viewAt(payloadSlot, sortedIndex) {
13069
+ let view = views.get(payloadSlot);
13070
+ if (view === void 0) {
13071
+ view = buildView(payloadSlot, sortedIndex, true);
13072
+ views.set(payloadSlot, view);
13073
+ }
13074
+ return view;
13075
+ },
13076
+ viewReadOnce(payloadSlot, sortedIndex) {
13077
+ return buildView(payloadSlot, sortedIndex, false);
13078
+ }
13079
+ };
13080
+ }
13081
+
13082
+ // src/core/partition/partial-words/spellings.ts
13083
+ function textsOf(value) {
13084
+ if (typeof value === "string") return [value];
13085
+ if (!Array.isArray(value)) return [];
13086
+ return value.filter((element) => typeof element === "string");
13087
+ }
13088
+ function visitFieldTexts(document, fields, visit) {
13089
+ for (const field of fields) {
13090
+ for (const text of textsOf(getNestedValue(document, field))) visit(text);
13091
+ }
13092
+ }
13093
+ function visitDocumentSpellings(document, analysis, visit) {
13094
+ const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
13095
+ const visitOccurrence = (token, _position, surface) => {
13096
+ visit(token, surface ?? token);
13097
+ };
13098
+ visitFieldTexts(document, analysis.fields, (text) => visitTokens(text, analysis.language, tokenizing, visitOccurrence));
13099
+ }
13100
+ function visitDocumentSurfaceSpellings(document, analysis, visit) {
13101
+ if (analysis.options?.customTokenizer !== void 0) return;
13102
+ visitFieldTexts(document, analysis.fields, (text) => visitSurfaceSpellings(text, analysis.language, visit));
13103
+ }
13104
+ function sameFields(left, right) {
13105
+ if (left.length !== right.length) return false;
13106
+ for (let at = 0; at < left.length; at++) {
13107
+ if (left[at] !== right[at]) return false;
13108
+ }
13109
+ return true;
13110
+ }
13111
+
13112
+ // src/core/partition/frozen/removal-pass.ts
13113
+ var NO_REMOVED_SPELLINGS = [];
13114
+ function createFrozenRemovalPass(documents, surfaces) {
13115
+ let bound = null;
13116
+ let pending = [];
13117
+ function heldSpellingsOf(ordinal, analysis) {
13118
+ const spellings = [];
13119
+ if (!surfaces.holdsAny()) return spellings;
13120
+ visitDocumentSurfaceSpellings(documents.documentAt(ordinal), analysis, (spelling) => {
13121
+ if (surfaces.holds(spelling)) spellings.push(spelling);
13122
+ });
13123
+ return spellings;
13124
+ }
13125
+ return {
13126
+ get settled() {
13127
+ return pending.length === 0;
13128
+ },
13129
+ bind(analysis) {
13130
+ if (bound !== null) return;
13131
+ const scoped = { ...analysis, fields: documents.textFields };
13132
+ bound = scoped;
13133
+ for (const ordinal of pending) surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, scoped));
13134
+ pending = [];
13135
+ },
13136
+ spellingsOf(ordinal) {
13137
+ return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
13138
+ },
13139
+ afterRemoval(ordinal, spellings) {
13140
+ if (spellings !== void 0) {
13141
+ surfaces.subtractSpellings(ordinal, spellings);
13142
+ } else if (bound !== null) {
13143
+ surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, bound));
13144
+ } else {
13145
+ pending.push(ordinal);
13146
+ }
13147
+ }
13148
+ };
13149
+ }
13150
+
13151
+ // src/core/partition/frozen/surface-slot.ts
13152
+ function applyDocumentSurfaces(target, document, analysis, direction) {
13153
+ if (direction === -1) {
13154
+ visitDocumentSurfaceSpellings(document, analysis, (spelling) => target.subtract(spelling, 1));
13155
+ return;
13156
+ }
13157
+ visitDocumentSpellings(document, analysis, (token, spelling) => {
13158
+ if (spelling !== token) target.add(spelling, token, 1);
13159
+ });
13160
+ }
13161
+ function addForms(target, forms) {
13162
+ for (const surface of Object.keys(forms)) {
13163
+ const value = forms[surface];
13164
+ if (Array.isArray(value)) target.add(surface, value[1], value[0]);
13165
+ }
13166
+ }
13167
+ function createFrozenSurfaceSlot(load, documents) {
13168
+ let registry = null;
13169
+ let uncounted = null;
13170
+ function loaded2() {
13171
+ if (registry === null) {
13172
+ const built = createSurfaceRegistry();
13173
+ const forms = load();
13174
+ if (forms !== null) built.deserialize(forms);
13175
+ registry = built;
13176
+ }
13177
+ return registry;
13178
+ }
13179
+ function isCounted(ordinal) {
13180
+ return uncounted === null || uncounted[ordinal] === 0;
13181
+ }
13182
+ return {
13183
+ reader: {
13184
+ candidatesForPrefix: (prefix) => loaded2().candidatesForPrefix(prefix),
13185
+ stemChangedTotalFor: (token) => loaded2().stemChangedTotalFor(token),
13186
+ size: () => loaded2().size(),
13187
+ serialize: () => loaded2().serialize()
13188
+ },
13189
+ holdsAny() {
13190
+ return loaded2().size() > 0;
13191
+ },
13192
+ holds(spelling) {
13193
+ return loaded2().holds(spelling);
13194
+ },
13195
+ subtractSpellings(ordinal, spellings) {
13196
+ uncounted ??= new Uint8Array(documents.count);
13197
+ uncounted[ordinal] = 1;
13198
+ if (spellings.length === 0) return;
13199
+ const target = loaded2();
13200
+ for (const spelling of spellings) target.subtract(spelling, 1);
13201
+ },
13202
+ addKeptForms(target, mergedOrdinals, analysis) {
13203
+ const source = loaded2();
13204
+ if (analysis === null) {
13205
+ addForms(target, source.serialize());
13206
+ return;
11982
13207
  }
11983
- return view;
11984
- },
11985
- viewReadOnce(payloadSlot, documentFrequency) {
11986
- return buildView(payloadSlot, documentFrequency, false);
13208
+ const scoped = { ...analysis, fields: documents.textFields };
13209
+ let kept = 0;
13210
+ let corrections = 0;
13211
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
13212
+ const keeps = mergedOrdinals[ordinal] >= 0;
13213
+ if (keeps) kept++;
13214
+ if (keeps !== isCounted(ordinal)) corrections++;
13215
+ }
13216
+ if (corrections > kept) {
13217
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
13218
+ if (mergedOrdinals[ordinal] >= 0) applyDocumentSurfaces(target, documents.documentAt(ordinal), scoped, 1);
13219
+ }
13220
+ return;
13221
+ }
13222
+ const corrected = createSurfaceRegistry();
13223
+ addForms(corrected, source.serialize());
13224
+ for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
13225
+ const keeps = mergedOrdinals[ordinal] >= 0;
13226
+ if (keeps === isCounted(ordinal)) continue;
13227
+ applyDocumentSurfaces(corrected, documents.documentAt(ordinal), scoped, keeps ? 1 : -1);
13228
+ }
13229
+ addForms(target, corrected.serialize());
11987
13230
  }
11988
13231
  };
11989
13232
  }
@@ -12011,27 +13254,9 @@ function decodeSurfaceTable(table) {
12011
13254
  }
12012
13255
  return forms;
12013
13256
  }
12014
- function createLazySurfaceReader(load) {
12015
- let registry = null;
12016
- function loaded2() {
12017
- if (registry === null) {
12018
- const built = createSurfaceRegistry();
12019
- const forms = load();
12020
- if (forms !== null) built.deserialize(forms);
12021
- registry = built;
12022
- }
12023
- return registry;
12024
- }
12025
- return {
12026
- candidatesForPrefix: (prefix) => loaded2().candidatesForPrefix(prefix),
12027
- stemChangedTotalFor: (token) => loaded2().stemChangedTotalFor(token),
12028
- size: () => loaded2().size(),
12029
- serialize: () => loaded2().serialize()
12030
- };
12031
- }
12032
13257
 
12033
13258
  // src/core/partition/frozen/token-table.ts
12034
- var encoder3 = new TextEncoder();
13259
+ var encoder4 = new TextEncoder();
12035
13260
  var decoder3 = new TextDecoder();
12036
13261
  function compareBytes2(blob, start, end, query) {
12037
13262
  const length = end - start;
@@ -12118,7 +13343,7 @@ function wrapFrozenTokenTable(data) {
12118
13343
  return documentFrequencies[sortedIndex];
12119
13344
  },
12120
13345
  prefixRange(prefix) {
12121
- const bytes = encoder3.encode(prefix);
13346
+ const bytes = encoder4.encode(prefix);
12122
13347
  const start = lowerBound3(bytes);
12123
13348
  return { start, end: prefixEnd(start, bytes) };
12124
13349
  },
@@ -12136,8 +13361,8 @@ function wrapFrozenTokenTable(data) {
12136
13361
  fromCodePoint = unit;
12137
13362
  toCodePoint = nextCodePoint(unit);
12138
13363
  }
12139
- const from = encoder3.encode(String.fromCodePoint(fromCodePoint));
12140
- const to = encoder3.encode(String.fromCodePoint(toCodePoint));
13364
+ const from = encoder4.encode(String.fromCodePoint(fromCodePoint));
13365
+ const to = encoder4.encode(String.fromCodePoint(toCodePoint));
12141
13366
  return { start: lowerBound3(from), end: lowerBound3(to) };
12142
13367
  }
12143
13368
  };
@@ -12149,6 +13374,7 @@ function buildFrozenTokenTable(tokens, docFrequencies) {
12149
13374
  // src/core/partition/frozen/tombstones.ts
12150
13375
  function createFrozenTombstones() {
12151
13376
  const removed = /* @__PURE__ */ new Set();
13377
+ const arrivals = [];
12152
13378
  let revision = 0;
12153
13379
  return {
12154
13380
  get size() {
@@ -12163,24 +13389,19 @@ function createFrozenTombstones() {
12163
13389
  add(internalId) {
12164
13390
  if (removed.has(internalId)) return false;
12165
13391
  removed.add(internalId);
13392
+ arrivals.push(internalId);
12166
13393
  revision++;
12167
13394
  return true;
12168
13395
  },
13396
+ at(arrival) {
13397
+ return arrivals[arrival];
13398
+ },
12169
13399
  values() {
12170
13400
  return removed.values();
12171
13401
  }
12172
13402
  };
12173
13403
  }
12174
13404
 
12175
- // src/core/partition/partial-words/spellings.ts
12176
- function sameFields(left, right) {
12177
- if (left.length !== right.length) return false;
12178
- for (let at = 0; at < left.length; at++) {
12179
- if (left[at] !== right[at]) return false;
12180
- }
12181
- return true;
12182
- }
12183
-
12184
13405
  // src/core/partition/frozen/shared-snapshot.ts
12185
13406
  function sharedUint32(source) {
12186
13407
  const copy = new Uint32Array(new SharedArrayBuffer(source.length * 4));
@@ -12345,26 +13566,36 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis =
12345
13566
  }
12346
13567
 
12347
13568
  // src/core/partition/frozen/index.ts
12348
- function buildStatsView(source) {
12349
- const averageFieldLengths = {};
12350
- if (source.documentCount > 0) {
12351
- for (const [fieldName, total] of Object.entries(source.totalFieldLengths)) {
12352
- averageFieldLengths[fieldName] = total / source.documentCount;
12353
- }
12354
- }
12355
- let materializedFrequencies = null;
13569
+ function buildStatsView(counts) {
12356
13570
  return {
12357
- totalDocuments: source.documentCount,
12358
- totalFieldLengths: source.totalFieldLengths,
12359
- averageFieldLengths,
12360
- get docFrequencies() {
12361
- if (materializedFrequencies === null) {
12362
- materializedFrequencies = source.docFrequencies();
13571
+ get totalDocuments() {
13572
+ return counts.totalDocuments();
13573
+ },
13574
+ get totalFieldLengths() {
13575
+ return counts.totalFieldLengths();
13576
+ },
13577
+ get averageFieldLengths() {
13578
+ const documents = counts.totalDocuments();
13579
+ const averages = {};
13580
+ if (documents > 0) {
13581
+ for (const [fieldName, total] of Object.entries(counts.totalFieldLengths())) {
13582
+ averages[fieldName] = total / documents;
13583
+ }
12363
13584
  }
12364
- return materializedFrequencies;
13585
+ return averages;
13586
+ },
13587
+ get docFrequencies() {
13588
+ return counts.docFrequencies();
12365
13589
  }
12366
13590
  };
12367
13591
  }
13592
+ function storedFrequenciesOf(source) {
13593
+ let materialized = null;
13594
+ return () => {
13595
+ if (materialized === null) materialized = source.docFrequencies();
13596
+ return materialized;
13597
+ };
13598
+ }
12368
13599
  function buildGeoReaders(entries) {
12369
13600
  const readers = /* @__PURE__ */ new Map();
12370
13601
  for (const entry of entries) {
@@ -12376,9 +13607,10 @@ function buildGeoReaders(entries) {
12376
13607
  }
12377
13608
  return readers;
12378
13609
  }
12379
- function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
13610
+ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
12380
13611
  const tombstones = createFrozenTombstones();
12381
- const postingViews = createFrozenPostingViews(source, tombstones);
13612
+ const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
13613
+ const postingViews = createFrozenPostingViews(source, tombstones, counts);
12382
13614
  const lengthColumns = /* @__PURE__ */ new Map();
12383
13615
  for (let i = 0; i < source.fieldLengthNames.length; i++) {
12384
13616
  lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
@@ -12389,6 +13621,14 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
12389
13621
  documentAt: (ordinal) => documentSource.readAt(ordinal),
12390
13622
  isTombstoned: (ordinal) => tombstones.has(ordinal)
12391
13623
  });
13624
+ const textDocuments = {
13625
+ count: documentSource.count,
13626
+ textFields: source.fieldLengthNames,
13627
+ documentAt: (ordinal) => documentSource.readAt(ordinal)
13628
+ };
13629
+ const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
13630
+ const surfaces = createFrozenSurfaceSlot(loadSurfaceForms, textDocuments);
13631
+ const removals = createFrozenRemovalPass(textDocuments, surfaces);
12392
13632
  const numericIndexes = /* @__PURE__ */ new Map();
12393
13633
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
12394
13634
  const sizeIndexes = /* @__PURE__ */ new Map();
@@ -12410,10 +13650,12 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
12410
13650
  documentSource,
12411
13651
  sharedSnapshot,
12412
13652
  arrays: source,
12413
- invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews),
13653
+ invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
12414
13654
  docStore,
12415
- stats: buildStatsView(source),
12416
- surfaceRegistry: createLazySurfaceReader(source.readSurfaceForms),
13655
+ stats: buildStatsView(counts),
13656
+ surfaceRegistry: surfaces.reader,
13657
+ surfaces,
13658
+ removals,
12417
13659
  numericIndexes,
12418
13660
  booleanIndexes,
12419
13661
  enumIndexes,
@@ -12436,15 +13678,21 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
12436
13678
  isTombstoned(ordinal) {
12437
13679
  return tombstones.has(ordinal);
12438
13680
  },
12439
- tombstoneDocument(docId) {
13681
+ removedSpellingsOf(docId) {
13682
+ const ordinal = docStore.getInternalId(docId);
13683
+ if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
13684
+ return removals.spellingsOf(ordinal);
13685
+ },
13686
+ tombstoneDocument(docId, removedSpellings) {
12440
13687
  const ordinal = docStore.getInternalId(docId);
12441
13688
  if (ordinal === void 0) return false;
12442
13689
  const added = tombstones.add(ordinal);
12443
13690
  if (added) {
13691
+ counts.subtractRemoved(ordinal);
12444
13692
  partialWords.afterRemoval(ordinal);
13693
+ removals.afterRemoval(ordinal, removedSpellings);
12445
13694
  docStore.releaseSortedDocIds();
12446
- segment.sortColumns = null;
12447
- segment.scoreBuffer = null;
13695
+ segment.sortColumns?.forget(ordinal);
12448
13696
  }
12449
13697
  return added;
12450
13698
  },
@@ -12452,6 +13700,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
12452
13700
  const docIds = [];
12453
13701
  for (const ordinal of tombstones.values()) docIds.push(source.idTable.idAt(ordinal));
12454
13702
  return docIds;
13703
+ },
13704
+ removalBytes() {
13705
+ return counts.bytes();
12455
13706
  }
12456
13707
  };
12457
13708
  return segment;
@@ -12473,7 +13724,7 @@ function createFrozenSegment(payload, documents, segmentId) {
12473
13724
  payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, tokenTable)
12474
13725
  );
12475
13726
  }
12476
- function createSharedFrozenSegment(snapshot) {
13727
+ function createSharedFrozenSegment(snapshot, survivingSurfaceForms) {
12477
13728
  const tokenTable = wrapFrozenTokenTable(snapshot.tokenTable);
12478
13729
  return assembleFrozenSegment(
12479
13730
  snapshot.segmentId,
@@ -12493,136 +13744,10 @@ function createSharedFrozenSegment(snapshot) {
12493
13744
  },
12494
13745
  wrapEncodedDocumentTable(snapshot.documentTable),
12495
13746
  snapshot,
12496
- snapshot.partialWords
13747
+ snapshot.partialWords,
13748
+ survivingSurfaceForms
12497
13749
  );
12498
13750
  }
12499
- var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
12500
- var keyDecoder = new TextDecoder("utf-8", { fatal: true });
12501
- function headerAt(view, offset) {
12502
- if (offset >= view.byteLength) return null;
12503
- const tag = view.getUint8(offset);
12504
- if (tag <= 127 || tag >= 224) return { headerBytes: 1, bodyBytes: 0, children: 0 };
12505
- if (tag <= 143) return { headerBytes: 1, bodyBytes: 0, children: (tag & 15) * 2 };
12506
- if (tag <= 159) return { headerBytes: 1, bodyBytes: 0, children: tag & 15 };
12507
- if (tag <= 191) return { headerBytes: 1, bodyBytes: tag & 31, children: 0 };
12508
- const lengthBytes = LENGTH_BYTES_BY_TAG.get(tag);
12509
- if (lengthBytes !== void 0) {
12510
- if (offset + 1 + lengthBytes > view.byteLength) return null;
12511
- const length = lengthBytes === 1 ? view.getUint8(offset + 1) : lengthBytes === 2 ? view.getUint16(offset + 1, false) : view.getUint32(offset + 1, false);
12512
- const headerBytes = 1 + lengthBytes;
12513
- if (ARRAY_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length };
12514
- if (MAP_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length * 2 };
12515
- return { headerBytes, bodyBytes: length + (EXT_TAGS.has(tag) ? 1 : 0), children: 0 };
12516
- }
12517
- const fixedBytes = FIXED_BODY_BYTES_BY_TAG.get(tag);
12518
- return fixedBytes === void 0 ? null : { headerBytes: 1, bodyBytes: fixedBytes, children: 0 };
12519
- }
12520
- var LENGTH_BYTES_BY_TAG = /* @__PURE__ */ new Map([
12521
- [196, 1],
12522
- [197, 2],
12523
- [198, 4],
12524
- [199, 1],
12525
- [200, 2],
12526
- [201, 4],
12527
- [217, 1],
12528
- [218, 2],
12529
- [219, 4],
12530
- [220, 2],
12531
- [221, 4],
12532
- [222, 2],
12533
- [223, 4]
12534
- ]);
12535
- var ARRAY_TAGS = /* @__PURE__ */ new Set([220, 221]);
12536
- var MAP_TAGS = /* @__PURE__ */ new Set([222, 223]);
12537
- var EXT_TAGS = /* @__PURE__ */ new Set([199, 200, 201]);
12538
- var FIXED_BODY_BYTES_BY_TAG = /* @__PURE__ */ new Map([
12539
- [192, 0],
12540
- [194, 0],
12541
- [195, 0],
12542
- [202, 4],
12543
- [203, 8],
12544
- [204, 1],
12545
- [205, 2],
12546
- [206, 4],
12547
- [207, 8],
12548
- [208, 1],
12549
- [209, 2],
12550
- [210, 4],
12551
- [211, 8],
12552
- [212, 2],
12553
- [213, 3],
12554
- [214, 5],
12555
- [215, 9],
12556
- [216, 17]
12557
- ]);
12558
- function endOfValue(view, start) {
12559
- let offset = start;
12560
- let pending = 1;
12561
- while (pending > 0) {
12562
- const header = headerAt(view, offset);
12563
- if (header === null) return null;
12564
- offset += header.headerBytes + header.bodyBytes;
12565
- if (offset > view.byteLength) return null;
12566
- pending += header.children - 1;
12567
- }
12568
- return offset;
12569
- }
12570
- function stringKeyAt(bytes, view, offset) {
12571
- const header = headerAt(view, offset);
12572
- if (header === null || header.children !== 0) return null;
12573
- const tag = view.getUint8(offset);
12574
- const isString = tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
12575
- const end = offset + header.headerBytes + header.bodyBytes;
12576
- if (!isString || end > bytes.length) return null;
12577
- try {
12578
- return { key: keyDecoder.decode(bytes.subarray(offset + header.headerBytes, end)), end };
12579
- } catch {
12580
- return null;
12581
- }
12582
- }
12583
- function topLevelMapKeys(bytes) {
12584
- const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
12585
- const top = headerAt(view, 0);
12586
- if (top === null || top.bodyBytes !== 0) return null;
12587
- const tag = view.getUint8(0);
12588
- if (!(tag >= 128 && tag <= 143 || MAP_TAGS.has(tag))) return null;
12589
- const keys = /* @__PURE__ */ new Set();
12590
- let offset = top.headerBytes;
12591
- for (let pair = 0; pair < top.children / 2; pair += 1) {
12592
- const key = stringKeyAt(bytes, view, offset);
12593
- if (key === null) return null;
12594
- const valueEnd = endOfValue(view, key.end);
12595
- if (valueEnd === null) return null;
12596
- keys.add(key.key);
12597
- offset = valueEnd;
12598
- }
12599
- return keys;
12600
- }
12601
- function decodeMapWithoutFields(bytes, skippedFields) {
12602
- const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
12603
- const top = headerAt(view, 0);
12604
- if (top === null || top.children === 0 || top.bodyBytes !== 0) return null;
12605
- const tag = view.getUint8(0);
12606
- const isMap = tag >= 128 && tag <= 143 || MAP_TAGS.has(tag);
12607
- if (!isMap) return null;
12608
- const document = {};
12609
- let offset = top.headerBytes;
12610
- for (let pair = 0; pair < top.children / 2; pair += 1) {
12611
- const key = stringKeyAt(bytes, view, offset);
12612
- if (key === null || FORBIDDEN_KEYS.has(key.key)) return null;
12613
- const valueEnd = endOfValue(view, key.end);
12614
- if (valueEnd === null) return null;
12615
- if (!skippedFields.has(key.key)) {
12616
- try {
12617
- document[key.key] = decode(bytes.subarray(key.end, valueEnd));
12618
- } catch {
12619
- return null;
12620
- }
12621
- }
12622
- offset = valueEnd;
12623
- }
12624
- return offset === bytes.length ? document : null;
12625
- }
12626
13751
 
12627
13752
  // src/core/partition/frozen/merge-fields.ts
12628
13753
  function indexByFieldPath(entries) {
@@ -13014,16 +14139,9 @@ function mergeFieldLengths(inputs, documentCount) {
13014
14139
  }
13015
14140
  return { fieldLengthNames: names, fieldLengthColumns: columns, totalFieldLengths: totals };
13016
14141
  }
13017
- function mergeSurfaceForms2(inputs) {
14142
+ function mergeSurfaceForms2(inputs, analysis) {
13018
14143
  const registry = createSurfaceRegistry();
13019
- for (const input of inputs) {
13020
- const forms = input.segment.arrays.readSurfaceForms();
13021
- if (forms === null) continue;
13022
- for (const surface of Object.keys(forms)) {
13023
- const value = forms[surface];
13024
- if (Array.isArray(value)) registry.add(surface, value[1], value[0]);
13025
- }
13026
- }
14144
+ for (const input of inputs) input.segment.surfaces.addKeptForms(registry, input.remap, analysis);
13027
14145
  return registry.size() === 0 ? null : registry.serialize();
13028
14146
  }
13029
14147
  function holdsADiscardedField(bytes, discardedFields) {
@@ -13093,15 +14211,16 @@ function mergePartialWords(inputs, mergedTokenCount, analysis) {
13093
14211
  for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
13094
14212
  return { spellings: encodePartialSpellingsOf(analysis.fields, counts), reused: null };
13095
14213
  }
13096
- function mergeFrozenSegments(segments, analysis) {
14214
+ function mergeFrozenSegments(segments, analysis, textAnalysis) {
13097
14215
  if (typeof SharedArrayBuffer !== "function") return null;
13098
14216
  const merged = mergeSegmentContents(
13099
14217
  segments.map((segment) => ({ segment })),
13100
- analysis
14218
+ analysis,
14219
+ textAnalysis
13101
14220
  );
13102
14221
  return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis, merged.reusedPartialWords);
13103
14222
  }
13104
- function mergeSegmentContents(selections, analysis = null) {
14223
+ function mergeSegmentContents(selections, analysis = null, textAnalysis = null) {
13105
14224
  const docIds = [];
13106
14225
  const fieldNames = [];
13107
14226
  const inputs = selections.map((selection) => ({
@@ -13128,7 +14247,7 @@ function mergeSegmentContents(selections, analysis = null) {
13128
14247
  fieldLengthColumns: lengths.fieldLengthColumns,
13129
14248
  totalFieldLengths: lengths.totalFieldLengths,
13130
14249
  docFrequencies: postings.docFrequencies,
13131
- surfaceForms: mergeSurfaceForms2(inputs),
14250
+ surfaceForms: mergeSurfaceForms2(inputs, textAnalysis),
13132
14251
  partialSpellings: partialWords.spellings,
13133
14252
  ...mergeFieldIndexes2(inputs)
13134
14253
  };
@@ -13530,22 +14649,29 @@ function runSegmentAction(entry, action) {
13530
14649
  case "attachSegments": {
13531
14650
  for (const segment of action.segments) {
13532
14651
  growPartitionsTo(entry.manager, segment.partitionId);
13533
- const frozen = createSharedFrozenSegment(segment.snapshot);
13534
- for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId);
14652
+ const surfaceForms = segment.survivingSurfaceForms;
14653
+ const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
14654
+ const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
14655
+ for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
13535
14656
  entry.manager.attachFrozenSegment(segment.partitionId, frozen);
13536
14657
  }
13537
14658
  return void 0;
13538
14659
  }
13539
14660
  case "compactSegments": {
13540
14661
  const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
13541
- return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds), entry.manager.partialWordAnalysis);
14662
+ return mergeFrozenSegments(
14663
+ partition.frozenSegmentsById(action.segmentIds),
14664
+ entry.manager.partialWordAnalysis,
14665
+ entry.manager.textAnalysis
14666
+ );
13542
14667
  }
13543
14668
  case "swapSegments": {
13544
14669
  requireComposite(entry, action.indexName, action.partitionId, "swap");
13545
14670
  entry.manager.swapFrozenSegments(
13546
14671
  action.partitionId,
13547
14672
  action.dropSegmentIds,
13548
- createSharedFrozenSegment(action.snapshot)
14673
+ createSharedFrozenSegment(action.snapshot),
14674
+ action.removedSpellings
13549
14675
  );
13550
14676
  return void 0;
13551
14677
  }
@@ -13829,12 +14955,12 @@ function createDirectExecutor(options) {
13829
14955
  }
13830
14956
  case "remove": {
13831
14957
  const entry = requireIndex(action.indexName);
13832
- entry.manager.remove(action.docId);
14958
+ entry.manager.remove(action.docId, action.removedSpellings);
13833
14959
  return void 0;
13834
14960
  }
13835
14961
  case "update": {
13836
14962
  const entry = requireIndex(action.indexName);
13837
- entry.manager.update(action.docId, action.document);
14963
+ entry.manager.update(action.docId, action.document, void 0, action.removedSpellings);
13838
14964
  return void 0;
13839
14965
  }
13840
14966
  case "query": {
@@ -15286,7 +16412,15 @@ function checkHeapAfterRecovery(core) {
15286
16412
  // src/core/partition/frozen/share.ts
15287
16413
  function shareFrozenSegment(segment, analysis) {
15288
16414
  if (segment.sharedSnapshot !== null) {
15289
- return { snapshot: segment.sharedSnapshot, tombstonedDocIds: segment.tombstonedDocIds() };
16415
+ const tombstonedDocIds = segment.tombstonedDocIds();
16416
+ if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
16417
+ return { snapshot: segment.sharedSnapshot, tombstonedDocIds };
16418
+ }
16419
+ return {
16420
+ snapshot: segment.sharedSnapshot,
16421
+ tombstonedDocIds,
16422
+ survivingSurfaceForms: segment.surfaces.reader.serialize()
16423
+ };
15290
16424
  }
15291
16425
  const payload = encodeSegmentState(segment);
15292
16426
  const documents = payload.docIds.map((docId) => {
@@ -15304,6 +16438,35 @@ function shareFrozenSegment(segment, analysis) {
15304
16438
 
15305
16439
  // src/engine/worker-resync.ts
15306
16440
  var copyLoadSequence = 0;
16441
+ function emptySerializablePartition(manager, partitionId) {
16442
+ return {
16443
+ indexName: manager.indexName,
16444
+ partitionId,
16445
+ totalPartitions: manager.partitionCount,
16446
+ language: manager.language.name,
16447
+ schema: {},
16448
+ docCount: 0,
16449
+ avgDocLength: 0,
16450
+ documents: {},
16451
+ invertedIndex: {},
16452
+ fieldIndexes: { numeric: {}, boolean: {}, enum: {}, geopoint: {} },
16453
+ statistics: { totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} }
16454
+ };
16455
+ }
16456
+ function captureLiveTail(partition) {
16457
+ if (partition.count() === 0) return null;
16458
+ const payload = partition.encodeSegment();
16459
+ const documents = payload.docIds.map((docId) => {
16460
+ const document = partition.getRef(docId);
16461
+ if (document === void 0) {
16462
+ throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while copying a partition`, {
16463
+ docId
16464
+ });
16465
+ }
16466
+ return document;
16467
+ });
16468
+ return { payload, documents };
16469
+ }
15307
16470
  function captureComposite(manager, partitionId) {
15308
16471
  const partition = manager.getPartition(partitionId);
15309
16472
  if (!isCompositePartition(partition)) return null;
@@ -15314,13 +16477,7 @@ function captureComposite(manager, partitionId) {
15314
16477
  if (shared2 === null) return null;
15315
16478
  frozen.push(shared2);
15316
16479
  }
15317
- const live = partition.live.serialize(
15318
- manager.indexName,
15319
- manager.partitionCount,
15320
- manager.language.name,
15321
- manager.schema
15322
- );
15323
- return { partitionId, live, frozen };
16480
+ return { partitionId, live: captureLiveTail(partition.live), frozen };
15324
16481
  }
15325
16482
  function capturePartitions(manager) {
15326
16483
  const captured = [];
@@ -15328,30 +16485,45 @@ function capturePartitions(manager) {
15328
16485
  captured.push(
15329
16486
  captureComposite(manager, partitionId) ?? {
15330
16487
  partitionId,
15331
- live: manager.serializePartition(partitionId),
16488
+ live: captureLiveTail(manager.getPartition(partitionId)),
15332
16489
  frozen: []
15333
16490
  }
15334
16491
  );
15335
16492
  }
15336
16493
  return captured;
15337
16494
  }
15338
- async function sendPartition(indexName, executors, captured) {
16495
+ async function sendPartition(manager, executors, captured) {
16496
+ const { indexName } = manager;
15339
16497
  await Promise.all(
15340
16498
  executors.map(
15341
16499
  (workerExecutor) => workerExecutor.execute({
15342
16500
  type: "deserialize",
15343
16501
  indexName,
15344
16502
  partitionId: captured.partitionId,
15345
- data: captured.live,
16503
+ data: emptySerializablePartition(manager, captured.partitionId),
15346
16504
  requestId: `resync-${indexName}-${captured.partitionId}`
15347
16505
  })
15348
16506
  )
15349
16507
  );
16508
+ const live = captured.live;
16509
+ if (live !== null) {
16510
+ await Promise.all(
16511
+ executors.map(
16512
+ (workerExecutor) => workerExecutor.execute({
16513
+ type: "mergeSegments",
16514
+ indexName,
16515
+ segments: [{ partitionId: captured.partitionId, payload: live.payload, documents: live.documents }],
16516
+ requestId: `resync-live-${indexName}-${captured.partitionId}`
16517
+ })
16518
+ )
16519
+ );
16520
+ }
15350
16521
  if (captured.frozen.length === 0) return;
15351
16522
  const segments = captured.frozen.map((shared2) => ({
15352
16523
  partitionId: captured.partitionId,
15353
16524
  snapshot: shared2.snapshot,
15354
- tombstonedDocIds: shared2.tombstonedDocIds
16525
+ tombstonedDocIds: shared2.tombstonedDocIds,
16526
+ survivingSurfaceForms: shared2.survivingSurfaceForms
15355
16527
  }));
15356
16528
  await Promise.all(
15357
16529
  executors.map(
@@ -15397,7 +16569,7 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
15397
16569
  )
15398
16570
  );
15399
16571
  for (const partition of captured) {
15400
- await sendPartition(indexName, executors, partition);
16572
+ await sendPartition(manager, executors, partition);
15401
16573
  }
15402
16574
  return copyLoad;
15403
16575
  }
@@ -16015,21 +17187,24 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
16015
17187
  const mainCanMerge = composite !== null && mainHoldsAll(composite, segmentIds);
16016
17188
  let snapshot = await compactOnWorker(state, indexName, partitionId, segmentIds);
16017
17189
  if (snapshot === null && mainCanMerge && composite !== null) {
16018
- snapshot = mergeFrozenSegments(composite.frozenSegmentsById(segmentIds), manager.partialWordAnalysis);
17190
+ snapshot = mergeFrozenSegments(
17191
+ composite.frozenSegmentsById(segmentIds),
17192
+ manager.partialWordAnalysis,
17193
+ manager.textAnalysis
17194
+ );
16019
17195
  }
16020
17196
  if (snapshot === null) {
16021
17197
  if (!mainCanMerge) forgetLedgerSegments(state, indexName, partitionId, segmentIds);
16022
17198
  return false;
16023
17199
  }
16024
- if (mainCanMerge && composite !== null) {
16025
- composite.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot));
16026
- }
17200
+ const removedSpellings = mainCanMerge && composite !== null ? composite.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot)) : void 0;
16027
17201
  await replicateToWorkers(state, {
16028
17202
  type: "swapSegments",
16029
17203
  indexName,
16030
17204
  partitionId,
16031
17205
  dropSegmentIds: [...segmentIds],
16032
17206
  snapshot,
17207
+ removedSpellings,
16033
17208
  requestId: createRequestId()
16034
17209
  });
16035
17210
  return true;
@@ -17533,7 +18708,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
17533
18708
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
17534
18709
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
17535
18710
  }
17536
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-7RUN7LNS.mjs');
18711
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-VSUBLUDK.mjs');
17537
18712
  const filesystemDurability = durabilityTier?.kind === "wal";
17538
18713
  const executor = createDirectExecutor({
17539
18714
  vectorWorkerCopies: vectorCopyPolicy,
@@ -19597,16 +20772,18 @@ async function prepareRemoval(ctx, indexName, docId) {
19597
20772
  if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
19598
20773
  throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
19599
20774
  }
19600
- const restoreRef = ctx.durability && !rebalancing ? ctx.requireManager(indexName).getRef(docId) : void 0;
20775
+ const manager = ctx.requireManager(indexName);
20776
+ const restoreRef = ctx.durability && !rebalancing ? manager.getRef(docId) : void 0;
19601
20777
  return {
19602
20778
  docId,
19603
20779
  document: null,
19604
20780
  restoreDoc: restoreRef ? structuredClone(restoreRef) : void 0,
20781
+ removedSpellings: manager.removedSpellingsOf(docId),
19605
20782
  buffered: false
19606
20783
  };
19607
20784
  }
19608
20785
  function applyOfRemoval(ctx, indexName, prepared) {
19609
- const { docId } = prepared;
20786
+ const { docId, removedSpellings } = prepared;
19610
20787
  return async () => {
19611
20788
  if (ctx.isRebalancing(indexName)) {
19612
20789
  const bufferedState = ctx.bufferedDocState(indexName, docId);
@@ -19619,11 +20796,12 @@ function applyOfRemoval(ctx, indexName, prepared) {
19619
20796
  prepared.buffered = true;
19620
20797
  return;
19621
20798
  }
19622
- await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId });
20799
+ await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId, removedSpellings });
19623
20800
  removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
19624
20801
  };
19625
20802
  }
19626
- async function settleRemoval(ctx, indexName, docId) {
20803
+ async function settleRemoval(ctx, indexName, prepared) {
20804
+ const { docId, removedSpellings } = prepared;
19627
20805
  try {
19628
20806
  await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
19629
20807
  } catch (err) {
@@ -19633,7 +20811,8 @@ async function settleRemoval(ctx, indexName, docId) {
19633
20811
  type: "remove",
19634
20812
  indexName,
19635
20813
  docId,
19636
- requestId: `replicate-remove-${docId}`
20814
+ requestId: `replicate-remove-${docId}`,
20815
+ removedSpellings
19637
20816
  });
19638
20817
  }
19639
20818
  async function removeDocument(ctx, indexName, docId, options) {
@@ -19651,7 +20830,7 @@ async function removeDocument(ctx, indexName, docId, options) {
19651
20830
  } else {
19652
20831
  await apply();
19653
20832
  }
19654
- if (!prepared.buffered) await settleRemoval(ctx, indexName, docId);
20833
+ if (!prepared.buffered) await settleRemoval(ctx, indexName, prepared);
19655
20834
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
19656
20835
  }
19657
20836
  async function removeDocumentBatch(ctx, indexName, docIds, options) {
@@ -19687,7 +20866,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
19687
20866
  failed.push({ docId: removal.docId, error: asRemoveError(error) });
19688
20867
  continue;
19689
20868
  }
19690
- if (!removal.buffered) await settleRemoval(ctx, indexName, removal.docId);
20869
+ if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
19691
20870
  succeeded.push(removal.docId);
19692
20871
  }
19693
20872
  if (chunkEnd < docIds.length) {
@@ -19735,7 +20914,8 @@ async function settleUpdate(ctx, indexName, prepared) {
19735
20914
  indexName,
19736
20915
  docId: prepared.docId,
19737
20916
  document: prepared.partitionDoc,
19738
- requestId: `replicate-update-${prepared.docId}`
20917
+ requestId: `replicate-update-${prepared.docId}`,
20918
+ removedSpellings: prepared.removedSpellings
19739
20919
  });
19740
20920
  const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
19741
20921
  for (const [fieldPath, vec] of prepared.extractedVectors) {
@@ -19802,12 +20982,13 @@ async function prepareUpdate(ctx, indexName, entry, docId, document) {
19802
20982
  oldDocument,
19803
20983
  rollbackDoc,
19804
20984
  extractedVectors,
20985
+ removedSpellings: manager.removedSpellingsOf(docId),
19805
20986
  buffered: false
19806
20987
  };
19807
20988
  }
19808
20989
  function applyOfUpdate(ctx, indexName, prepared) {
19809
20990
  const manager = ctx.requireManager(indexName);
19810
- const { docId, document, partitionDoc, rollbackDoc, extractedVectors } = prepared;
20991
+ const { docId, document, partitionDoc, rollbackDoc, extractedVectors, removedSpellings } = prepared;
19811
20992
  return async () => {
19812
20993
  if (ctx.isRebalancing(indexName)) {
19813
20994
  const bufferedState = ctx.bufferedDocState(indexName, docId);
@@ -19820,7 +21001,14 @@ function applyOfUpdate(ctx, indexName, prepared) {
19820
21001
  prepared.buffered = true;
19821
21002
  return;
19822
21003
  }
19823
- await ctx.executor.execute({ type: "update", indexName, docId, document: partitionDoc, requestId: docId });
21004
+ await ctx.executor.execute({
21005
+ type: "update",
21006
+ indexName,
21007
+ docId,
21008
+ document: partitionDoc,
21009
+ requestId: docId,
21010
+ removedSpellings
21011
+ });
19824
21012
  try {
19825
21013
  updateDocumentVectors(docId, extractedVectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
19826
21014
  } catch (err) {
@@ -20005,9 +21193,10 @@ async function replayEntry(manager, indexName, entry, ctx) {
20005
21193
  vectorFieldPaths2
20006
21194
  );
20007
21195
  const overwrote = manager.has(entry.docId);
21196
+ const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
20008
21197
  if (overwrote) {
20009
21198
  removeDocumentVectors(entry.docId, vecIndexes);
20010
- manager.remove(entry.docId);
21199
+ manager.remove(entry.docId, removedSpellings);
20011
21200
  }
20012
21201
  manager.insert(entry.docId, partitionDoc);
20013
21202
  if (extractedVectors.size > 0) {
@@ -20021,17 +21210,20 @@ async function replayEntry(manager, indexName, entry, ctx) {
20021
21210
  } catch (err) {
20022
21211
  warnHookError("afterInsert", err);
20023
21212
  }
20024
- await ctx.orchestrator.replicateToWorkers({
20025
- type: overwrote ? "update" : "insert",
21213
+ const replicated = {
20026
21214
  indexName,
20027
21215
  docId: entry.docId,
20028
21216
  document: entry.document,
20029
21217
  requestId: `replicate-insert-${entry.docId}`
20030
- });
21218
+ };
21219
+ await ctx.orchestrator.replicateToWorkers(
21220
+ overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
21221
+ );
20031
21222
  return;
20032
21223
  }
20033
21224
  if (entry.action === "remove") {
20034
- manager.remove(entry.docId);
21225
+ const removedSpellings = manager.removedSpellingsOf(entry.docId);
21226
+ manager.remove(entry.docId, removedSpellings);
20035
21227
  removeDocumentVectors(entry.docId, vecIndexes);
20036
21228
  try {
20037
21229
  await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
@@ -20042,15 +21234,17 @@ async function replayEntry(manager, indexName, entry, ctx) {
20042
21234
  type: "remove",
20043
21235
  indexName,
20044
21236
  docId: entry.docId,
20045
- requestId: `replicate-remove-${entry.docId}`
21237
+ requestId: `replicate-remove-${entry.docId}`,
21238
+ removedSpellings
20046
21239
  });
20047
21240
  return;
20048
21241
  }
20049
21242
  if (entry.action === "update" && entry.document) {
20050
21243
  const oldDocument = manager.get(entry.docId);
20051
21244
  const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths2);
21245
+ const removedSpellings = manager.removedSpellingsOf(entry.docId);
20052
21246
  if (manager.has(entry.docId)) {
20053
- manager.update(entry.docId, updateDoc);
21247
+ manager.update(entry.docId, updateDoc, void 0, removedSpellings);
20054
21248
  } else {
20055
21249
  manager.insert(entry.docId, updateDoc);
20056
21250
  }
@@ -20087,7 +21281,8 @@ async function replayEntry(manager, indexName, entry, ctx) {
20087
21281
  indexName,
20088
21282
  docId: entry.docId,
20089
21283
  document: entry.document,
20090
- requestId: `replicate-update-${entry.docId}`
21284
+ requestId: `replicate-update-${entry.docId}`,
21285
+ removedSpellings
20091
21286
  });
20092
21287
  }
20093
21288
  }
@@ -20908,7 +22103,7 @@ function portabilityOperations(core, config, engine) {
20908
22103
  };
20909
22104
  }
20910
22105
  async function restoreIndex(indexName, data) {
20911
- const { restoreFromSnapshot } = await import('./snapshot-A42DRERF.mjs');
22106
+ const { restoreFromSnapshot } = await import('./snapshot-5MXBEDXF.mjs');
20912
22107
  await restoreFromSnapshot(indexName, data, {
20913
22108
  executor: core.executor,
20914
22109
  indexRegistry: core.indexRegistry,
@@ -20928,7 +22123,7 @@ function portabilityOperations(core, config, engine) {
20928
22123
  return {
20929
22124
  snapshot(indexName) {
20930
22125
  return withOpenIndex(indexName, async () => {
20931
- const { createSnapshot } = await import('./snapshot-A42DRERF.mjs');
22126
+ const { createSnapshot } = await import('./snapshot-5MXBEDXF.mjs');
20932
22127
  return createSnapshot(snapshotDepsOf(indexName));
20933
22128
  });
20934
22129
  },
@@ -20940,8 +22135,8 @@ function portabilityOperations(core, config, engine) {
20940
22135
  exportDocuments(indexName) {
20941
22136
  return withOpenIndex(indexName, async () => {
20942
22137
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
20943
- import('./snapshot-A42DRERF.mjs'),
20944
- import('./write-4U6Q26TJ.mjs')
22138
+ import('./snapshot-5MXBEDXF.mjs'),
22139
+ import('./write-6SBYTS62.mjs')
20945
22140
  ]);
20946
22141
  const header = indexConfigurationOf(core.requireIndex(indexName));
20947
22142
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -20965,7 +22160,7 @@ var LIST_TAG = 6;
20965
22160
  var MAP_TAG = 7;
20966
22161
  var LENGTH_BYTES = 4;
20967
22162
  var LAST_ASCII_UNIT = 127;
20968
- var encoder4 = new TextEncoder();
22163
+ var encoder5 = new TextEncoder();
20969
22164
  function ensureCapacity(stream, extra) {
20970
22165
  if (stream.length + extra <= stream.bytes.length) return;
20971
22166
  const grown = new Uint8Array(Math.max(stream.bytes.length * 2, stream.length + extra));
@@ -20995,7 +22190,7 @@ function writeNumber(stream, value) {
20995
22190
  stream.length += 8;
20996
22191
  }
20997
22192
  function writeEncodedString(stream, value) {
20998
- const encoded = encoder4.encode(value);
22193
+ const encoded = encoder5.encode(value);
20999
22194
  writeUint32(stream, encoded.length);
21000
22195
  ensureCapacity(stream, encoded.length);
21001
22196
  stream.bytes.set(encoded, stream.length);
@@ -21163,18 +22358,27 @@ function requireFacetableFields(facets, flatSchema, strict) {
21163
22358
  }
21164
22359
  }
21165
22360
  }
21166
- function requireDeclaredGroupFields(fields, flatSchema) {
22361
+ function requireGroupableFields(fields, flatSchema, strict) {
21167
22362
  for (const field of fields) {
21168
- if (flatSchema[field] === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
22363
+ const fieldType = flatSchema[field];
22364
+ if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
22365
+ if (fieldType !== void 0 && VECTOR_PATTERN.test(fieldType)) {
22366
+ throw new NarsilError(
22367
+ ErrorCodes.SEARCH_INVALID_FIELD,
22368
+ `The engine groups hits by a text, number, boolean, enum, or geopoint field, and the schema declares field "${field}" as "${fieldType}"`,
22369
+ { field, fieldType }
22370
+ );
22371
+ }
21169
22372
  }
21170
22373
  }
21171
22374
  function requireUsableQueryFields(params, schema, strict = false) {
21172
- const checksGroup = strict && Array.isArray(params.group?.fields);
22375
+ const groupFields = params.group?.fields;
22376
+ const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
21173
22377
  if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
21174
22378
  const flatSchema = flattenSchema(schema);
21175
22379
  if (params.boost !== void 0) requireBoostableFields(params.boost, flatSchema);
21176
22380
  if (params.facets !== void 0) requireFacetableFields(params.facets, flatSchema, strict);
21177
- if (checksGroup && params.group !== void 0) requireDeclaredGroupFields(params.group.fields, flatSchema);
22381
+ if (checksGroup) requireGroupableFields(groupFields, flatSchema, strict);
21178
22382
  }
21179
22383
 
21180
22384
  // src/search/typo-tolerance.ts
@@ -21420,6 +22624,7 @@ function executeListDocuments(params, context) {
21420
22624
  const signature = sortSignatureOf(params.sort);
21421
22625
  requireSortableFields(params.sort, schema, context.strict === true);
21422
22626
  const filters = validQueryFilter(params.filters, schema);
22627
+ const projection = resolveProjection(params.document);
21423
22628
  const binding = listBindingOf(params.filters);
21424
22629
  let cursor = null;
21425
22630
  if (params.cursor !== void 0) {
@@ -21430,7 +22635,6 @@ function executeListDocuments(params, context) {
21430
22635
  const filtered = filters === void 0 ? null : collectFilterMatches(partitions, filters, schema, context.patternWorkCap);
21431
22636
  const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
21432
22637
  const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
21433
- const projection = resolveProjection(params.document);
21434
22638
  const documents = [];
21435
22639
  for (const docId of page.ids) {
21436
22640
  if (projection.kind === "none") {
@@ -21453,37 +22657,23 @@ function executeListDocuments(params, context) {
21453
22657
  }
21454
22658
 
21455
22659
  // src/search/grouping.ts
21456
- function getNestedValue2(obj, path) {
21457
- const segments = path.split(".");
21458
- let current = obj;
21459
- for (const segment of segments) {
21460
- if (current === null || current === void 0 || typeof current !== "object") {
21461
- return void 0;
21462
- }
21463
- if (!Object.hasOwn(current, segment)) {
21464
- return void 0;
21465
- }
21466
- current = current[segment];
21467
- }
21468
- return current;
21469
- }
21470
22660
  function hitsKeptPerGroup(maxPerGroup) {
21471
22661
  if (maxPerGroup === void 0 || !Number.isFinite(maxPerGroup) || maxPerGroup < 1) return DEFAULT_MAX_PER_GROUP;
21472
22662
  return Math.floor(maxPerGroup);
21473
22663
  }
21474
- function applyGrouping(hits, group, getDocument) {
22664
+ function applyGrouping(hits, group, documents) {
21475
22665
  if (!group.fields || group.fields.length === 0) {
21476
22666
  return [{ values: {}, hits }];
21477
22667
  }
21478
22668
  const groupMap = /* @__PURE__ */ new Map();
21479
22669
  for (const hit of hits) {
21480
- const doc = getDocument(hit.id);
22670
+ const stored = documents.fieldValues(hit.id, group.fields);
21481
22671
  const fieldValues = [];
21482
22672
  const valuesRecord = {};
21483
- for (const field of group.fields) {
21484
- const value = doc ? getNestedValue2(doc, field) : void 0;
22673
+ for (let at = 0; at < group.fields.length; at++) {
22674
+ const value = stored?.[at];
21485
22675
  fieldValues.push(value);
21486
- valuesRecord[field] = value;
22676
+ valuesRecord[group.fields[at]] = value;
21487
22677
  }
21488
22678
  const compositeKey = JSON.stringify(fieldValues);
21489
22679
  const existing = groupMap.get(compositeKey);
@@ -21501,7 +22691,7 @@ function applyGrouping(hits, group, getDocument) {
21501
22691
  if (group.reduce) {
21502
22692
  const folded = foldGroupReducer(
21503
22693
  group.reduce,
21504
- everyHit.map((hit) => ({ document: getDocument(hit.id), score: hit.score ?? 0 }))
22694
+ everyHit.map((hit) => ({ document: documents.document(hit.id), score: hit.score ?? 0 }))
21505
22695
  );
21506
22696
  if (folded.reducerError !== void 0) {
21507
22697
  result.reducerError = folded.reducerError;
@@ -22149,6 +23339,7 @@ async function executeQuery(request, callerContext) {
22149
23339
  }
22150
23340
  const params = validQueryParams(request, config);
22151
23341
  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
23342
+ const projection = resolveProjection(params.document);
22152
23343
  let paginated;
22153
23344
  let nextCursor;
22154
23345
  let count;
@@ -22232,7 +23423,10 @@ async function executeQuery(request, callerContext) {
22232
23423
  hits = applySorting(hits, params.sort, sortKeyOf);
22233
23424
  }
22234
23425
  if (params.group) {
22235
- groups = applyGrouping(hits, params.group, (docId) => manager.getRef(docId));
23426
+ groups = applyGrouping(hits, params.group, {
23427
+ fieldValues: (docId, fieldPaths) => manager.fieldValues(docId, fieldPaths),
23428
+ document: (docId) => manager.getRef(docId)
23429
+ });
22236
23430
  }
22237
23431
  let pinsFromOutside = 0;
22238
23432
  if (params.pinned && params.searchAfter === void 0 && context.partitionIds === void 0) {
@@ -22272,7 +23466,6 @@ async function executeQuery(request, callerContext) {
22272
23466
  for (const hit of hits) hit.score = void 0;
22273
23467
  }
22274
23468
  }
22275
- const projection = resolveProjection(params.document);
22276
23469
  if (projection.kind === "none") {
22277
23470
  for (const hit of paginated) {
22278
23471
  hit.document = {};