@delali/narsil 0.3.1-next.87 → 0.3.1-next.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
- package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
- package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
- package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
- package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
- package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
- package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
- package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
- package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
- package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
- package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
- package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
- package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
- package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
- package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
- package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
- package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
- package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
- package/dist/distribution.mjs +14 -27
- package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +5 -5
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
- package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
- package/dist/workers/entry.mjs +3 -3
- package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
- package/package.json +13 -8
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, toReducedSortValue,
|
|
2
|
-
import { encode, decode } from '@msgpack/msgpack';
|
|
1
|
+
import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, toReducedSortValue, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, resolveRequestThreadCount, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, defaultSortMode, holdsUnpairedSurrogate, MIN_SWEEP_INTERVAL_MS, MAX_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_INDEX_CONFIGURATION_BYTES, compareComparableValues, readSortField, exceedsCodePoints, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, compareComparableKeys, sortModeOf, toComparableSortValue, DEFAULT_FUZZY_PREFIX_LENGTH, SORT_VALUE_MAX_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, truncateSortString, MAX_IMPORT_REFUSALS_REPORTED, MIN_OVERSAMPLE, DEFAULT_MAX_PER_GROUP, compareSortValues, isSortMode, SORT_MODES, MIN_DOCUMENTS_FOR_SEGMENTS, SHUTDOWN_TIMEOUT_MS, DEFAULT_PAGE_SIZE, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-FHFFWLEK.mjs';
|
|
2
|
+
import { Decoder, encode, decode } from '@msgpack/msgpack';
|
|
3
3
|
|
|
4
4
|
// src/analysis/registry.ts
|
|
5
5
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -896,6 +896,17 @@ function visitTokens(text, language, options, visit) {
|
|
|
896
896
|
visit(processed, position, surface === processed ? void 0 : surface, part);
|
|
897
897
|
}
|
|
898
898
|
}
|
|
899
|
+
function visitSurfaceSpellings(text, language, visit) {
|
|
900
|
+
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
901
|
+
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
902
|
+
const removeDiacritics = language.tokenizer?.normalizeDiacritics ?? false;
|
|
903
|
+
const stripPossessives = language.tokenizer?.stripPossessive ?? false;
|
|
904
|
+
for (const part of rawParts) {
|
|
905
|
+
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
906
|
+
if (candidate.length < minLength) continue;
|
|
907
|
+
visit(removeDiacritics ? transformToken(candidate, language, false, true) : candidate);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
899
910
|
function tokenize(text, language, options) {
|
|
900
911
|
if (options?.customTokenizer) {
|
|
901
912
|
const customResult = options.customTokenizer.tokenize(text);
|
|
@@ -2713,6 +2724,197 @@ function validateVectorPromotion(config) {
|
|
|
2713
2724
|
}
|
|
2714
2725
|
}
|
|
2715
2726
|
|
|
2727
|
+
// src/core/partition/utils.ts
|
|
2728
|
+
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
2729
|
+
const existing = table.indexMap.get(fieldName);
|
|
2730
|
+
if (existing !== void 0) return existing;
|
|
2731
|
+
const idx = table.names.length;
|
|
2732
|
+
table.names.push(fieldName);
|
|
2733
|
+
table.indexMap.set(fieldName, idx);
|
|
2734
|
+
return idx;
|
|
2735
|
+
}
|
|
2736
|
+
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
2737
|
+
function getNestedValue(obj, path) {
|
|
2738
|
+
const parts = path.split(".");
|
|
2739
|
+
let current = obj;
|
|
2740
|
+
for (let i = 0; i < parts.length; i++) {
|
|
2741
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
2742
|
+
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
2743
|
+
current = current[parts[i]];
|
|
2744
|
+
}
|
|
2745
|
+
return current;
|
|
2746
|
+
}
|
|
2747
|
+
function stringArraysEqual(a, b) {
|
|
2748
|
+
if (a === b) return true;
|
|
2749
|
+
if (!a || !b) return false;
|
|
2750
|
+
if (a.length !== b.length) return false;
|
|
2751
|
+
for (let i = 0; i < a.length; i++) {
|
|
2752
|
+
if (a[i] !== b[i]) return false;
|
|
2753
|
+
}
|
|
2754
|
+
return true;
|
|
2755
|
+
}
|
|
2756
|
+
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
2757
|
+
for (const { path, base } of fields) {
|
|
2758
|
+
if (!isWordIndexedBase(base)) continue;
|
|
2759
|
+
const oldVal = getNestedValue(oldDoc, path);
|
|
2760
|
+
const newVal = getNestedValue(newDoc, path);
|
|
2761
|
+
if (base === "string") {
|
|
2762
|
+
if (oldVal !== newVal) return true;
|
|
2763
|
+
} else {
|
|
2764
|
+
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
2765
|
+
}
|
|
2766
|
+
}
|
|
2767
|
+
return false;
|
|
2768
|
+
}
|
|
2769
|
+
function parseVectorDimension(fieldType) {
|
|
2770
|
+
const match = VECTOR_PATTERN2.exec(fieldType);
|
|
2771
|
+
return match ? Number.parseInt(match[1], 10) : null;
|
|
2772
|
+
}
|
|
2773
|
+
function tokenizeOptions(options) {
|
|
2774
|
+
return {
|
|
2775
|
+
stem: true,
|
|
2776
|
+
removeStopWords: true,
|
|
2777
|
+
collectSurfaces: options?.collectSurfaces === true,
|
|
2778
|
+
stopWordOverride: options?.stopWordOverride,
|
|
2779
|
+
customTokenizer: options?.customTokenizer
|
|
2780
|
+
};
|
|
2781
|
+
}
|
|
2782
|
+
function getFlatSchema(state, schema) {
|
|
2783
|
+
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
2784
|
+
state.flatSchemaCache = flattenSchema(schema);
|
|
2785
|
+
state.lastSchemaRef = schema;
|
|
2786
|
+
return state.flatSchemaCache;
|
|
2787
|
+
}
|
|
2788
|
+
|
|
2789
|
+
// src/core/projection.ts
|
|
2790
|
+
function requireFieldPathList(paths, option) {
|
|
2791
|
+
if (paths === void 0) return;
|
|
2792
|
+
if (!Array.isArray(paths)) {
|
|
2793
|
+
throw new NarsilError(
|
|
2794
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
2795
|
+
`The "document.${option}" option takes an array of field paths`,
|
|
2796
|
+
{ option: `document.${option}`, received: paths === null ? "null" : typeof paths }
|
|
2797
|
+
);
|
|
2798
|
+
}
|
|
2799
|
+
for (let position = 0; position < paths.length; position++) {
|
|
2800
|
+
const entry = paths[position];
|
|
2801
|
+
if (typeof entry === "string") continue;
|
|
2802
|
+
throw new NarsilError(
|
|
2803
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
2804
|
+
`The "document.${option}" option takes field paths as strings, and entry ${position} is not a string`,
|
|
2805
|
+
{ option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
|
|
2806
|
+
);
|
|
2807
|
+
}
|
|
2808
|
+
}
|
|
2809
|
+
function splitPaths(paths) {
|
|
2810
|
+
if (!paths) return [];
|
|
2811
|
+
const split = [];
|
|
2812
|
+
for (const path of paths) {
|
|
2813
|
+
const segments = path.split(".").filter((segment) => segment.length > 0);
|
|
2814
|
+
if (segments.length === 0) continue;
|
|
2815
|
+
if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) continue;
|
|
2816
|
+
split.push(segments);
|
|
2817
|
+
}
|
|
2818
|
+
return split;
|
|
2819
|
+
}
|
|
2820
|
+
function overlaps(a, b) {
|
|
2821
|
+
const shared2 = Math.min(a.length, b.length);
|
|
2822
|
+
for (let i = 0; i < shared2; i++) {
|
|
2823
|
+
if (a[i] !== b[i]) return false;
|
|
2824
|
+
}
|
|
2825
|
+
return true;
|
|
2826
|
+
}
|
|
2827
|
+
function covers(outer, inner) {
|
|
2828
|
+
if (outer.length > inner.length) return false;
|
|
2829
|
+
for (let i = 0; i < outer.length; i++) {
|
|
2830
|
+
if (outer[i] !== inner[i]) return false;
|
|
2831
|
+
}
|
|
2832
|
+
return true;
|
|
2833
|
+
}
|
|
2834
|
+
function readProjectedPath(source, path) {
|
|
2835
|
+
let current = source;
|
|
2836
|
+
for (const segment of path) {
|
|
2837
|
+
if (typeof current !== "object" || current === null || Array.isArray(current))
|
|
2838
|
+
return { found: false, value: void 0 };
|
|
2839
|
+
const record = current;
|
|
2840
|
+
if (!Object.hasOwn(record, segment)) return { found: false, value: void 0 };
|
|
2841
|
+
current = record[segment];
|
|
2842
|
+
}
|
|
2843
|
+
return { found: true, value: current };
|
|
2844
|
+
}
|
|
2845
|
+
function writePath(target, path, value) {
|
|
2846
|
+
let current = target;
|
|
2847
|
+
for (let i = 0; i < path.length - 1; i++) {
|
|
2848
|
+
const segment = path[i];
|
|
2849
|
+
const existing = current[segment];
|
|
2850
|
+
if (typeof existing === "object" && existing !== null && !Array.isArray(existing)) {
|
|
2851
|
+
current = existing;
|
|
2852
|
+
continue;
|
|
2853
|
+
}
|
|
2854
|
+
const created = {};
|
|
2855
|
+
current[segment] = created;
|
|
2856
|
+
current = created;
|
|
2857
|
+
}
|
|
2858
|
+
current[path[path.length - 1]] = value;
|
|
2859
|
+
}
|
|
2860
|
+
function cloneExcept(value, path, exclude) {
|
|
2861
|
+
const reachesDeeper = exclude.some((excluded) => excluded.length > path.length && covers(path, excluded));
|
|
2862
|
+
if (!reachesDeeper) return structuredClone(value);
|
|
2863
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return structuredClone(value);
|
|
2864
|
+
const source = value;
|
|
2865
|
+
const kept = {};
|
|
2866
|
+
for (const key of Object.keys(source)) {
|
|
2867
|
+
const childPath = [...path, key];
|
|
2868
|
+
if (exclude.some((excluded) => covers(excluded, childPath))) continue;
|
|
2869
|
+
Object.defineProperty(kept, key, {
|
|
2870
|
+
value: cloneExcept(source[key], childPath, exclude),
|
|
2871
|
+
enumerable: true,
|
|
2872
|
+
writable: true,
|
|
2873
|
+
configurable: true
|
|
2874
|
+
});
|
|
2875
|
+
}
|
|
2876
|
+
return kept;
|
|
2877
|
+
}
|
|
2878
|
+
function resolveProjection(projection) {
|
|
2879
|
+
if (projection === void 0 || projection === true) return { kind: "full" };
|
|
2880
|
+
if (projection === false) return { kind: "none" };
|
|
2881
|
+
if (typeof projection !== "object" || projection === null) return { kind: "full" };
|
|
2882
|
+
requireFieldPathList(projection.include, "include");
|
|
2883
|
+
requireFieldPathList(projection.exclude, "exclude");
|
|
2884
|
+
const askedInclude = Array.isArray(projection.include) && projection.include.length > 0;
|
|
2885
|
+
const askedExclude = Array.isArray(projection.exclude) && projection.exclude.length > 0;
|
|
2886
|
+
if (!askedInclude && !askedExclude) return { kind: "full" };
|
|
2887
|
+
return {
|
|
2888
|
+
kind: "fields",
|
|
2889
|
+
include: askedInclude ? splitPaths(projection.include) : null,
|
|
2890
|
+
exclude: splitPaths(projection.exclude)
|
|
2891
|
+
};
|
|
2892
|
+
}
|
|
2893
|
+
function projectionKeepsField(projection, fieldPath) {
|
|
2894
|
+
if (projection.kind === "full") return true;
|
|
2895
|
+
if (projection.kind === "none") return false;
|
|
2896
|
+
const path = fieldPath.split(".").filter((segment) => segment.length > 0);
|
|
2897
|
+
if (path.length === 0) return false;
|
|
2898
|
+
if (projection.include && !projection.include.some((included) => overlaps(included, path))) return false;
|
|
2899
|
+
return !projection.exclude.some((excluded) => covers(excluded, path));
|
|
2900
|
+
}
|
|
2901
|
+
function cloneProjected(document, projection) {
|
|
2902
|
+
if (projection === void 0 || projection.kind === "full") return structuredClone(document);
|
|
2903
|
+
if (projection.kind === "none") return {};
|
|
2904
|
+
if (projection.include === null) return cloneExcept(document, [], projection.exclude);
|
|
2905
|
+
return projectIncludedPaths(projection.include, projection.exclude, (path) => readProjectedPath(document, path));
|
|
2906
|
+
}
|
|
2907
|
+
function projectIncludedPaths(include, exclude, read) {
|
|
2908
|
+
const projected = {};
|
|
2909
|
+
for (const path of include) {
|
|
2910
|
+
if (exclude.some((excluded) => covers(excluded, path))) continue;
|
|
2911
|
+
const { found, value } = read(path);
|
|
2912
|
+
if (!found) continue;
|
|
2913
|
+
writePath(projected, path, cloneExcept(value, path, exclude));
|
|
2914
|
+
}
|
|
2915
|
+
return projected;
|
|
2916
|
+
}
|
|
2917
|
+
|
|
2716
2918
|
// src/core/document-store.ts
|
|
2717
2919
|
function detachedView(view) {
|
|
2718
2920
|
if (view.byteOffset === 0 && view.byteLength === view.buffer.byteLength) return view;
|
|
@@ -2737,7 +2939,7 @@ function sharesABiggerBuffer(value) {
|
|
|
2737
2939
|
return false;
|
|
2738
2940
|
}
|
|
2739
2941
|
function withOwnBuffers(value) {
|
|
2740
|
-
if (value === null || typeof value !== "object") return value;
|
|
2942
|
+
if (value === null || typeof value !== "object" || value instanceof Date) return value;
|
|
2741
2943
|
if (ArrayBuffer.isView(value)) return detachedView(value);
|
|
2742
2944
|
if (Array.isArray(value)) {
|
|
2743
2945
|
const copy2 = new Array(value.length);
|
|
@@ -2762,6 +2964,9 @@ function documentWithOwnBuffers(document) {
|
|
|
2762
2964
|
if (!sharesABiggerBuffer(document)) return document;
|
|
2763
2965
|
return withOwnBuffers(document);
|
|
2764
2966
|
}
|
|
2967
|
+
function valueWithOwnBuffers(value) {
|
|
2968
|
+
return sharesABiggerBuffer(value) ? withOwnBuffers(value) : value;
|
|
2969
|
+
}
|
|
2765
2970
|
var STRING_HEADER_BYTES = 16;
|
|
2766
2971
|
var TWO_BYTE_SAMPLE_LIMIT = 16;
|
|
2767
2972
|
var NUMBER_BYTES = 16;
|
|
@@ -2959,12 +3164,22 @@ function createDocumentStore() {
|
|
|
2959
3164
|
},
|
|
2960
3165
|
resolver() {
|
|
2961
3166
|
return idResolver;
|
|
3167
|
+
},
|
|
3168
|
+
fieldReader(fieldPath) {
|
|
3169
|
+
return (internalId) => {
|
|
3170
|
+
const docId = reverseMap[internalId];
|
|
3171
|
+
const stored = docId === void 0 ? void 0 : docs.get(docId);
|
|
3172
|
+
return stored === void 0 ? void 0 : getNestedValue(stored.fields, fieldPath);
|
|
3173
|
+
};
|
|
3174
|
+
},
|
|
3175
|
+
projectedDocument(docId, projection) {
|
|
3176
|
+
const stored = docs.get(docId);
|
|
3177
|
+
return stored === void 0 ? void 0 : cloneProjected(stored.fields, projection);
|
|
2962
3178
|
}
|
|
2963
3179
|
};
|
|
2964
3180
|
}
|
|
2965
3181
|
|
|
2966
3182
|
// src/core/constants.ts
|
|
2967
|
-
var POSTING_LIST_INITIAL_CAPACITY = 8;
|
|
2968
3183
|
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
2969
3184
|
var MAX_TERM_FREQUENCY = 65535;
|
|
2970
3185
|
|
|
@@ -3035,433 +3250,912 @@ function bandedDistance(a, b, tolerance) {
|
|
|
3035
3250
|
return { distance, withinTolerance: distance <= tolerance, cells };
|
|
3036
3251
|
}
|
|
3037
3252
|
|
|
3038
|
-
// src/core/
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
const
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
let
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
list.termFrequencies[writeIdx] = list.termFrequencies[i];
|
|
3073
|
-
list.fieldNameIndices[writeIdx] = list.fieldNameIndices[i];
|
|
3074
|
-
if (list.positions) {
|
|
3075
|
-
list.positions[writeIdx] = list.positions[i];
|
|
3076
|
-
}
|
|
3077
|
-
}
|
|
3078
|
-
writeIdx++;
|
|
3079
|
-
}
|
|
3253
|
+
// src/core/inverted-index/constants.ts
|
|
3254
|
+
var TERM_TABLE_START_CAPACITY = 1 << 10;
|
|
3255
|
+
var POSTING_POOL_START_ENTRIES = 1 << 12;
|
|
3256
|
+
var POSITION_POOL_START_VALUES = 1 << 12;
|
|
3257
|
+
var RETIRED_DOCUMENTS_START_CAPACITY = 1 << 10;
|
|
3258
|
+
var SLICE_GROWTH_FACTOR = 1.5;
|
|
3259
|
+
var POOL_GROWTH_FACTOR = 1.5;
|
|
3260
|
+
var POOL_VACUUM_FREE_FRACTION = 0.25;
|
|
3261
|
+
var CACHED_VIEW_MIN_ENTRIES = 128;
|
|
3262
|
+
var TERM_VACUUM_MIN_DEAD_TERMS = 1 << 10;
|
|
3263
|
+
var POOL_VACUUM_MIN_FREE_ENTRIES = 1 << 14;
|
|
3264
|
+
var POSITION_VACUUM_MIN_FREE_VALUES = 1 << 14;
|
|
3265
|
+
var TERM_ENTRY_ESTIMATED_BYTES = 64;
|
|
3266
|
+
var TOKEN_STRING_HEADER_BYTES = 16;
|
|
3267
|
+
|
|
3268
|
+
// src/core/inverted-index/pools.ts
|
|
3269
|
+
var NO_SLICE = 255;
|
|
3270
|
+
var EMPTY_POSITIONS = 0;
|
|
3271
|
+
var MAX_POOL_ENTRIES = 2 ** 32 - 1;
|
|
3272
|
+
var SLICE_CAPACITIES = (() => {
|
|
3273
|
+
const capacities = [1];
|
|
3274
|
+
while (capacities[capacities.length - 1] < MAX_POOL_ENTRIES) {
|
|
3275
|
+
const previous = capacities[capacities.length - 1];
|
|
3276
|
+
capacities.push(Math.min(MAX_POOL_ENTRIES, Math.max(previous + 1, Math.ceil(previous * SLICE_GROWTH_FACTOR))));
|
|
3277
|
+
}
|
|
3278
|
+
return Uint32Array.from(capacities);
|
|
3279
|
+
})();
|
|
3280
|
+
function sliceClassFor(entries) {
|
|
3281
|
+
let low = 0;
|
|
3282
|
+
let high = SLICE_CAPACITIES.length - 1;
|
|
3283
|
+
while (low < high) {
|
|
3284
|
+
const middle = low + high >>> 1;
|
|
3285
|
+
if (SLICE_CAPACITIES[middle] < entries) low = middle + 1;
|
|
3286
|
+
else high = middle;
|
|
3080
3287
|
}
|
|
3081
|
-
|
|
3082
|
-
if (list.positions) list.positions.length = writeIdx;
|
|
3083
|
-
list.length = writeIdx;
|
|
3084
|
-
list.deletedDocs.clear();
|
|
3085
|
-
list.ordered = ascending;
|
|
3288
|
+
return low;
|
|
3086
3289
|
}
|
|
3087
|
-
function
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3290
|
+
function sliceCapacity(sliceClass) {
|
|
3291
|
+
return sliceClass === NO_SLICE ? 0 : SLICE_CAPACITIES[sliceClass];
|
|
3292
|
+
}
|
|
3293
|
+
function grownCapacity(current, needed) {
|
|
3294
|
+
return Math.min(MAX_POOL_ENTRIES, Math.max(needed, Math.ceil(current * POOL_GROWTH_FACTOR)));
|
|
3295
|
+
}
|
|
3296
|
+
var PostingPool = class {
|
|
3297
|
+
docIds;
|
|
3298
|
+
frequencies;
|
|
3299
|
+
fieldIndices;
|
|
3300
|
+
positionStarts = null;
|
|
3301
|
+
end = 0;
|
|
3302
|
+
epoch = 0;
|
|
3303
|
+
freeEntries = 0;
|
|
3304
|
+
freeLists = [];
|
|
3305
|
+
constructor(capacity) {
|
|
3306
|
+
this.docIds = new Uint32Array(capacity);
|
|
3307
|
+
this.frequencies = new Uint16Array(capacity);
|
|
3308
|
+
this.fieldIndices = new Uint16Array(capacity);
|
|
3309
|
+
}
|
|
3310
|
+
allocate(sliceClass) {
|
|
3311
|
+
const size = sliceCapacity(sliceClass);
|
|
3312
|
+
const free = this.freeLists[sliceClass];
|
|
3313
|
+
if (free !== void 0) {
|
|
3314
|
+
const reused = free.pop();
|
|
3315
|
+
if (reused !== void 0) {
|
|
3316
|
+
this.freeEntries -= size;
|
|
3317
|
+
return reused;
|
|
3318
|
+
}
|
|
3319
|
+
}
|
|
3320
|
+
const start = this.end;
|
|
3321
|
+
this.reserve(start + size);
|
|
3322
|
+
this.end = start + size;
|
|
3323
|
+
return start;
|
|
3324
|
+
}
|
|
3325
|
+
release(start, sliceClass) {
|
|
3326
|
+
if (sliceClass === NO_SLICE) return;
|
|
3327
|
+
let free = this.freeLists[sliceClass];
|
|
3328
|
+
if (free === void 0) {
|
|
3329
|
+
free = [];
|
|
3330
|
+
this.freeLists[sliceClass] = free;
|
|
3331
|
+
}
|
|
3332
|
+
free.push(start);
|
|
3333
|
+
this.freeEntries += sliceCapacity(sliceClass);
|
|
3334
|
+
}
|
|
3335
|
+
trackPositions() {
|
|
3336
|
+
if (this.positionStarts === null) {
|
|
3337
|
+
this.positionStarts = new Uint32Array(this.docIds.length);
|
|
3338
|
+
this.epoch++;
|
|
3339
|
+
}
|
|
3340
|
+
return this.positionStarts;
|
|
3341
|
+
}
|
|
3342
|
+
move(from, to, count) {
|
|
3343
|
+
this.docIds.copyWithin(to, from, from + count);
|
|
3344
|
+
this.frequencies.copyWithin(to, from, from + count);
|
|
3345
|
+
this.fieldIndices.copyWithin(to, from, from + count);
|
|
3346
|
+
if (this.positionStarts !== null) this.positionStarts.copyWithin(to, from, from + count);
|
|
3347
|
+
}
|
|
3348
|
+
byteLength() {
|
|
3349
|
+
const positionBytes = this.positionStarts === null ? 0 : this.positionStarts.byteLength;
|
|
3350
|
+
return this.docIds.byteLength + this.frequencies.byteLength + this.fieldIndices.byteLength + positionBytes;
|
|
3351
|
+
}
|
|
3352
|
+
reserve(needed) {
|
|
3353
|
+
if (needed <= this.docIds.length) return;
|
|
3354
|
+
const capacity = grownCapacity(this.docIds.length, needed);
|
|
3355
|
+
this.docIds = grownCopy(this.docIds, new Uint32Array(capacity), this.end);
|
|
3356
|
+
this.frequencies = grownCopy(this.frequencies, new Uint16Array(capacity), this.end);
|
|
3357
|
+
this.fieldIndices = grownCopy(this.fieldIndices, new Uint16Array(capacity), this.end);
|
|
3358
|
+
if (this.positionStarts !== null) {
|
|
3359
|
+
this.positionStarts = grownCopy(this.positionStarts, new Uint32Array(capacity), this.end);
|
|
3360
|
+
}
|
|
3361
|
+
this.epoch++;
|
|
3104
3362
|
}
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3363
|
+
};
|
|
3364
|
+
function grownCopy(source, target, used) {
|
|
3365
|
+
target.set(source.subarray(0, used));
|
|
3366
|
+
return target;
|
|
3109
3367
|
}
|
|
3368
|
+
var PositionPool = class {
|
|
3369
|
+
values;
|
|
3370
|
+
end = 1;
|
|
3371
|
+
freeValues = 0;
|
|
3372
|
+
constructor(capacity) {
|
|
3373
|
+
this.values = new Uint32Array(Math.max(1, capacity));
|
|
3374
|
+
}
|
|
3375
|
+
append(positions) {
|
|
3376
|
+
const count = positions.length;
|
|
3377
|
+
if (count === 0) return EMPTY_POSITIONS;
|
|
3378
|
+
const start = this.end;
|
|
3379
|
+
this.reserve(start + 1 + count);
|
|
3380
|
+
const values = this.values;
|
|
3381
|
+
values[start] = count;
|
|
3382
|
+
for (let i = 0; i < count; i++) values[start + 1 + i] = positions[i];
|
|
3383
|
+
this.end = start + 1 + count;
|
|
3384
|
+
return start;
|
|
3385
|
+
}
|
|
3386
|
+
appendFrom(source, start) {
|
|
3387
|
+
if (start === EMPTY_POSITIONS) return EMPTY_POSITIONS;
|
|
3388
|
+
const count = source.values[start];
|
|
3389
|
+
const target = this.end;
|
|
3390
|
+
this.reserve(target + 1 + count);
|
|
3391
|
+
this.values.set(source.values.subarray(start, start + 1 + count), target);
|
|
3392
|
+
this.end = target + 1 + count;
|
|
3393
|
+
return target;
|
|
3394
|
+
}
|
|
3395
|
+
countAt(start) {
|
|
3396
|
+
return this.values[start];
|
|
3397
|
+
}
|
|
3398
|
+
read(start) {
|
|
3399
|
+
const count = this.values[start];
|
|
3400
|
+
const out = new Array(count);
|
|
3401
|
+
for (let i = 0; i < count; i++) out[i] = this.values[start + 1 + i];
|
|
3402
|
+
return out;
|
|
3403
|
+
}
|
|
3404
|
+
copyInto(start, target, offset) {
|
|
3405
|
+
const count = this.values[start];
|
|
3406
|
+
if (count > 0) target.set(this.values.subarray(start + 1, start + 1 + count), offset);
|
|
3407
|
+
return count;
|
|
3408
|
+
}
|
|
3409
|
+
release(start) {
|
|
3410
|
+
if (start === EMPTY_POSITIONS) return;
|
|
3411
|
+
this.freeValues += this.values[start] + 1;
|
|
3412
|
+
}
|
|
3413
|
+
reserve(needed) {
|
|
3414
|
+
if (needed <= this.values.length) return;
|
|
3415
|
+
const capacity = grownCapacity(this.values.length, needed);
|
|
3416
|
+
const grown = new Uint32Array(capacity);
|
|
3417
|
+
grown.set(this.values.subarray(0, this.end));
|
|
3418
|
+
this.values = grown;
|
|
3419
|
+
}
|
|
3420
|
+
};
|
|
3110
3421
|
|
|
3111
|
-
// src/core/inverted-index.ts
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3422
|
+
// src/core/inverted-index/terms.ts
|
|
3423
|
+
var TERM_LIVE = 1;
|
|
3424
|
+
var TERM_ORDERED = 2;
|
|
3425
|
+
var TERM_POSITIONS = 4;
|
|
3426
|
+
function grow(source, target) {
|
|
3427
|
+
target.set(source);
|
|
3428
|
+
return target;
|
|
3429
|
+
}
|
|
3430
|
+
var TermColumns = class _TermColumns {
|
|
3431
|
+
constructor(startCapacity) {
|
|
3432
|
+
this.startCapacity = startCapacity;
|
|
3433
|
+
this.sliceStart = new Uint32Array(startCapacity);
|
|
3434
|
+
this.sliceClass = new Uint8Array(startCapacity);
|
|
3435
|
+
this.lengths = new Uint32Array(startCapacity);
|
|
3436
|
+
this.docFrequency = new Uint32Array(startCapacity);
|
|
3437
|
+
this.deadDocuments = new Uint32Array(startCapacity);
|
|
3438
|
+
this.totalTermFrequency = new Float64Array(startCapacity);
|
|
3439
|
+
this.revision = new Uint32Array(startCapacity);
|
|
3440
|
+
this.flags = new Uint8Array(startCapacity);
|
|
3441
|
+
}
|
|
3442
|
+
ids = /* @__PURE__ */ new Map();
|
|
3443
|
+
tokens = [];
|
|
3444
|
+
sliceStart;
|
|
3445
|
+
sliceClass;
|
|
3446
|
+
lengths;
|
|
3447
|
+
docFrequency;
|
|
3448
|
+
deadDocuments;
|
|
3449
|
+
totalTermFrequency;
|
|
3450
|
+
revision;
|
|
3451
|
+
flags;
|
|
3452
|
+
count = 0;
|
|
3453
|
+
liveCount = 0;
|
|
3454
|
+
generation = 0;
|
|
3455
|
+
tokenCodeUnits = 0;
|
|
3456
|
+
buckets = /* @__PURE__ */ new Map();
|
|
3457
|
+
get deadCount() {
|
|
3458
|
+
return this.count - this.liveCount;
|
|
3459
|
+
}
|
|
3460
|
+
add(token) {
|
|
3461
|
+
const ordinal = this.count;
|
|
3462
|
+
if (ordinal === this.flags.length) this.growTo(ordinal * 2);
|
|
3463
|
+
this.count = ordinal + 1;
|
|
3464
|
+
this.liveCount++;
|
|
3465
|
+
this.tokenCodeUnits += token.length;
|
|
3466
|
+
this.ids.set(token, ordinal);
|
|
3467
|
+
this.tokens.push(token);
|
|
3468
|
+
this.sliceClass[ordinal] = NO_SLICE;
|
|
3469
|
+
this.flags[ordinal] = TERM_LIVE | TERM_ORDERED;
|
|
3470
|
+
this.track(token, ordinal);
|
|
3471
|
+
return ordinal;
|
|
3472
|
+
}
|
|
3473
|
+
kill(ordinal) {
|
|
3474
|
+
const token = this.tokens[ordinal];
|
|
3475
|
+
this.ids.delete(token);
|
|
3476
|
+
const bucket = token.length > 0 ? this.buckets.get(token[0]) : void 0;
|
|
3477
|
+
if (bucket !== void 0) bucket.dead++;
|
|
3478
|
+
this.tokens[ordinal] = "";
|
|
3479
|
+
this.flags[ordinal] = 0;
|
|
3480
|
+
this.lengths[ordinal] = 0;
|
|
3481
|
+
this.docFrequency[ordinal] = 0;
|
|
3482
|
+
this.deadDocuments[ordinal] = 0;
|
|
3483
|
+
this.totalTermFrequency[ordinal] = 0;
|
|
3484
|
+
this.sliceClass[ordinal] = NO_SLICE;
|
|
3485
|
+
this.liveCount--;
|
|
3486
|
+
this.tokenCodeUnits -= token.length;
|
|
3487
|
+
}
|
|
3488
|
+
isLive(ordinal) {
|
|
3489
|
+
return (this.flags[ordinal] & TERM_LIVE) !== 0;
|
|
3490
|
+
}
|
|
3491
|
+
forEachLive(visit) {
|
|
3492
|
+
const flags = this.flags;
|
|
3493
|
+
for (let ordinal = 0; ordinal < this.count; ordinal++) {
|
|
3494
|
+
if ((flags[ordinal] & TERM_LIVE) !== 0) visit(ordinal);
|
|
3495
|
+
}
|
|
3496
|
+
}
|
|
3497
|
+
forEachStartingWith(prefix, visit) {
|
|
3498
|
+
if (prefix.length === 0) {
|
|
3499
|
+
this.forEachLive(visit);
|
|
3500
|
+
return;
|
|
3124
3501
|
}
|
|
3125
|
-
bucket.
|
|
3502
|
+
const bucket = this.buckets.get(prefix[0]);
|
|
3503
|
+
if (bucket === void 0) return;
|
|
3504
|
+
this.compactBucket(prefix[0], bucket);
|
|
3505
|
+
const { ordinals, length } = bucket;
|
|
3506
|
+
const flags = this.flags;
|
|
3507
|
+
const tokens = this.tokens;
|
|
3508
|
+
const checkPrefix = prefix.length > 1;
|
|
3509
|
+
for (let i = 0; i < length; i++) {
|
|
3510
|
+
const ordinal = ordinals[i];
|
|
3511
|
+
if ((flags[ordinal] & TERM_LIVE) === 0) continue;
|
|
3512
|
+
if (checkPrefix && !tokens[ordinal].startsWith(prefix)) continue;
|
|
3513
|
+
visit(ordinal);
|
|
3514
|
+
}
|
|
3515
|
+
}
|
|
3516
|
+
renumber() {
|
|
3517
|
+
const remap = new Int32Array(this.count).fill(-1);
|
|
3518
|
+
const capacity = Math.max(this.startCapacity, this.liveCount);
|
|
3519
|
+
const next = new _TermColumns(capacity);
|
|
3520
|
+
for (let ordinal = 0; ordinal < this.count; ordinal++) {
|
|
3521
|
+
if ((this.flags[ordinal] & TERM_LIVE) === 0) continue;
|
|
3522
|
+
const target = next.add(this.tokens[ordinal]);
|
|
3523
|
+
remap[ordinal] = target;
|
|
3524
|
+
next.sliceStart[target] = this.sliceStart[ordinal];
|
|
3525
|
+
next.sliceClass[target] = this.sliceClass[ordinal];
|
|
3526
|
+
next.lengths[target] = this.lengths[ordinal];
|
|
3527
|
+
next.docFrequency[target] = this.docFrequency[ordinal];
|
|
3528
|
+
next.deadDocuments[target] = this.deadDocuments[ordinal];
|
|
3529
|
+
next.totalTermFrequency[target] = this.totalTermFrequency[ordinal];
|
|
3530
|
+
next.revision[target] = this.revision[ordinal];
|
|
3531
|
+
next.flags[target] = this.flags[ordinal];
|
|
3532
|
+
}
|
|
3533
|
+
this.ids = next.ids;
|
|
3534
|
+
this.tokens = next.tokens;
|
|
3535
|
+
this.sliceStart = next.sliceStart;
|
|
3536
|
+
this.sliceClass = next.sliceClass;
|
|
3537
|
+
this.lengths = next.lengths;
|
|
3538
|
+
this.docFrequency = next.docFrequency;
|
|
3539
|
+
this.deadDocuments = next.deadDocuments;
|
|
3540
|
+
this.totalTermFrequency = next.totalTermFrequency;
|
|
3541
|
+
this.revision = next.revision;
|
|
3542
|
+
this.flags = next.flags;
|
|
3543
|
+
this.buckets = next.buckets;
|
|
3544
|
+
this.count = next.count;
|
|
3545
|
+
this.liveCount = next.liveCount;
|
|
3546
|
+
this.generation++;
|
|
3547
|
+
return remap;
|
|
3126
3548
|
}
|
|
3127
|
-
|
|
3549
|
+
clear() {
|
|
3550
|
+
this.ids = /* @__PURE__ */ new Map();
|
|
3551
|
+
this.tokens = [];
|
|
3552
|
+
this.buckets = /* @__PURE__ */ new Map();
|
|
3553
|
+
this.count = 0;
|
|
3554
|
+
this.liveCount = 0;
|
|
3555
|
+
this.tokenCodeUnits = 0;
|
|
3556
|
+
this.generation++;
|
|
3557
|
+
this.sliceStart = new Uint32Array(this.startCapacity);
|
|
3558
|
+
this.sliceClass = new Uint8Array(this.startCapacity);
|
|
3559
|
+
this.lengths = new Uint32Array(this.startCapacity);
|
|
3560
|
+
this.docFrequency = new Uint32Array(this.startCapacity);
|
|
3561
|
+
this.deadDocuments = new Uint32Array(this.startCapacity);
|
|
3562
|
+
this.totalTermFrequency = new Float64Array(this.startCapacity);
|
|
3563
|
+
this.revision = new Uint32Array(this.startCapacity);
|
|
3564
|
+
this.flags = new Uint8Array(this.startCapacity);
|
|
3565
|
+
}
|
|
3566
|
+
byteLength() {
|
|
3567
|
+
let bucketBytes = 0;
|
|
3568
|
+
for (const bucket of this.buckets.values()) bucketBytes += bucket.ordinals.byteLength;
|
|
3569
|
+
return this.sliceStart.byteLength + this.sliceClass.byteLength + this.lengths.byteLength + this.docFrequency.byteLength + this.deadDocuments.byteLength + this.totalTermFrequency.byteLength + this.revision.byteLength + this.flags.byteLength + bucketBytes;
|
|
3570
|
+
}
|
|
3571
|
+
track(token, ordinal) {
|
|
3128
3572
|
if (token.length === 0) return;
|
|
3129
|
-
const
|
|
3130
|
-
|
|
3131
|
-
if (bucket) {
|
|
3132
|
-
bucket
|
|
3133
|
-
|
|
3134
|
-
}
|
|
3573
|
+
const first = token[0];
|
|
3574
|
+
let bucket = this.buckets.get(first);
|
|
3575
|
+
if (bucket === void 0) {
|
|
3576
|
+
bucket = { ordinals: new Uint32Array(4), length: 0, dead: 0 };
|
|
3577
|
+
this.buckets.set(first, bucket);
|
|
3578
|
+
}
|
|
3579
|
+
if (bucket.length === bucket.ordinals.length) {
|
|
3580
|
+
bucket.ordinals = grow(bucket.ordinals, new Uint32Array(bucket.ordinals.length * 2));
|
|
3581
|
+
}
|
|
3582
|
+
bucket.ordinals[bucket.length++] = ordinal;
|
|
3583
|
+
}
|
|
3584
|
+
compactBucket(first, bucket) {
|
|
3585
|
+
if (bucket.dead * 2 <= bucket.length) return;
|
|
3586
|
+
const { ordinals } = bucket;
|
|
3587
|
+
let kept = 0;
|
|
3588
|
+
for (let i = 0; i < bucket.length; i++) {
|
|
3589
|
+
const ordinal = ordinals[i];
|
|
3590
|
+
if ((this.flags[ordinal] & TERM_LIVE) !== 0) ordinals[kept++] = ordinal;
|
|
3591
|
+
}
|
|
3592
|
+
bucket.length = kept;
|
|
3593
|
+
bucket.dead = 0;
|
|
3594
|
+
if (kept === 0) this.buckets.delete(first);
|
|
3595
|
+
}
|
|
3596
|
+
growTo(capacity) {
|
|
3597
|
+
this.sliceStart = grow(this.sliceStart, new Uint32Array(capacity));
|
|
3598
|
+
this.sliceClass = grow(this.sliceClass, new Uint8Array(capacity));
|
|
3599
|
+
this.lengths = grow(this.lengths, new Uint32Array(capacity));
|
|
3600
|
+
this.docFrequency = grow(this.docFrequency, new Uint32Array(capacity));
|
|
3601
|
+
this.deadDocuments = grow(this.deadDocuments, new Uint32Array(capacity));
|
|
3602
|
+
this.totalTermFrequency = grow(this.totalTermFrequency, new Float64Array(capacity));
|
|
3603
|
+
this.revision = grow(this.revision, new Uint32Array(capacity));
|
|
3604
|
+
this.flags = grow(this.flags, new Uint8Array(capacity));
|
|
3135
3605
|
}
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3606
|
+
};
|
|
3607
|
+
|
|
3608
|
+
// src/core/inverted-index/view.ts
|
|
3609
|
+
var NO_TOMBSTONES = { size: 0, has: () => false };
|
|
3610
|
+
var NO_DOCUMENT_IDS = new Uint32Array(0);
|
|
3611
|
+
var NO_FREQUENCIES = new Uint16Array(0);
|
|
3612
|
+
var DETACHED = -1;
|
|
3613
|
+
var LivePostingView = class {
|
|
3614
|
+
constructor(store, token, ordinal, keepsPositions) {
|
|
3615
|
+
this.store = store;
|
|
3616
|
+
this.token = token;
|
|
3617
|
+
this.keepsPositions = keepsPositions;
|
|
3618
|
+
this.ordinal = ordinal;
|
|
3619
|
+
this.generation = store.terms.generation;
|
|
3620
|
+
const view = this;
|
|
3621
|
+
this.docIdSet = {
|
|
3622
|
+
get size() {
|
|
3623
|
+
const live = view.sync();
|
|
3624
|
+
return live === DETACHED ? 0 : view.store.terms.docFrequency[live];
|
|
3145
3625
|
}
|
|
3626
|
+
};
|
|
3627
|
+
this.retiredRows = {
|
|
3628
|
+
get size() {
|
|
3629
|
+
const live = view.sync();
|
|
3630
|
+
return live === DETACHED ? 0 : view.store.terms.deadDocuments[live];
|
|
3631
|
+
},
|
|
3632
|
+
has: (internalId) => store.isRetired(internalId)
|
|
3633
|
+
};
|
|
3634
|
+
}
|
|
3635
|
+
docIdSet;
|
|
3636
|
+
retiredRows;
|
|
3637
|
+
ordinal;
|
|
3638
|
+
generation;
|
|
3639
|
+
syncedEpoch = -1;
|
|
3640
|
+
syncedStart = -1;
|
|
3641
|
+
syncedLength = 0;
|
|
3642
|
+
syncedRevision = -1;
|
|
3643
|
+
ids = NO_DOCUMENT_IDS;
|
|
3644
|
+
frequencies = NO_FREQUENCIES;
|
|
3645
|
+
fields = NO_FREQUENCIES;
|
|
3646
|
+
materialized = null;
|
|
3647
|
+
get length() {
|
|
3648
|
+
this.sync();
|
|
3649
|
+
return this.syncedLength;
|
|
3650
|
+
}
|
|
3651
|
+
get docIds() {
|
|
3652
|
+
this.sync();
|
|
3653
|
+
return this.ids;
|
|
3654
|
+
}
|
|
3655
|
+
get termFrequencies() {
|
|
3656
|
+
this.sync();
|
|
3657
|
+
return this.frequencies;
|
|
3658
|
+
}
|
|
3659
|
+
get fieldNameIndices() {
|
|
3660
|
+
this.sync();
|
|
3661
|
+
return this.fields;
|
|
3662
|
+
}
|
|
3663
|
+
get holdsPositions() {
|
|
3664
|
+
const live = this.sync();
|
|
3665
|
+
return live !== DETACHED && (this.store.terms.flags[live] & TERM_POSITIONS) !== 0;
|
|
3666
|
+
}
|
|
3667
|
+
get positions() {
|
|
3668
|
+
if (!this.holdsPositions) return null;
|
|
3669
|
+
if (this.materialized !== null) return this.materialized;
|
|
3670
|
+
const starts = this.store.pool.positionStarts;
|
|
3671
|
+
const rows = new Array(this.syncedLength);
|
|
3672
|
+
for (let row = 0; row < this.syncedLength; row++) {
|
|
3673
|
+
rows[row] = starts === null ? [] : this.store.positions.read(starts[this.syncedStart + row]);
|
|
3674
|
+
}
|
|
3675
|
+
if (this.keepsPositions) this.materialized = rows;
|
|
3676
|
+
return rows;
|
|
3677
|
+
}
|
|
3678
|
+
positionCountAt(row) {
|
|
3679
|
+
if (!this.holdsPositions) return 0;
|
|
3680
|
+
const starts = this.store.pool.positionStarts;
|
|
3681
|
+
return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
|
|
3682
|
+
}
|
|
3683
|
+
copyPositionsAt(row, target, offset) {
|
|
3684
|
+
if (!this.holdsPositions) return 0;
|
|
3685
|
+
const starts = this.store.pool.positionStarts;
|
|
3686
|
+
return starts === null ? 0 : this.store.positions.copyInto(starts[this.syncedStart + row], target, offset);
|
|
3687
|
+
}
|
|
3688
|
+
get deletedDocs() {
|
|
3689
|
+
const live = this.sync();
|
|
3690
|
+
if (live === DETACHED || this.store.terms.deadDocuments[live] === 0) return NO_TOMBSTONES;
|
|
3691
|
+
return this.retiredRows;
|
|
3692
|
+
}
|
|
3693
|
+
get totalTermFrequency() {
|
|
3694
|
+
const live = this.sync();
|
|
3695
|
+
return live === DETACHED ? 0 : this.store.terms.totalTermFrequency[live];
|
|
3696
|
+
}
|
|
3697
|
+
get structureRevision() {
|
|
3698
|
+
const live = this.sync();
|
|
3699
|
+
return live === DETACHED ? this.syncedRevision : this.store.terms.revision[live];
|
|
3700
|
+
}
|
|
3701
|
+
get ordered() {
|
|
3702
|
+
const live = this.sync();
|
|
3703
|
+
return live === DETACHED || (this.store.terms.flags[live] & TERM_ORDERED) !== 0;
|
|
3704
|
+
}
|
|
3705
|
+
sync() {
|
|
3706
|
+
const terms = this.store.terms;
|
|
3707
|
+
if (this.generation !== terms.generation) {
|
|
3708
|
+
this.ordinal = terms.ids.get(this.token) ?? DETACHED;
|
|
3709
|
+
this.generation = terms.generation;
|
|
3710
|
+
}
|
|
3711
|
+
const ordinal = this.ordinal;
|
|
3712
|
+
if (ordinal === DETACHED || !terms.isLive(ordinal)) {
|
|
3713
|
+
this.detach();
|
|
3714
|
+
return DETACHED;
|
|
3715
|
+
}
|
|
3716
|
+
const pool = this.store.pool;
|
|
3717
|
+
const start = terms.sliceStart[ordinal];
|
|
3718
|
+
const length = terms.lengths[ordinal];
|
|
3719
|
+
const revision = terms.revision[ordinal];
|
|
3720
|
+
if (pool.epoch !== this.syncedEpoch || start !== this.syncedStart || length !== this.syncedLength || revision !== this.syncedRevision) {
|
|
3721
|
+
this.ids = pool.docIds.subarray(start, start + length);
|
|
3722
|
+
this.frequencies = pool.frequencies.subarray(start, start + length);
|
|
3723
|
+
this.fields = pool.fieldIndices.subarray(start, start + length);
|
|
3724
|
+
this.materialized = null;
|
|
3725
|
+
this.syncedEpoch = pool.epoch;
|
|
3726
|
+
this.syncedStart = start;
|
|
3727
|
+
this.syncedLength = length;
|
|
3728
|
+
this.syncedRevision = revision;
|
|
3146
3729
|
}
|
|
3147
|
-
return
|
|
3730
|
+
return ordinal;
|
|
3731
|
+
}
|
|
3732
|
+
detach() {
|
|
3733
|
+
if (this.syncedStart === DETACHED && this.syncedLength === 0) return;
|
|
3734
|
+
this.ids = NO_DOCUMENT_IDS;
|
|
3735
|
+
this.frequencies = NO_FREQUENCIES;
|
|
3736
|
+
this.fields = NO_FREQUENCIES;
|
|
3737
|
+
this.materialized = null;
|
|
3738
|
+
this.syncedStart = DETACHED;
|
|
3739
|
+
this.syncedLength = 0;
|
|
3740
|
+
this.syncedEpoch = -1;
|
|
3148
3741
|
}
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3742
|
+
};
|
|
3743
|
+
|
|
3744
|
+
// src/core/inverted-index/postings.ts
|
|
3745
|
+
function holdsDocument(docIds, start, length, internalId, ordered) {
|
|
3746
|
+
if (!ordered) {
|
|
3747
|
+
for (let i = start; i < start + length; i++) {
|
|
3748
|
+
if (docIds[i] === internalId) return true;
|
|
3155
3749
|
}
|
|
3156
|
-
return
|
|
3750
|
+
return false;
|
|
3157
3751
|
}
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3752
|
+
let low = start;
|
|
3753
|
+
let high = start + length;
|
|
3754
|
+
while (low < high) {
|
|
3755
|
+
const middle = low + high >>> 1;
|
|
3756
|
+
if (docIds[middle] < internalId) low = middle + 1;
|
|
3757
|
+
else high = middle;
|
|
3758
|
+
}
|
|
3759
|
+
return low < start + length && docIds[low] === internalId;
|
|
3760
|
+
}
|
|
3761
|
+
var LivePostings = class {
|
|
3762
|
+
terms = new TermColumns(TERM_TABLE_START_CAPACITY);
|
|
3763
|
+
pool = new PostingPool(POSTING_POOL_START_ENTRIES);
|
|
3764
|
+
positions = new PositionPool(POSITION_POOL_START_VALUES);
|
|
3765
|
+
retired = new Uint8Array(RETIRED_DOCUMENTS_START_CAPACITY);
|
|
3766
|
+
batchMode = false;
|
|
3767
|
+
dirty = /* @__PURE__ */ new Set();
|
|
3768
|
+
cachedViews = /* @__PURE__ */ new Map();
|
|
3769
|
+
compactedSlack = 0;
|
|
3770
|
+
view(ordinal, cached) {
|
|
3771
|
+
if (!cached || this.terms.lengths[ordinal] < CACHED_VIEW_MIN_ENTRIES) {
|
|
3772
|
+
return new LivePostingView(this, this.terms.tokens[ordinal], ordinal, cached);
|
|
3773
|
+
}
|
|
3774
|
+
let view = this.cachedViews.get(ordinal);
|
|
3775
|
+
if (view === void 0) {
|
|
3776
|
+
view = new LivePostingView(this, this.terms.tokens[ordinal], ordinal, true);
|
|
3777
|
+
this.cachedViews.set(ordinal, view);
|
|
3778
|
+
}
|
|
3779
|
+
return view;
|
|
3780
|
+
}
|
|
3781
|
+
reserve(token, entries) {
|
|
3782
|
+
const terms = this.terms;
|
|
3783
|
+
const ordinal = terms.ids.get(token) ?? terms.add(token);
|
|
3784
|
+
if (sliceCapacity(terms.sliceClass[ordinal]) < terms.lengths[ordinal] + entries) {
|
|
3785
|
+
this.relocate(ordinal, terms.lengths[ordinal] + entries);
|
|
3786
|
+
}
|
|
3787
|
+
}
|
|
3788
|
+
insert(token, internalId, termFrequency, fieldIndex, positions) {
|
|
3789
|
+
const terms = this.terms;
|
|
3790
|
+
if (this.isRetired(internalId)) this.purgeRetired();
|
|
3791
|
+
const ordinal = terms.ids.get(token) ?? terms.add(token);
|
|
3792
|
+
const length = terms.lengths[ordinal];
|
|
3793
|
+
if (length >= sliceCapacity(terms.sliceClass[ordinal])) this.relocate(ordinal, length + 1);
|
|
3794
|
+
const pool = this.pool;
|
|
3795
|
+
const start = terms.sliceStart[ordinal];
|
|
3796
|
+
const at = start + length;
|
|
3797
|
+
const flags = terms.flags[ordinal];
|
|
3798
|
+
let newDocument = true;
|
|
3799
|
+
if (length > 0) {
|
|
3800
|
+
const last = pool.docIds[at - 1];
|
|
3801
|
+
const ordered = (flags & TERM_ORDERED) !== 0;
|
|
3802
|
+
if (internalId === last) {
|
|
3803
|
+
newDocument = false;
|
|
3804
|
+
} else if (!ordered || internalId < last) {
|
|
3805
|
+
newDocument = !holdsDocument(pool.docIds, start, length, internalId, ordered);
|
|
3806
|
+
if (internalId < last) terms.flags[ordinal] = flags & ~TERM_ORDERED;
|
|
3807
|
+
}
|
|
3808
|
+
}
|
|
3809
|
+
const stored = termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : termFrequency;
|
|
3810
|
+
pool.docIds[at] = internalId;
|
|
3811
|
+
pool.frequencies[at] = stored;
|
|
3812
|
+
pool.fieldIndices[at] = fieldIndex;
|
|
3813
|
+
terms.totalTermFrequency[ordinal] += stored;
|
|
3814
|
+
if (positions !== null) {
|
|
3815
|
+
const starts = pool.trackPositions();
|
|
3816
|
+
if ((flags & TERM_POSITIONS) === 0) {
|
|
3817
|
+
terms.flags[ordinal] |= TERM_POSITIONS;
|
|
3818
|
+
starts.fill(EMPTY_POSITIONS, start, at);
|
|
3819
|
+
}
|
|
3820
|
+
starts[at] = this.positions.append(positions);
|
|
3821
|
+
} else if ((flags & TERM_POSITIONS) !== 0) {
|
|
3822
|
+
pool.trackPositions()[at] = EMPTY_POSITIONS;
|
|
3823
|
+
}
|
|
3824
|
+
if (newDocument) terms.docFrequency[ordinal]++;
|
|
3825
|
+
terms.lengths[ordinal] = length + 1;
|
|
3826
|
+
this.repackIfSparse();
|
|
3827
|
+
}
|
|
3828
|
+
remove(token, internalId, occurrences) {
|
|
3829
|
+
const terms = this.terms;
|
|
3830
|
+
const ordinal = terms.ids.get(token);
|
|
3831
|
+
if (ordinal === void 0) return;
|
|
3832
|
+
const length = terms.lengths[ordinal];
|
|
3833
|
+
const ordered = (terms.flags[ordinal] & TERM_ORDERED) !== 0;
|
|
3834
|
+
if (!holdsDocument(this.pool.docIds, terms.sliceStart[ordinal], length, internalId, ordered)) return;
|
|
3835
|
+
terms.revision[ordinal]++;
|
|
3836
|
+
terms.docFrequency[ordinal]--;
|
|
3837
|
+
terms.deadDocuments[ordinal]++;
|
|
3838
|
+
this.retire(internalId);
|
|
3839
|
+
terms.totalTermFrequency[ordinal] = Math.max(0, terms.totalTermFrequency[ordinal] - occurrences);
|
|
3840
|
+
if (terms.docFrequency[ordinal] === 0) {
|
|
3841
|
+
this.dropTerm(ordinal);
|
|
3842
|
+
return;
|
|
3843
|
+
}
|
|
3844
|
+
if (this.batchMode) {
|
|
3845
|
+
this.dirty.add(ordinal);
|
|
3846
|
+
return;
|
|
3847
|
+
}
|
|
3848
|
+
if (terms.deadDocuments[ordinal] / length > POSTING_LIST_COMPACTION_THRESHOLD) this.compact(ordinal);
|
|
3849
|
+
this.repackIfSparse();
|
|
3850
|
+
}
|
|
3851
|
+
isRetired(internalId) {
|
|
3852
|
+
return internalId < this.retired.length && this.retired[internalId] === 1;
|
|
3853
|
+
}
|
|
3854
|
+
beginBatch() {
|
|
3855
|
+
this.batchMode = true;
|
|
3856
|
+
this.dirty.clear();
|
|
3857
|
+
}
|
|
3858
|
+
endBatch() {
|
|
3859
|
+
this.batchMode = false;
|
|
3860
|
+
const terms = this.terms;
|
|
3861
|
+
for (const ordinal of this.dirty) {
|
|
3862
|
+
if (!terms.isLive(ordinal)) continue;
|
|
3863
|
+
if (terms.deadDocuments[ordinal] / terms.lengths[ordinal] > POSTING_LIST_COMPACTION_THRESHOLD) {
|
|
3864
|
+
this.compact(ordinal);
|
|
3200
3865
|
}
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3866
|
+
}
|
|
3867
|
+
this.dirty.clear();
|
|
3868
|
+
this.repackIfSparse();
|
|
3869
|
+
}
|
|
3870
|
+
clear() {
|
|
3871
|
+
const epoch = this.pool.epoch + 1;
|
|
3872
|
+
this.terms.clear();
|
|
3873
|
+
this.pool = new PostingPool(POSTING_POOL_START_ENTRIES);
|
|
3874
|
+
this.pool.epoch = epoch;
|
|
3875
|
+
this.positions = new PositionPool(POSITION_POOL_START_VALUES);
|
|
3876
|
+
this.retired = new Uint8Array(RETIRED_DOCUMENTS_START_CAPACITY);
|
|
3877
|
+
this.dirty.clear();
|
|
3878
|
+
this.cachedViews.clear();
|
|
3879
|
+
this.compactedSlack = 0;
|
|
3880
|
+
}
|
|
3881
|
+
byteLength() {
|
|
3882
|
+
return this.terms.byteLength() + this.pool.byteLength() + this.positions.values.byteLength + this.retired.byteLength;
|
|
3883
|
+
}
|
|
3884
|
+
retire(internalId) {
|
|
3885
|
+
if (internalId >= this.retired.length) {
|
|
3886
|
+
const grown = new Uint8Array(Math.max(internalId + 1, this.retired.length * 2));
|
|
3887
|
+
grown.set(this.retired);
|
|
3888
|
+
this.retired = grown;
|
|
3889
|
+
}
|
|
3890
|
+
this.retired[internalId] = 1;
|
|
3891
|
+
}
|
|
3892
|
+
purgeRetired() {
|
|
3893
|
+
const terms = this.terms;
|
|
3894
|
+
for (let ordinal = 0; ordinal < terms.count; ordinal++) {
|
|
3895
|
+
if (terms.isLive(ordinal) && terms.deadDocuments[ordinal] > 0) this.compact(ordinal);
|
|
3896
|
+
}
|
|
3897
|
+
this.retired.fill(0);
|
|
3898
|
+
}
|
|
3899
|
+
compact(ordinal) {
|
|
3900
|
+
if (this.terms.deadDocuments[ordinal] === 0) return;
|
|
3901
|
+
const retired = this.retired;
|
|
3902
|
+
this.filterEntries(ordinal, (id) => id < retired.length && retired[id] === 1);
|
|
3903
|
+
this.terms.deadDocuments[ordinal] = 0;
|
|
3904
|
+
}
|
|
3905
|
+
filterEntries(ordinal, drop) {
|
|
3906
|
+
const terms = this.terms;
|
|
3907
|
+
const pool = this.pool;
|
|
3908
|
+
const start = terms.sliceStart[ordinal];
|
|
3909
|
+
const length = terms.lengths[ordinal];
|
|
3910
|
+
const starts = (terms.flags[ordinal] & TERM_POSITIONS) !== 0 ? pool.positionStarts : null;
|
|
3911
|
+
const { docIds, frequencies, fieldIndices } = pool;
|
|
3912
|
+
terms.revision[ordinal]++;
|
|
3913
|
+
let kept = 0;
|
|
3914
|
+
let ascending = true;
|
|
3915
|
+
for (let i = start; i < start + length; i++) {
|
|
3916
|
+
const internalId = docIds[i];
|
|
3917
|
+
if (drop(internalId)) {
|
|
3918
|
+
if (starts !== null) this.positions.release(starts[i]);
|
|
3919
|
+
continue;
|
|
3204
3920
|
}
|
|
3205
|
-
|
|
3206
|
-
|
|
3921
|
+
const target = start + kept;
|
|
3922
|
+
if (kept > 0 && internalId < docIds[target - 1]) ascending = false;
|
|
3923
|
+
if (target !== i) {
|
|
3924
|
+
docIds[target] = internalId;
|
|
3925
|
+
frequencies[target] = frequencies[i];
|
|
3926
|
+
fieldIndices[target] = fieldIndices[i];
|
|
3927
|
+
if (starts !== null) starts[target] = starts[i];
|
|
3928
|
+
}
|
|
3929
|
+
kept++;
|
|
3930
|
+
}
|
|
3931
|
+
terms.lengths[ordinal] = kept;
|
|
3932
|
+
terms.flags[ordinal] = ascending ? terms.flags[ordinal] | TERM_ORDERED : terms.flags[ordinal] & ~TERM_ORDERED;
|
|
3933
|
+
this.compactedSlack += length - kept;
|
|
3934
|
+
}
|
|
3935
|
+
relocate(ordinal, needed) {
|
|
3936
|
+
const terms = this.terms;
|
|
3937
|
+
const previousClass = terms.sliceClass[ordinal];
|
|
3938
|
+
const previousStart = terms.sliceStart[ordinal];
|
|
3939
|
+
const length = terms.lengths[ordinal];
|
|
3940
|
+
const grownClass = previousClass === NO_SLICE ? 0 : previousClass + 1;
|
|
3941
|
+
const nextClass = Math.max(grownClass, sliceClassFor(needed));
|
|
3942
|
+
const nextStart = this.pool.allocate(nextClass);
|
|
3943
|
+
if (length > 0) this.pool.move(previousStart, nextStart, length);
|
|
3944
|
+
this.pool.release(previousStart, previousClass);
|
|
3945
|
+
terms.sliceStart[ordinal] = nextStart;
|
|
3946
|
+
terms.sliceClass[ordinal] = nextClass;
|
|
3947
|
+
}
|
|
3948
|
+
dropTerm(ordinal) {
|
|
3949
|
+
const terms = this.terms;
|
|
3950
|
+
const pool = this.pool;
|
|
3951
|
+
if ((terms.flags[ordinal] & TERM_POSITIONS) !== 0 && pool.positionStarts !== null) {
|
|
3952
|
+
const start = terms.sliceStart[ordinal];
|
|
3953
|
+
for (let i = start; i < start + terms.lengths[ordinal]; i++) this.positions.release(pool.positionStarts[i]);
|
|
3954
|
+
}
|
|
3955
|
+
pool.release(terms.sliceStart[ordinal], terms.sliceClass[ordinal]);
|
|
3956
|
+
this.dirty.delete(ordinal);
|
|
3957
|
+
this.cachedViews.delete(ordinal);
|
|
3958
|
+
terms.kill(ordinal);
|
|
3959
|
+
if (terms.deadCount >= TERM_VACUUM_MIN_DEAD_TERMS && terms.deadCount > terms.liveCount) this.renumberTerms();
|
|
3960
|
+
}
|
|
3961
|
+
renumberTerms() {
|
|
3962
|
+
const remap = this.terms.renumber();
|
|
3963
|
+
const dirty = /* @__PURE__ */ new Set();
|
|
3964
|
+
for (const ordinal of this.dirty) {
|
|
3965
|
+
if (remap[ordinal] >= 0) dirty.add(remap[ordinal]);
|
|
3966
|
+
}
|
|
3967
|
+
this.dirty = dirty;
|
|
3968
|
+
this.cachedViews.clear();
|
|
3969
|
+
}
|
|
3970
|
+
repackIfSparse() {
|
|
3971
|
+
const pool = this.pool;
|
|
3972
|
+
const freeEntries = pool.freeEntries + this.compactedSlack;
|
|
3973
|
+
const sparsePostings = freeEntries >= POOL_VACUUM_MIN_FREE_ENTRIES && freeEntries > pool.end * POOL_VACUUM_FREE_FRACTION;
|
|
3974
|
+
const positions = this.positions;
|
|
3975
|
+
const sparsePositions = positions.freeValues >= POSITION_VACUUM_MIN_FREE_VALUES && positions.freeValues > positions.end * POOL_VACUUM_FREE_FRACTION;
|
|
3976
|
+
if (sparsePostings || sparsePositions) this.repack(sparsePositions);
|
|
3977
|
+
}
|
|
3978
|
+
repack(rebuildPositions) {
|
|
3979
|
+
const terms = this.terms;
|
|
3980
|
+
const pool = this.pool;
|
|
3981
|
+
const positions = this.positions;
|
|
3982
|
+
let capacity = 0;
|
|
3983
|
+
for (let ordinal = 0; ordinal < terms.count; ordinal++) {
|
|
3984
|
+
if (terms.isLive(ordinal)) capacity += sliceCapacity(sliceClassFor(terms.lengths[ordinal]));
|
|
3985
|
+
}
|
|
3986
|
+
const nextPool = new PostingPool(Math.max(1, capacity));
|
|
3987
|
+
nextPool.epoch = pool.epoch + 1;
|
|
3988
|
+
const sourceStarts = pool.positionStarts;
|
|
3989
|
+
const targetStarts = sourceStarts === null ? null : nextPool.trackPositions();
|
|
3990
|
+
const nextPositions = rebuildPositions ? new PositionPool(positions.end - positions.freeValues) : positions;
|
|
3991
|
+
for (let ordinal = 0; ordinal < terms.count; ordinal++) {
|
|
3992
|
+
if (!terms.isLive(ordinal)) continue;
|
|
3993
|
+
const start = terms.sliceStart[ordinal];
|
|
3994
|
+
const length = terms.lengths[ordinal];
|
|
3995
|
+
const sliceClass = sliceClassFor(length);
|
|
3996
|
+
const target = nextPool.allocate(sliceClass);
|
|
3997
|
+
nextPool.docIds.set(pool.docIds.subarray(start, start + length), target);
|
|
3998
|
+
nextPool.frequencies.set(pool.frequencies.subarray(start, start + length), target);
|
|
3999
|
+
nextPool.fieldIndices.set(pool.fieldIndices.subarray(start, start + length), target);
|
|
4000
|
+
if ((terms.flags[ordinal] & TERM_POSITIONS) !== 0 && sourceStarts !== null && targetStarts !== null) {
|
|
4001
|
+
if (rebuildPositions) {
|
|
4002
|
+
for (let i = 0; i < length; i++)
|
|
4003
|
+
targetStarts[target + i] = nextPositions.appendFrom(positions, sourceStarts[start + i]);
|
|
4004
|
+
} else {
|
|
4005
|
+
targetStarts.set(sourceStarts.subarray(start, start + length), target);
|
|
4006
|
+
}
|
|
3207
4007
|
}
|
|
4008
|
+
terms.sliceStart[ordinal] = target;
|
|
4009
|
+
terms.sliceClass[ordinal] = sliceClass;
|
|
4010
|
+
}
|
|
4011
|
+
this.pool = nextPool;
|
|
4012
|
+
this.positions = nextPositions;
|
|
4013
|
+
this.compactedSlack = 0;
|
|
4014
|
+
}
|
|
4015
|
+
};
|
|
4016
|
+
|
|
4017
|
+
// src/core/inverted-index/index.ts
|
|
4018
|
+
function fieldIndexFor(fieldNameTable, fieldName) {
|
|
4019
|
+
let index = fieldNameTable.indexMap.get(fieldName);
|
|
4020
|
+
if (index === void 0) {
|
|
4021
|
+
index = fieldNameTable.names.length;
|
|
4022
|
+
fieldNameTable.names.push(fieldName);
|
|
4023
|
+
fieldNameTable.indexMap.set(fieldName, index);
|
|
4024
|
+
}
|
|
4025
|
+
return index;
|
|
4026
|
+
}
|
|
4027
|
+
function createInvertedIndex(fieldNameTable) {
|
|
4028
|
+
const postings = new LivePostings();
|
|
4029
|
+
function* liveTokens() {
|
|
4030
|
+
const terms = postings.terms;
|
|
4031
|
+
const generation = terms.generation;
|
|
4032
|
+
for (let ordinal = 0; ordinal < terms.count && terms.generation === generation; ordinal++) {
|
|
4033
|
+
if (terms.isLive(ordinal)) yield terms.tokens[ordinal];
|
|
4034
|
+
}
|
|
4035
|
+
}
|
|
4036
|
+
function serializeTerm(ordinal, resolver) {
|
|
4037
|
+
const list = postings.view(ordinal, false);
|
|
4038
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, length } = list;
|
|
4039
|
+
const positions = list.positions;
|
|
4040
|
+
const entries = [];
|
|
4041
|
+
for (let i = 0; i < length; i++) {
|
|
4042
|
+
if (deletedDocs.size > 0 && deletedDocs.has(docIds[i])) continue;
|
|
4043
|
+
const externalId = resolver.toExternal(docIds[i]);
|
|
4044
|
+
if (externalId === void 0) continue;
|
|
4045
|
+
entries.push({
|
|
4046
|
+
docId: externalId,
|
|
4047
|
+
termFrequency: termFrequencies[i],
|
|
4048
|
+
fieldName: fieldNameTable.names[fieldNameIndices[i]],
|
|
4049
|
+
positions: positions === null ? [] : [...positions[i]]
|
|
4050
|
+
});
|
|
4051
|
+
}
|
|
4052
|
+
return entries;
|
|
4053
|
+
}
|
|
4054
|
+
return {
|
|
4055
|
+
insert(token, internalId, termFrequency, fieldNameIndex2, positions) {
|
|
4056
|
+
postings.insert(token, internalId, termFrequency, fieldNameIndex2, positions);
|
|
4057
|
+
},
|
|
4058
|
+
remove(token, internalId, occurrences) {
|
|
4059
|
+
postings.remove(token, internalId, occurrences);
|
|
3208
4060
|
},
|
|
3209
4061
|
beginBatch() {
|
|
3210
|
-
|
|
3211
|
-
batchDirtyTokens.clear();
|
|
4062
|
+
postings.beginBatch();
|
|
3212
4063
|
},
|
|
3213
4064
|
endBatch() {
|
|
3214
|
-
|
|
3215
|
-
for (const token of batchDirtyTokens) {
|
|
3216
|
-
const list = index.get(token);
|
|
3217
|
-
if (!list) continue;
|
|
3218
|
-
if (list.deletedDocs.size / list.length > POSTING_LIST_COMPACTION_THRESHOLD) {
|
|
3219
|
-
compactList(list);
|
|
3220
|
-
}
|
|
3221
|
-
}
|
|
3222
|
-
batchDirtyTokens.clear();
|
|
4065
|
+
postings.endBatch();
|
|
3223
4066
|
},
|
|
3224
4067
|
lookup(token) {
|
|
3225
|
-
|
|
4068
|
+
const ordinal = postings.terms.ids.get(token);
|
|
4069
|
+
return ordinal === void 0 ? void 0 : postings.view(ordinal, true);
|
|
4070
|
+
},
|
|
4071
|
+
lookupReadOnce(token) {
|
|
4072
|
+
const ordinal = postings.terms.ids.get(token);
|
|
4073
|
+
return ordinal === void 0 ? void 0 : postings.view(ordinal, false);
|
|
3226
4074
|
},
|
|
3227
4075
|
documentFrequency(token) {
|
|
3228
|
-
|
|
4076
|
+
const ordinal = postings.terms.ids.get(token);
|
|
4077
|
+
return ordinal === void 0 ? 0 : postings.terms.docFrequency[ordinal];
|
|
3229
4078
|
},
|
|
3230
4079
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
4080
|
+
const terms = postings.terms;
|
|
3231
4081
|
if (tolerance === 0) {
|
|
3232
|
-
const exact =
|
|
3233
|
-
return exact ? [{ token, postingList: exact }]
|
|
4082
|
+
const exact = terms.ids.get(token);
|
|
4083
|
+
return exact === void 0 ? [] : [{ token, postingList: postings.view(exact, true) }];
|
|
3234
4084
|
}
|
|
3235
4085
|
const results = [];
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
const postingList = index.get(candidate);
|
|
3241
|
-
if (postingList) results.push({ token: candidate, postingList });
|
|
4086
|
+
terms.forEachStartingWith(fuzzyPrefixOf(token, prefixLength), (ordinal) => {
|
|
4087
|
+
const candidate = terms.tokens[ordinal];
|
|
4088
|
+
if (boundedLevenshtein(token, candidate, tolerance).withinTolerance) {
|
|
4089
|
+
results.push({ token: candidate, postingList: postings.view(ordinal, true) });
|
|
3242
4090
|
}
|
|
3243
|
-
}
|
|
4091
|
+
});
|
|
3244
4092
|
return results;
|
|
3245
4093
|
},
|
|
3246
4094
|
prefixSearch(prefix, limit) {
|
|
3247
4095
|
if (prefix.length === 0 || limit <= 0) return [];
|
|
3248
|
-
const
|
|
4096
|
+
const terms = postings.terms;
|
|
3249
4097
|
const results = [];
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
if (!list) continue;
|
|
3254
|
-
results.push({ term, documentFrequency: list.docIdSet.size });
|
|
3255
|
-
}
|
|
4098
|
+
terms.forEachStartingWith(prefix, (ordinal) => {
|
|
4099
|
+
results.push({ term: terms.tokens[ordinal], documentFrequency: terms.docFrequency[ordinal] });
|
|
4100
|
+
});
|
|
3256
4101
|
results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.term, b.term));
|
|
3257
4102
|
if (results.length > limit) results.length = limit;
|
|
3258
4103
|
return results;
|
|
3259
4104
|
},
|
|
3260
4105
|
has(token) {
|
|
3261
|
-
return
|
|
4106
|
+
return postings.terms.ids.has(token);
|
|
3262
4107
|
},
|
|
3263
4108
|
tokens() {
|
|
3264
|
-
return
|
|
4109
|
+
return liveTokens();
|
|
3265
4110
|
},
|
|
3266
4111
|
size() {
|
|
3267
|
-
return
|
|
4112
|
+
return postings.terms.liveCount;
|
|
3268
4113
|
},
|
|
3269
4114
|
clear() {
|
|
3270
|
-
|
|
3271
|
-
|
|
4115
|
+
postings.clear();
|
|
4116
|
+
},
|
|
4117
|
+
estimateBytes() {
|
|
4118
|
+
const terms = postings.terms;
|
|
4119
|
+
const perTerm = TERM_ENTRY_ESTIMATED_BYTES + TOKEN_STRING_HEADER_BYTES;
|
|
4120
|
+
return postings.byteLength() + terms.liveCount * perTerm + terms.tokenCodeUnits * 2;
|
|
3272
4121
|
},
|
|
3273
4122
|
serialize(resolver) {
|
|
3274
4123
|
const result = /* @__PURE__ */ Object.create(null);
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
postings.push({
|
|
3282
|
-
docId: externalId,
|
|
3283
|
-
termFrequency: list.termFrequencies[i],
|
|
3284
|
-
fieldName: fieldNameTable.names[list.fieldNameIndices[i]],
|
|
3285
|
-
positions: list.positions ? list.positions[i] : []
|
|
3286
|
-
});
|
|
3287
|
-
}
|
|
3288
|
-
if (postings.length > 0) {
|
|
3289
|
-
result[token] = { docFrequency: list.docIdSet.size, postings };
|
|
4124
|
+
const terms = postings.terms;
|
|
4125
|
+
for (let ordinal = 0; ordinal < terms.count; ordinal++) {
|
|
4126
|
+
if (!terms.isLive(ordinal)) continue;
|
|
4127
|
+
const entries = serializeTerm(ordinal, resolver);
|
|
4128
|
+
if (entries.length > 0) {
|
|
4129
|
+
result[terms.tokens[ordinal]] = { docFrequency: terms.docFrequency[ordinal], postings: entries };
|
|
3290
4130
|
}
|
|
3291
4131
|
}
|
|
3292
4132
|
return result;
|
|
3293
4133
|
},
|
|
3294
4134
|
deserialize(data, resolver) {
|
|
3295
|
-
|
|
3296
|
-
charBuckets.clear();
|
|
4135
|
+
postings.clear();
|
|
3297
4136
|
for (const token of Object.keys(data)) {
|
|
3298
|
-
const
|
|
3299
|
-
const count = src.postings.length;
|
|
3300
|
-
const docIdSet = /* @__PURE__ */ new Set();
|
|
3301
|
-
const docIds = new Array(count);
|
|
3302
|
-
const termFrequencies = new Uint16Array(count);
|
|
3303
|
-
const fieldNameIndices = new Uint16Array(count);
|
|
4137
|
+
const source = data[token].postings;
|
|
3304
4138
|
let hasPositions = false;
|
|
3305
|
-
let
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
|
|
3309
|
-
|
|
3310
|
-
if (internalId === void 0) continue;
|
|
3311
|
-
docIds[validCount] = internalId;
|
|
3312
|
-
termFrequencies[validCount] = p.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : p.termFrequency;
|
|
3313
|
-
totalTermFrequency += termFrequencies[validCount];
|
|
3314
|
-
let fnIndex = fieldNameTable.indexMap.get(p.fieldName);
|
|
3315
|
-
if (fnIndex === void 0) {
|
|
3316
|
-
fnIndex = fieldNameTable.names.length;
|
|
3317
|
-
fieldNameTable.names.push(p.fieldName);
|
|
3318
|
-
fieldNameTable.indexMap.set(p.fieldName, fnIndex);
|
|
3319
|
-
}
|
|
3320
|
-
fieldNameIndices[validCount] = fnIndex;
|
|
3321
|
-
docIdSet.add(internalId);
|
|
3322
|
-
if (p.positions.length > 0) hasPositions = true;
|
|
3323
|
-
validCount++;
|
|
4139
|
+
let valid = 0;
|
|
4140
|
+
for (const posting of source) {
|
|
4141
|
+
if (resolver.toInternal(posting.docId) === void 0) continue;
|
|
4142
|
+
valid++;
|
|
4143
|
+
if (posting.positions.length > 0) hasPositions = true;
|
|
3324
4144
|
}
|
|
3325
|
-
if (
|
|
3326
|
-
|
|
3327
|
-
const
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
for (let i = 0; i < count; i++) {
|
|
3334
|
-
const internalId = resolver.toInternal(src.postings[i].docId);
|
|
3335
|
-
if (internalId === void 0) continue;
|
|
3336
|
-
positions[vi] = src.postings[i].positions;
|
|
3337
|
-
vi++;
|
|
3338
|
-
}
|
|
3339
|
-
}
|
|
3340
|
-
let ordered = true;
|
|
3341
|
-
for (let i = 1; i < validCount; i++) {
|
|
3342
|
-
if (docIds[i] < docIds[i - 1]) {
|
|
3343
|
-
ordered = false;
|
|
3344
|
-
break;
|
|
3345
|
-
}
|
|
4145
|
+
if (valid === 0) continue;
|
|
4146
|
+
postings.reserve(token, valid);
|
|
4147
|
+
for (const posting of source) {
|
|
4148
|
+
const internalId = resolver.toInternal(posting.docId);
|
|
4149
|
+
if (internalId === void 0) continue;
|
|
4150
|
+
const termFrequency = posting.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : posting.termFrequency;
|
|
4151
|
+
const fieldIndex = fieldIndexFor(fieldNameTable, posting.fieldName);
|
|
4152
|
+
postings.insert(token, internalId, termFrequency, fieldIndex, hasPositions ? posting.positions : null);
|
|
3346
4153
|
}
|
|
3347
|
-
index.set(token, {
|
|
3348
|
-
length: validCount,
|
|
3349
|
-
docIds,
|
|
3350
|
-
termFrequencies: finalTF,
|
|
3351
|
-
fieldNameIndices: finalFNI,
|
|
3352
|
-
positions,
|
|
3353
|
-
docIdSet,
|
|
3354
|
-
deletedDocs: /* @__PURE__ */ new Set(),
|
|
3355
|
-
totalTermFrequency,
|
|
3356
|
-
structureRevision: 0,
|
|
3357
|
-
ordered
|
|
3358
|
-
});
|
|
3359
|
-
trackToken(token);
|
|
3360
4154
|
}
|
|
3361
4155
|
}
|
|
3362
4156
|
};
|
|
3363
4157
|
}
|
|
3364
4158
|
|
|
3365
|
-
// src/core/projection.ts
|
|
3366
|
-
function splitPaths(paths) {
|
|
3367
|
-
if (!paths) return [];
|
|
3368
|
-
const split = [];
|
|
3369
|
-
for (const path of paths) {
|
|
3370
|
-
const segments = path.split(".").filter((segment) => segment.length > 0);
|
|
3371
|
-
if (segments.length === 0) continue;
|
|
3372
|
-
if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) continue;
|
|
3373
|
-
split.push(segments);
|
|
3374
|
-
}
|
|
3375
|
-
return split;
|
|
3376
|
-
}
|
|
3377
|
-
function overlaps(a, b) {
|
|
3378
|
-
const shared2 = Math.min(a.length, b.length);
|
|
3379
|
-
for (let i = 0; i < shared2; i++) {
|
|
3380
|
-
if (a[i] !== b[i]) return false;
|
|
3381
|
-
}
|
|
3382
|
-
return true;
|
|
3383
|
-
}
|
|
3384
|
-
function covers(outer, inner) {
|
|
3385
|
-
if (outer.length > inner.length) return false;
|
|
3386
|
-
for (let i = 0; i < outer.length; i++) {
|
|
3387
|
-
if (outer[i] !== inner[i]) return false;
|
|
3388
|
-
}
|
|
3389
|
-
return true;
|
|
3390
|
-
}
|
|
3391
|
-
function readPath(source, path) {
|
|
3392
|
-
let current = source;
|
|
3393
|
-
for (const segment of path) {
|
|
3394
|
-
if (typeof current !== "object" || current === null || Array.isArray(current))
|
|
3395
|
-
return { found: false, value: void 0 };
|
|
3396
|
-
const record = current;
|
|
3397
|
-
if (!Object.hasOwn(record, segment)) return { found: false, value: void 0 };
|
|
3398
|
-
current = record[segment];
|
|
3399
|
-
}
|
|
3400
|
-
return { found: true, value: current };
|
|
3401
|
-
}
|
|
3402
|
-
function writePath(target, path, value) {
|
|
3403
|
-
let current = target;
|
|
3404
|
-
for (let i = 0; i < path.length - 1; i++) {
|
|
3405
|
-
const segment = path[i];
|
|
3406
|
-
const existing = current[segment];
|
|
3407
|
-
if (typeof existing === "object" && existing !== null && !Array.isArray(existing)) {
|
|
3408
|
-
current = existing;
|
|
3409
|
-
continue;
|
|
3410
|
-
}
|
|
3411
|
-
const created = {};
|
|
3412
|
-
current[segment] = created;
|
|
3413
|
-
current = created;
|
|
3414
|
-
}
|
|
3415
|
-
current[path[path.length - 1]] = value;
|
|
3416
|
-
}
|
|
3417
|
-
function cloneExcept(value, path, exclude) {
|
|
3418
|
-
const reachesDeeper = exclude.some((excluded) => excluded.length > path.length && covers(path, excluded));
|
|
3419
|
-
if (!reachesDeeper) return structuredClone(value);
|
|
3420
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return structuredClone(value);
|
|
3421
|
-
const source = value;
|
|
3422
|
-
const kept = {};
|
|
3423
|
-
for (const key of Object.keys(source)) {
|
|
3424
|
-
const childPath = [...path, key];
|
|
3425
|
-
if (exclude.some((excluded) => covers(excluded, childPath))) continue;
|
|
3426
|
-
kept[key] = cloneExcept(source[key], childPath, exclude);
|
|
3427
|
-
}
|
|
3428
|
-
return kept;
|
|
3429
|
-
}
|
|
3430
|
-
function resolveProjection(projection) {
|
|
3431
|
-
if (projection === void 0 || projection === true) return { kind: "full" };
|
|
3432
|
-
if (projection === false) return { kind: "none" };
|
|
3433
|
-
if (typeof projection !== "object" || projection === null) return { kind: "full" };
|
|
3434
|
-
const askedInclude = Array.isArray(projection.include) && projection.include.length > 0;
|
|
3435
|
-
const askedExclude = Array.isArray(projection.exclude) && projection.exclude.length > 0;
|
|
3436
|
-
if (!askedInclude && !askedExclude) return { kind: "full" };
|
|
3437
|
-
return {
|
|
3438
|
-
kind: "fields",
|
|
3439
|
-
include: askedInclude ? splitPaths(projection.include) : null,
|
|
3440
|
-
exclude: splitPaths(projection.exclude)
|
|
3441
|
-
};
|
|
3442
|
-
}
|
|
3443
|
-
function projectionKeepsField(projection, fieldPath) {
|
|
3444
|
-
if (projection.kind === "full") return true;
|
|
3445
|
-
if (projection.kind === "none") return false;
|
|
3446
|
-
const path = fieldPath.split(".").filter((segment) => segment.length > 0);
|
|
3447
|
-
if (path.length === 0) return false;
|
|
3448
|
-
if (projection.include && !projection.include.some((included) => overlaps(included, path))) return false;
|
|
3449
|
-
return !projection.exclude.some((excluded) => covers(excluded, path));
|
|
3450
|
-
}
|
|
3451
|
-
function cloneProjected(document, projection) {
|
|
3452
|
-
if (projection === void 0 || projection.kind === "full") return structuredClone(document);
|
|
3453
|
-
if (projection.kind === "none") return {};
|
|
3454
|
-
if (projection.include === null) return cloneExcept(document, [], projection.exclude);
|
|
3455
|
-
const projected = {};
|
|
3456
|
-
for (const path of projection.include) {
|
|
3457
|
-
if (projection.exclude.some((excluded) => covers(excluded, path))) continue;
|
|
3458
|
-
const { found, value } = readPath(document, path);
|
|
3459
|
-
if (!found) continue;
|
|
3460
|
-
writePath(projected, path, cloneExcept(value, path, projection.exclude));
|
|
3461
|
-
}
|
|
3462
|
-
return projected;
|
|
3463
|
-
}
|
|
3464
|
-
|
|
3465
4159
|
// src/core/statistics.ts
|
|
3466
4160
|
function applyUniqueTokens(tokens, docFrequencies, increment) {
|
|
3467
4161
|
const fields = Object.keys(tokens);
|
|
@@ -3625,6 +4319,9 @@ function createSurfaceRegistry() {
|
|
|
3625
4319
|
else totalsByToken.set(token, next);
|
|
3626
4320
|
}
|
|
3627
4321
|
return {
|
|
4322
|
+
holds(surface) {
|
|
4323
|
+
return entries.has(surface);
|
|
4324
|
+
},
|
|
3628
4325
|
add(surface, token, occurrences) {
|
|
3629
4326
|
if (surface.length === 0 || token.length === 0 || occurrences <= 0) return;
|
|
3630
4327
|
if (surface === token) return;
|
|
@@ -3747,78 +4444,6 @@ function bitsetFromSet(set, capacity) {
|
|
|
3747
4444
|
return bs;
|
|
3748
4445
|
}
|
|
3749
4446
|
|
|
3750
|
-
// src/core/partition/utils.ts
|
|
3751
|
-
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
3752
|
-
const existing = table.indexMap.get(fieldName);
|
|
3753
|
-
if (existing !== void 0) return existing;
|
|
3754
|
-
const idx = table.names.length;
|
|
3755
|
-
table.names.push(fieldName);
|
|
3756
|
-
table.indexMap.set(fieldName, idx);
|
|
3757
|
-
return idx;
|
|
3758
|
-
}
|
|
3759
|
-
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
3760
|
-
function getNestedValue(obj, path) {
|
|
3761
|
-
const parts = path.split(".");
|
|
3762
|
-
let current = obj;
|
|
3763
|
-
for (let i = 0; i < parts.length; i++) {
|
|
3764
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
3765
|
-
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
3766
|
-
current = current[parts[i]];
|
|
3767
|
-
}
|
|
3768
|
-
return current;
|
|
3769
|
-
}
|
|
3770
|
-
function stringArraysEqual(a, b) {
|
|
3771
|
-
if (a === b) return true;
|
|
3772
|
-
if (!a || !b) return false;
|
|
3773
|
-
if (a.length !== b.length) return false;
|
|
3774
|
-
for (let i = 0; i < a.length; i++) {
|
|
3775
|
-
if (a[i] !== b[i]) return false;
|
|
3776
|
-
}
|
|
3777
|
-
return true;
|
|
3778
|
-
}
|
|
3779
|
-
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
3780
|
-
for (const { path, base } of fields) {
|
|
3781
|
-
if (!isWordIndexedBase(base)) continue;
|
|
3782
|
-
const oldVal = getNestedValue(oldDoc, path);
|
|
3783
|
-
const newVal = getNestedValue(newDoc, path);
|
|
3784
|
-
if (base === "string") {
|
|
3785
|
-
if (oldVal !== newVal) return true;
|
|
3786
|
-
} else {
|
|
3787
|
-
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
3788
|
-
}
|
|
3789
|
-
}
|
|
3790
|
-
return false;
|
|
3791
|
-
}
|
|
3792
|
-
function parseVectorDimension(fieldType) {
|
|
3793
|
-
const match = VECTOR_PATTERN2.exec(fieldType);
|
|
3794
|
-
return match ? Number.parseInt(match[1], 10) : null;
|
|
3795
|
-
}
|
|
3796
|
-
function tokenizeOptions(options) {
|
|
3797
|
-
return {
|
|
3798
|
-
stem: true,
|
|
3799
|
-
removeStopWords: true,
|
|
3800
|
-
collectSurfaces: options?.collectSurfaces === true,
|
|
3801
|
-
stopWordOverride: options?.stopWordOverride,
|
|
3802
|
-
customTokenizer: options?.customTokenizer
|
|
3803
|
-
};
|
|
3804
|
-
}
|
|
3805
|
-
function getFlatSchema(state, schema) {
|
|
3806
|
-
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
3807
|
-
state.flatSchemaCache = flattenSchema(schema);
|
|
3808
|
-
state.lastSchemaRef = schema;
|
|
3809
|
-
return state.flatSchemaCache;
|
|
3810
|
-
}
|
|
3811
|
-
function getFieldValueForDoc(docStore, docId, fieldPath) {
|
|
3812
|
-
const stored = docStore.get(docId);
|
|
3813
|
-
if (!stored) return void 0;
|
|
3814
|
-
return getNestedValue(stored.fields, fieldPath);
|
|
3815
|
-
}
|
|
3816
|
-
function getFieldValueByInternalId(docStore, internalId, fieldPath) {
|
|
3817
|
-
const externalId = docStore.getExternalId(internalId);
|
|
3818
|
-
if (externalId === void 0) return void 0;
|
|
3819
|
-
return getFieldValueForDoc(docStore, externalId, fieldPath);
|
|
3820
|
-
}
|
|
3821
|
-
|
|
3822
4447
|
// src/core/partition/facets.ts
|
|
3823
4448
|
var UNDECLARED_FIELD_TYPE = "";
|
|
3824
4449
|
function keptFacetValues(limit) {
|
|
@@ -3834,6 +4459,7 @@ function ordinalBitsetOf(state, matched) {
|
|
|
3834
4459
|
return bitset;
|
|
3835
4460
|
}
|
|
3836
4461
|
function* matchedFieldValues(state, fieldPath, matched) {
|
|
4462
|
+
const read = state.docStore.fieldReader(fieldPath);
|
|
3837
4463
|
if ("ordinalBitset" in matched) {
|
|
3838
4464
|
const bits = matched.ordinalBitset;
|
|
3839
4465
|
for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
|
|
@@ -3842,14 +4468,15 @@ function* matchedFieldValues(state, fieldPath, matched) {
|
|
|
3842
4468
|
const base = wordIndex << 5;
|
|
3843
4469
|
while (word !== 0) {
|
|
3844
4470
|
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
3845
|
-
yield
|
|
4471
|
+
yield read(base + trailingZeros);
|
|
3846
4472
|
word &= word - 1;
|
|
3847
4473
|
}
|
|
3848
4474
|
}
|
|
3849
4475
|
return;
|
|
3850
4476
|
}
|
|
3851
4477
|
for (const docId of matched) {
|
|
3852
|
-
|
|
4478
|
+
const internalId = state.docStore.getInternalId(docId);
|
|
4479
|
+
yield internalId === void 0 ? void 0 : read(internalId);
|
|
3853
4480
|
}
|
|
3854
4481
|
}
|
|
3855
4482
|
function countValuesFromDocuments(state, fieldPath, matched) {
|
|
@@ -6071,7 +6698,7 @@ var BYTES_PER_BOOLEAN_SLOT = 1;
|
|
|
6071
6698
|
var BYTES_PER_MIXED_SLOT = 8;
|
|
6072
6699
|
var BYTES_PER_STRING_CODE_UNIT = 2;
|
|
6073
6700
|
var STRING_HEADER_BYTES2 = 24;
|
|
6074
|
-
function
|
|
6701
|
+
function grow2(length, wanted) {
|
|
6075
6702
|
let next = length === 0 ? 64 : length;
|
|
6076
6703
|
while (next <= wanted) next *= 2;
|
|
6077
6704
|
return next;
|
|
@@ -6106,7 +6733,7 @@ function createValueStore(initialKind) {
|
|
|
6106
6733
|
function setNumber(internalId, value) {
|
|
6107
6734
|
if (numbers === null) return;
|
|
6108
6735
|
if (internalId >= numbers.length) {
|
|
6109
|
-
const grown = new Float64Array(
|
|
6736
|
+
const grown = new Float64Array(grow2(numbers.length, internalId));
|
|
6110
6737
|
grown.fill(Number.NaN, numbers.length);
|
|
6111
6738
|
grown.set(numbers);
|
|
6112
6739
|
numbers = grown;
|
|
@@ -6116,7 +6743,7 @@ function createValueStore(initialKind) {
|
|
|
6116
6743
|
function setBoolean(internalId, value) {
|
|
6117
6744
|
if (booleans === null) return;
|
|
6118
6745
|
if (internalId >= booleans.length) {
|
|
6119
|
-
const grown = new Uint8Array(
|
|
6746
|
+
const grown = new Uint8Array(grow2(booleans.length, internalId));
|
|
6120
6747
|
grown.fill(BOOLEAN_MISSING, booleans.length);
|
|
6121
6748
|
grown.set(booleans);
|
|
6122
6749
|
booleans = grown;
|
|
@@ -6227,10 +6854,9 @@ function createSortColumnSet(docStore) {
|
|
|
6227
6854
|
}
|
|
6228
6855
|
function backfill(key, field, fieldType, mode) {
|
|
6229
6856
|
const store = createValueStore(kindForFieldType(fieldType));
|
|
6230
|
-
|
|
6231
|
-
|
|
6232
|
-
|
|
6233
|
-
store.set(internalId, toReducedSortValue(readSortField(stored.fields, field), mode));
|
|
6857
|
+
const read = docStore.fieldReader(field);
|
|
6858
|
+
for (const internalId of docStore.allInternalIds()) {
|
|
6859
|
+
store.set(internalId, toReducedSortValue(read(internalId), mode));
|
|
6234
6860
|
}
|
|
6235
6861
|
const entry = {
|
|
6236
6862
|
field,
|
|
@@ -6562,15 +7188,18 @@ function sortedPageOf(state, request) {
|
|
|
6562
7188
|
function sortValuesOf(state, docId, fields, fieldTypes, modes) {
|
|
6563
7189
|
const set = state.sortColumns;
|
|
6564
7190
|
const internalId = state.docStore.getInternalId(docId);
|
|
6565
|
-
const stored = state.docStore.get(docId);
|
|
6566
7191
|
const key = new Array(fields.length);
|
|
6567
7192
|
for (let i = 0; i < fields.length; i++) {
|
|
6568
7193
|
const mode = modes[i] ?? "min";
|
|
6569
|
-
if (
|
|
7194
|
+
if (internalId === void 0) {
|
|
7195
|
+
key[i] = null;
|
|
7196
|
+
continue;
|
|
7197
|
+
}
|
|
7198
|
+
if (set?.holds(fields[i], fieldTypes[i], mode)) {
|
|
6570
7199
|
key[i] = set.column(fields[i], fieldTypes[i], mode).valueOf(internalId);
|
|
6571
7200
|
continue;
|
|
6572
7201
|
}
|
|
6573
|
-
key[i] =
|
|
7202
|
+
key[i] = toReducedSortValue(state.docStore.fieldReader(fields[i])(internalId), mode);
|
|
6574
7203
|
}
|
|
6575
7204
|
return key;
|
|
6576
7205
|
}
|
|
@@ -6613,7 +7242,15 @@ function buildFilterContext(state, schema, meter) {
|
|
|
6613
7242
|
for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
|
|
6614
7243
|
if (pattern) patternFields.add(fieldPath);
|
|
6615
7244
|
}
|
|
6616
|
-
const
|
|
7245
|
+
const fieldReaders = /* @__PURE__ */ new Map();
|
|
7246
|
+
const readerOf = (fieldPath) => {
|
|
7247
|
+
let read = fieldReaders.get(fieldPath);
|
|
7248
|
+
if (read === void 0) {
|
|
7249
|
+
read = state.docStore.fieldReader(fieldPath);
|
|
7250
|
+
fieldReaders.set(fieldPath, read);
|
|
7251
|
+
}
|
|
7252
|
+
return read;
|
|
7253
|
+
};
|
|
6617
7254
|
let cachedAllDocIdsBitset = null;
|
|
6618
7255
|
const cachedPresentBitsets = /* @__PURE__ */ new Map();
|
|
6619
7256
|
return {
|
|
@@ -6637,7 +7274,7 @@ function buildFilterContext(state, schema, meter) {
|
|
|
6637
7274
|
matches: into,
|
|
6638
7275
|
meter,
|
|
6639
7276
|
host: PATTERN_HOST,
|
|
6640
|
-
valueOf: (
|
|
7277
|
+
valueOf: readerOf(fieldPath)
|
|
6641
7278
|
});
|
|
6642
7279
|
},
|
|
6643
7280
|
presentBitset: (fieldPath) => {
|
|
@@ -6996,38 +7633,18 @@ function searchFulltextMatches(state, params) {
|
|
|
6996
7633
|
// src/core/partition/memory.ts
|
|
6997
7634
|
var DOC_ENTRY_OVERHEAD_BYTES = 260;
|
|
6998
7635
|
var LENGTH_COLUMN_SLOT_BYTES = 6;
|
|
6999
|
-
var
|
|
7000
|
-
var POSITIONS_ARRAY_OVERHEAD_BYTES = 88;
|
|
7001
|
-
var POSITION_BYTES = 8;
|
|
7002
|
-
var PER_TERM_OVERHEAD_BYTES = 180;
|
|
7636
|
+
var DOC_FREQUENCY_ENTRY_BYTES = 44;
|
|
7003
7637
|
var SURFACE_ENTRY_OVERHEAD_BYTES = 140;
|
|
7004
7638
|
var FIELD_ENTRY_OVERHEAD_BYTES = 42;
|
|
7005
7639
|
function estimatePartitionBytes(state) {
|
|
7006
7640
|
const docCount = state.docStore.count();
|
|
7007
7641
|
if (docCount === 0) return 0;
|
|
7008
|
-
const
|
|
7009
|
-
let totalPostings = 0;
|
|
7010
|
-
let termCount = 0;
|
|
7011
|
-
for (const term in docFreqs) {
|
|
7012
|
-
totalPostings += docFreqs[term];
|
|
7013
|
-
termCount++;
|
|
7014
|
-
}
|
|
7015
|
-
const fieldLengthTotals = state.stats.totalFieldLengths;
|
|
7016
|
-
let totalTokenOccurrences = 0;
|
|
7017
|
-
let textFieldCount = 0;
|
|
7018
|
-
for (const field in fieldLengthTotals) {
|
|
7019
|
-
totalTokenOccurrences += fieldLengthTotals[field];
|
|
7020
|
-
textFieldCount++;
|
|
7021
|
-
}
|
|
7642
|
+
const textFieldCount = Object.keys(state.stats.totalFieldLengths).length;
|
|
7022
7643
|
let bytes = state.docStore.contentBytes();
|
|
7023
7644
|
bytes += docCount * DOC_ENTRY_OVERHEAD_BYTES;
|
|
7024
7645
|
bytes += textFieldCount * state.docStore.internalIdCapacity() * LENGTH_COLUMN_SLOT_BYTES;
|
|
7025
|
-
bytes +=
|
|
7026
|
-
|
|
7027
|
-
bytes += totalPostings * POSITIONS_ARRAY_OVERHEAD_BYTES;
|
|
7028
|
-
bytes += totalTokenOccurrences * POSITION_BYTES;
|
|
7029
|
-
}
|
|
7030
|
-
bytes += termCount * PER_TERM_OVERHEAD_BYTES;
|
|
7646
|
+
bytes += state.invertedIdx.estimateBytes();
|
|
7647
|
+
bytes += state.invertedIdx.size() * DOC_FREQUENCY_ENTRY_BYTES;
|
|
7031
7648
|
bytes += state.surfaceRegistry.size() * SURFACE_ENTRY_OVERHEAD_BYTES;
|
|
7032
7649
|
bytes += docCount * state.numericIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7033
7650
|
bytes += docCount * state.booleanIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
@@ -7063,17 +7680,18 @@ function mergeDocuments(target, source) {
|
|
|
7063
7680
|
}
|
|
7064
7681
|
function mergePostings(target, source, idMapping) {
|
|
7065
7682
|
for (const token of source.invertedIdx.tokens()) {
|
|
7066
|
-
const list = source.invertedIdx.lookup(token);
|
|
7683
|
+
const list = source.invertedIdx.lookupReadOnce?.(token) ?? source.invertedIdx.lookup(token);
|
|
7067
7684
|
if (list === void 0) continue;
|
|
7068
|
-
|
|
7069
|
-
|
|
7070
|
-
|
|
7685
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, positions, length } = list;
|
|
7686
|
+
for (let i = 0; i < length; i++) {
|
|
7687
|
+
const sourceInternalId = docIds[i];
|
|
7688
|
+
if (deletedDocs.has(sourceInternalId)) continue;
|
|
7071
7689
|
const targetInternalId = idMapping.get(sourceInternalId);
|
|
7072
7690
|
if (targetInternalId === void 0) continue;
|
|
7073
|
-
const fieldName = source.fieldNameTable.names[
|
|
7691
|
+
const fieldName = source.fieldNameTable.names[fieldNameIndices[i]];
|
|
7074
7692
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(target.fieldNameTable, fieldName);
|
|
7075
|
-
const
|
|
7076
|
-
target.invertedIdx.insert(token, targetInternalId,
|
|
7693
|
+
const entryPositions = positions === null ? null : positions[i];
|
|
7694
|
+
target.invertedIdx.insert(token, targetInternalId, termFrequencies[i], fieldNameIndex2, entryPositions ?? null);
|
|
7077
7695
|
}
|
|
7078
7696
|
}
|
|
7079
7697
|
}
|
|
@@ -7185,8 +7803,9 @@ function entriesCovered(bounds) {
|
|
|
7185
7803
|
return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
|
|
7186
7804
|
}
|
|
7187
7805
|
function scan(list, columns, previous, keepBlocks) {
|
|
7806
|
+
const { docIds, termFrequencies, fieldNameIndices, length } = list;
|
|
7188
7807
|
const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
|
|
7189
|
-
const capacity = keepBlocks + Math.ceil((
|
|
7808
|
+
const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
|
|
7190
7809
|
const entryEnd = new Int32Array(capacity);
|
|
7191
7810
|
const maxTermFrequency = new Int32Array(capacity);
|
|
7192
7811
|
const minFieldLength = new Float64Array(capacity);
|
|
@@ -7208,16 +7827,16 @@ function scan(list, columns, previous, keepBlocks) {
|
|
|
7208
7827
|
let blockMaxRun = 0;
|
|
7209
7828
|
let blockDocs = 0;
|
|
7210
7829
|
let entry = startEntry;
|
|
7211
|
-
while (entry <
|
|
7212
|
-
const docId =
|
|
7830
|
+
while (entry < length) {
|
|
7831
|
+
const docId = docIds[entry];
|
|
7213
7832
|
const tombstoned = hasDeleted && deleted.has(docId);
|
|
7214
7833
|
let run = 0;
|
|
7215
|
-
while (entry + run <
|
|
7834
|
+
while (entry + run < length && docIds[entry + run] === docId) {
|
|
7216
7835
|
if (!tombstoned) {
|
|
7217
7836
|
const position = entry + run;
|
|
7218
|
-
const termFrequency =
|
|
7837
|
+
const termFrequency = termFrequencies[position];
|
|
7219
7838
|
if (termFrequency > blockMaxTf) blockMaxTf = termFrequency;
|
|
7220
|
-
const fieldLength = fieldLengthBound(columns,
|
|
7839
|
+
const fieldLength = fieldLengthBound(columns, fieldNameIndices[position], docId);
|
|
7221
7840
|
if (fieldLength < blockMinLen) blockMinLen = fieldLength;
|
|
7222
7841
|
}
|
|
7223
7842
|
run++;
|
|
@@ -7227,7 +7846,7 @@ function scan(list, columns, previous, keepBlocks) {
|
|
|
7227
7846
|
blockDocs++;
|
|
7228
7847
|
}
|
|
7229
7848
|
entry += run;
|
|
7230
|
-
if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry ===
|
|
7849
|
+
if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === length) {
|
|
7231
7850
|
entryEnd[blockIndex] = entry;
|
|
7232
7851
|
maxTermFrequency[blockIndex] = blockMaxTf;
|
|
7233
7852
|
minFieldLength[blockIndex] = blockMinLen === Number.POSITIVE_INFINITY ? ABSENT_FIELD_LENGTH_BOUND : blockMinLen;
|
|
@@ -7984,10 +8603,11 @@ function countPostings(state, remap) {
|
|
|
7984
8603
|
const list = lookupReadOnce(state, token);
|
|
7985
8604
|
if (list === void 0) continue;
|
|
7986
8605
|
const listPositions = list.positionCountAt === void 0 ? list.positions : null;
|
|
7987
|
-
const tracksPositions = list.positionCountAt === void 0 ? listPositions !== null : state.trackPositions;
|
|
7988
|
-
|
|
7989
|
-
|
|
7990
|
-
|
|
8606
|
+
const tracksPositions = list.holdsPositions ?? (list.positionCountAt === void 0 ? listPositions !== null : state.trackPositions);
|
|
8607
|
+
const { docIds, deletedDocs, length } = list;
|
|
8608
|
+
for (let i = 0; i < length; i++) {
|
|
8609
|
+
const internalId = docIds[i];
|
|
8610
|
+
if (deletedDocs.has(internalId) || remap[internalId] < 0) continue;
|
|
7991
8611
|
postings++;
|
|
7992
8612
|
if (!tracksPositions) continue;
|
|
7993
8613
|
hasPositions = true;
|
|
@@ -8038,18 +8658,20 @@ function encodeSegmentState(state) {
|
|
|
8038
8658
|
for (const token of state.invertedIdx.tokens()) {
|
|
8039
8659
|
const list = lookupReadOnce(state, token);
|
|
8040
8660
|
if (list === void 0) continue;
|
|
8041
|
-
const
|
|
8661
|
+
const copyPositionsAt = list.copyPositionsAt?.bind(list);
|
|
8662
|
+
const listPositions = positionOffsets === null || copyPositionsAt !== void 0 ? null : list.positions;
|
|
8042
8663
|
const tokenStart = postingCursor;
|
|
8043
8664
|
const countViaSet = !list.ordered;
|
|
8044
8665
|
if (countViaSet) unorderedTokenDocs.clear();
|
|
8045
8666
|
let uniqueDocs = 0;
|
|
8046
8667
|
let lastCountedDoc = -1;
|
|
8047
|
-
|
|
8048
|
-
|
|
8049
|
-
|
|
8668
|
+
const { docIds: docIds2, termFrequencies, fieldNameIndices, deletedDocs, length } = list;
|
|
8669
|
+
for (let i = 0; i < length; i++) {
|
|
8670
|
+
const internalId = docIds2[i];
|
|
8671
|
+
if (deletedDocs.has(internalId) || remap[internalId] < 0) continue;
|
|
8050
8672
|
postingDocIds[postingCursor] = remap[internalId];
|
|
8051
|
-
postingFrequencies[postingCursor] =
|
|
8052
|
-
postingFieldIndices[postingCursor] =
|
|
8673
|
+
postingFrequencies[postingCursor] = termFrequencies[i];
|
|
8674
|
+
postingFieldIndices[postingCursor] = fieldNameIndices[i];
|
|
8053
8675
|
if (countViaSet) {
|
|
8054
8676
|
unorderedTokenDocs.add(internalId);
|
|
8055
8677
|
} else if (internalId !== lastCountedDoc) {
|
|
@@ -8059,7 +8681,9 @@ function encodeSegmentState(state) {
|
|
|
8059
8681
|
if (positionOffsets !== null && positionValues !== null) {
|
|
8060
8682
|
positionOffsets[postingCursor] = positionCursor;
|
|
8061
8683
|
const entryPositions = listPositions === null ? null : listPositions[i];
|
|
8062
|
-
if (
|
|
8684
|
+
if (copyPositionsAt !== void 0) {
|
|
8685
|
+
positionCursor += copyPositionsAt(i, positionValues, positionCursor);
|
|
8686
|
+
} else if (entryPositions !== null && entryPositions !== void 0) {
|
|
8063
8687
|
for (const position of entryPositions) positionValues[positionCursor++] = position;
|
|
8064
8688
|
}
|
|
8065
8689
|
}
|
|
@@ -8269,17 +8893,18 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
|
|
|
8269
8893
|
for (const token of state.invertedIdx.tokens()) {
|
|
8270
8894
|
const list = state.invertedIdx.lookup(token);
|
|
8271
8895
|
if (!list) continue;
|
|
8272
|
-
const
|
|
8896
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, positions, length } = list;
|
|
8897
|
+
const hasDeleted = deletedDocs.size > 0;
|
|
8273
8898
|
const postings = [];
|
|
8274
|
-
for (let i = 0; i <
|
|
8275
|
-
if (hasDeleted &&
|
|
8276
|
-
const externalId = resolver.toExternal(
|
|
8899
|
+
for (let i = 0; i < length; i++) {
|
|
8900
|
+
if (hasDeleted && deletedDocs.has(docIds[i])) continue;
|
|
8901
|
+
const externalId = resolver.toExternal(docIds[i]);
|
|
8277
8902
|
if (externalId === void 0) continue;
|
|
8278
8903
|
postings.push({
|
|
8279
8904
|
docId: externalId,
|
|
8280
|
-
termFrequency:
|
|
8281
|
-
field: fieldNames[
|
|
8282
|
-
positions:
|
|
8905
|
+
termFrequency: termFrequencies[i],
|
|
8906
|
+
field: fieldNames[fieldNameIndices[i]],
|
|
8907
|
+
positions: positions === null ? [] : Array.from(positions[i] ?? [])
|
|
8283
8908
|
});
|
|
8284
8909
|
}
|
|
8285
8910
|
if (postings.length > 0) {
|
|
@@ -8674,6 +9299,11 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8674
9299
|
if (!stored) return void 0;
|
|
8675
9300
|
return stored.fields;
|
|
8676
9301
|
},
|
|
9302
|
+
fieldValues(docId, fieldPaths) {
|
|
9303
|
+
const stored = state.docStore.get(docId);
|
|
9304
|
+
if (!stored) return void 0;
|
|
9305
|
+
return fieldPaths.map((fieldPath) => getNestedValue(stored.fields, fieldPath));
|
|
9306
|
+
},
|
|
8677
9307
|
has(docId) {
|
|
8678
9308
|
return state.docStore.has(docId);
|
|
8679
9309
|
},
|
|
@@ -10177,7 +10807,7 @@ function frozenSegmentBytes(segment) {
|
|
|
10177
10807
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
10178
10808
|
}
|
|
10179
10809
|
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
10180
|
-
return bytes + segment.partialWords.bytes();
|
|
10810
|
+
return bytes + segment.partialWords.bytes() + segment.removalBytes();
|
|
10181
10811
|
}
|
|
10182
10812
|
|
|
10183
10813
|
// src/core/partition/composite/compaction.ts
|
|
@@ -10215,19 +10845,40 @@ function freezeLiveTailInto(live, liveState, frozen, freeze) {
|
|
|
10215
10845
|
frozen.push(segment);
|
|
10216
10846
|
return segment;
|
|
10217
10847
|
}
|
|
10218
|
-
function swapFrozenSegmentList(frozen, dropSegmentIds, replacement) {
|
|
10848
|
+
function swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings) {
|
|
10219
10849
|
const dropSet = new Set(dropSegmentIds);
|
|
10220
10850
|
const dropped = frozen.filter((segment) => dropSet.has(segment.segmentId));
|
|
10851
|
+
const applied = /* @__PURE__ */ new Map();
|
|
10221
10852
|
for (const segment of dropped) {
|
|
10222
10853
|
for (const docId of segment.tombstonedDocIds()) {
|
|
10223
10854
|
if (!replacement.hasDocument(docId)) continue;
|
|
10224
10855
|
if (dropped.some((other) => other.hasDocument(docId))) continue;
|
|
10225
|
-
replacement.
|
|
10856
|
+
const spellings = removedSpellings?.get(docId) ?? replacement.removedSpellingsOf(docId);
|
|
10857
|
+
if (spellings !== void 0) applied.set(docId, spellings);
|
|
10858
|
+
replacement.tombstoneDocument(docId, spellings);
|
|
10226
10859
|
}
|
|
10227
10860
|
}
|
|
10228
10861
|
const kept = frozen.filter((segment) => !dropSet.has(segment.segmentId));
|
|
10229
10862
|
frozen.length = 0;
|
|
10230
10863
|
frozen.push(...kept, replacement);
|
|
10864
|
+
return applied;
|
|
10865
|
+
}
|
|
10866
|
+
|
|
10867
|
+
// src/core/partition/composite/documents.ts
|
|
10868
|
+
function compositeDocument(frozen, live, docId, projection) {
|
|
10869
|
+
for (const segment of frozen) {
|
|
10870
|
+
const projected = segment.docStore.projectedDocument(docId, projection);
|
|
10871
|
+
if (projected !== void 0) return projected;
|
|
10872
|
+
}
|
|
10873
|
+
return live.get(docId, projection);
|
|
10874
|
+
}
|
|
10875
|
+
function compositeFieldValues(frozen, live, docId, fieldPaths) {
|
|
10876
|
+
for (const segment of frozen) {
|
|
10877
|
+
const internalId = segment.docStore.getInternalId(docId);
|
|
10878
|
+
if (internalId === void 0) continue;
|
|
10879
|
+
return fieldPaths.map((fieldPath) => segment.docStore.fieldReader(fieldPath)(internalId));
|
|
10880
|
+
}
|
|
10881
|
+
return live.fieldValues(docId, fieldPaths);
|
|
10231
10882
|
}
|
|
10232
10883
|
|
|
10233
10884
|
// src/core/partition/composite/filters.ts
|
|
@@ -10553,7 +11204,7 @@ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
|
10553
11204
|
function isCompositePartition(partition) {
|
|
10554
11205
|
return "attachFrozenSegment" in partition;
|
|
10555
11206
|
}
|
|
10556
|
-
function createCompositePartition(partitionId, trackPositions = true, existingLive) {
|
|
11207
|
+
function createCompositePartition(partitionId, trackPositions = true, existingLive, textAnalysis) {
|
|
10557
11208
|
const live = existingLive ?? createPartitionIndex(partitionId, trackPositions);
|
|
10558
11209
|
const liveState = partitionStateOf(live);
|
|
10559
11210
|
const frozen = [];
|
|
@@ -10561,6 +11212,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10561
11212
|
function invalidateDocFrequencies() {
|
|
10562
11213
|
docFrequenciesCache = null;
|
|
10563
11214
|
}
|
|
11215
|
+
function bindSegments() {
|
|
11216
|
+
invalidateDocFrequencies();
|
|
11217
|
+
if (textAnalysis === void 0) return;
|
|
11218
|
+
for (const segment of frozen) segment.removals.bind(textAnalysis);
|
|
11219
|
+
}
|
|
10564
11220
|
function cachedDocFrequencies() {
|
|
10565
11221
|
if (docFrequenciesCache === null) {
|
|
10566
11222
|
docFrequenciesCache = mergeDocFrequencies(subs());
|
|
@@ -10614,23 +11270,25 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10614
11270
|
frozenSegmentsById(segmentIds) {
|
|
10615
11271
|
return resolveFrozenSegments(frozen, segmentIds, partitionId);
|
|
10616
11272
|
},
|
|
10617
|
-
swapFrozenSegments(dropSegmentIds, replacement) {
|
|
10618
|
-
|
|
10619
|
-
|
|
11273
|
+
swapFrozenSegments(dropSegmentIds, replacement, removedSpellings) {
|
|
11274
|
+
if (textAnalysis !== void 0) replacement.removals.bind(textAnalysis);
|
|
11275
|
+
const applied = swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings);
|
|
11276
|
+
bindSegments();
|
|
11277
|
+
return applied;
|
|
10620
11278
|
},
|
|
10621
11279
|
freezeLiveTail(freeze) {
|
|
10622
11280
|
const segment = freezeLiveTailInto(live, liveState, frozen, freeze);
|
|
10623
|
-
if (segment !== null)
|
|
11281
|
+
if (segment !== null) bindSegments();
|
|
10624
11282
|
return segment;
|
|
10625
11283
|
},
|
|
10626
11284
|
replaceLiveTail(segment) {
|
|
10627
11285
|
live.clear();
|
|
10628
11286
|
frozen.push(segment);
|
|
10629
|
-
|
|
11287
|
+
bindSegments();
|
|
10630
11288
|
},
|
|
10631
11289
|
attachFrozenSegment(segment) {
|
|
10632
11290
|
frozen.push(segment);
|
|
10633
|
-
|
|
11291
|
+
bindSegments();
|
|
10634
11292
|
},
|
|
10635
11293
|
mergeSegment(segment) {
|
|
10636
11294
|
invalidateDocFrequencies();
|
|
@@ -10669,7 +11327,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10669
11327
|
invalidateDocFrequencies();
|
|
10670
11328
|
live.insert(docId, document, schema, language, options);
|
|
10671
11329
|
},
|
|
10672
|
-
update(docId, document, schema, language, options) {
|
|
11330
|
+
update(docId, document, schema, language, options, removedSpellings) {
|
|
10673
11331
|
invalidateDocFrequencies();
|
|
10674
11332
|
const owner = frozenOwner(docId);
|
|
10675
11333
|
if (owner === void 0) {
|
|
@@ -10677,9 +11335,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10677
11335
|
return;
|
|
10678
11336
|
}
|
|
10679
11337
|
live.insert(docId, document, schema, language, options);
|
|
10680
|
-
owner.tombstoneDocument(docId);
|
|
11338
|
+
owner.tombstoneDocument(docId, removedSpellings);
|
|
10681
11339
|
},
|
|
10682
|
-
remove(docId, schema, language, options) {
|
|
11340
|
+
remove(docId, schema, language, options, removedSpellings) {
|
|
10683
11341
|
if (live.has(docId)) {
|
|
10684
11342
|
invalidateDocFrequencies();
|
|
10685
11343
|
live.remove(docId, schema, language, options);
|
|
@@ -10692,7 +11350,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10692
11350
|
partitionId
|
|
10693
11351
|
});
|
|
10694
11352
|
}
|
|
10695
|
-
|
|
11353
|
+
invalidateDocFrequencies();
|
|
11354
|
+
owner.tombstoneDocument(docId, removedSpellings);
|
|
11355
|
+
},
|
|
11356
|
+
removedSpellingsOf(docId) {
|
|
11357
|
+
return live.has(docId) ? void 0 : frozenOwner(docId)?.removedSpellingsOf(docId);
|
|
10696
11358
|
},
|
|
10697
11359
|
beginBatch() {
|
|
10698
11360
|
live.beginBatch();
|
|
@@ -10755,9 +11417,10 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10755
11417
|
expandPartialWords(subs(), analysis, []);
|
|
10756
11418
|
},
|
|
10757
11419
|
get(docId, projection) {
|
|
10758
|
-
|
|
10759
|
-
|
|
10760
|
-
|
|
11420
|
+
return compositeDocument(frozen, live, docId, projection);
|
|
11421
|
+
},
|
|
11422
|
+
fieldValues(docId, fieldPaths) {
|
|
11423
|
+
return compositeFieldValues(frozen, live, docId, fieldPaths);
|
|
10761
11424
|
},
|
|
10762
11425
|
getRef(docId) {
|
|
10763
11426
|
const stored = frozenStored(docId);
|
|
@@ -10776,8 +11439,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10776
11439
|
},
|
|
10777
11440
|
*docIds() {
|
|
10778
11441
|
for (const segment of frozen) {
|
|
10779
|
-
for (const
|
|
10780
|
-
|
|
11442
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11443
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
11444
|
+
if (docId !== void 0) yield docId;
|
|
10781
11445
|
}
|
|
10782
11446
|
}
|
|
10783
11447
|
yield* live.docIds();
|
|
@@ -10886,6 +11550,25 @@ function setNestedValue(obj, path, value) {
|
|
|
10886
11550
|
}
|
|
10887
11551
|
current[segments[segments.length - 1]] = value;
|
|
10888
11552
|
}
|
|
11553
|
+
function copyVectorsInto(document, docId, vectorIndexes, projection) {
|
|
11554
|
+
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
11555
|
+
if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
|
|
11556
|
+
const vector = vectorIndex.getVector(docId);
|
|
11557
|
+
if (vector) setNestedValue(document, fieldPath, vector);
|
|
11558
|
+
}
|
|
11559
|
+
}
|
|
11560
|
+
|
|
11561
|
+
// src/partitioning/manager/segments.ts
|
|
11562
|
+
function requireNewSegmentDocuments(segment, docPartitionMap) {
|
|
11563
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11564
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
11565
|
+
if (docId === void 0 || !docPartitionMap.has(docId)) continue;
|
|
11566
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
|
|
11567
|
+
docId,
|
|
11568
|
+
partitionId: docPartitionMap.get(docId)
|
|
11569
|
+
});
|
|
11570
|
+
}
|
|
11571
|
+
}
|
|
10889
11572
|
|
|
10890
11573
|
// src/partitioning/manager/stats.ts
|
|
10891
11574
|
function aggregateStatsOf(partitions) {
|
|
@@ -10969,6 +11652,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
10969
11652
|
const count = initialPartitionCount ?? 1;
|
|
10970
11653
|
const analysis = resolveIndexAnalysis(config);
|
|
10971
11654
|
const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
11655
|
+
const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
10972
11656
|
const trackPositions = config.trackPositions ?? true;
|
|
10973
11657
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
10974
11658
|
let partitions = [];
|
|
@@ -10997,10 +11681,8 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
10997
11681
|
}
|
|
10998
11682
|
function asCompositePartition(partitionId) {
|
|
10999
11683
|
const existing = partitions[partitionId];
|
|
11000
|
-
if (
|
|
11001
|
-
|
|
11002
|
-
}
|
|
11003
|
-
const composite = createCompositePartition(partitionId, trackPositions, existing);
|
|
11684
|
+
if (isCompositePartition(existing)) return existing;
|
|
11685
|
+
const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
|
|
11004
11686
|
partitions[partitionId] = composite;
|
|
11005
11687
|
return composite;
|
|
11006
11688
|
}
|
|
@@ -11050,6 +11732,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11050
11732
|
get partialWordAnalysis() {
|
|
11051
11733
|
return partialWordAnalysis;
|
|
11052
11734
|
},
|
|
11735
|
+
get textAnalysis() {
|
|
11736
|
+
return textAnalysis;
|
|
11737
|
+
},
|
|
11053
11738
|
getPartition(partitionId) {
|
|
11054
11739
|
validatePartitionId(partitionId);
|
|
11055
11740
|
return partitions[partitionId];
|
|
@@ -11133,12 +11818,17 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11133
11818
|
docPartitionMap.set(docId, pid);
|
|
11134
11819
|
writeLog.documentWritten(docId);
|
|
11135
11820
|
},
|
|
11136
|
-
|
|
11821
|
+
removedSpellingsOf(docId) {
|
|
11822
|
+
const pid = locateDocument(docId);
|
|
11823
|
+
const partition = pid === void 0 ? void 0 : partitions[pid];
|
|
11824
|
+
return partition !== void 0 && isCompositePartition(partition) ? partition.removedSpellingsOf(docId) : void 0;
|
|
11825
|
+
},
|
|
11826
|
+
remove(docId, removedSpellings) {
|
|
11137
11827
|
const pid = locateDocument(docId);
|
|
11138
11828
|
if (pid === void 0) {
|
|
11139
11829
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
11140
11830
|
}
|
|
11141
|
-
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
|
|
11831
|
+
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
|
|
11142
11832
|
docPartitionMap.delete(docId);
|
|
11143
11833
|
writeLog.documentWritten(docId);
|
|
11144
11834
|
},
|
|
@@ -11152,13 +11842,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11152
11842
|
partitions[i].endBatch();
|
|
11153
11843
|
}
|
|
11154
11844
|
},
|
|
11155
|
-
update(docId, document, options) {
|
|
11845
|
+
update(docId, document, options, removedSpellings) {
|
|
11156
11846
|
const pid = locateDocument(docId);
|
|
11157
11847
|
if (pid === void 0) {
|
|
11158
11848
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
11159
11849
|
}
|
|
11160
11850
|
const updateOpts = resolveInsertOptions(options);
|
|
11161
|
-
partitions[pid].update(docId, document, config.schema, language, updateOpts);
|
|
11851
|
+
partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
|
|
11162
11852
|
docPartitionMap.set(docId, pid);
|
|
11163
11853
|
writeLog.documentWritten(docId);
|
|
11164
11854
|
},
|
|
@@ -11166,14 +11856,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11166
11856
|
const pid = locateDocument(docId);
|
|
11167
11857
|
if (pid === void 0) return void 0;
|
|
11168
11858
|
const doc = partitions[pid].get(docId, projection);
|
|
11169
|
-
if (
|
|
11170
|
-
for (const [fieldPath, vecIndex] of vecIndexes) {
|
|
11171
|
-
if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
|
|
11172
|
-
const vector = vecIndex.getVector(docId);
|
|
11173
|
-
if (vector) {
|
|
11174
|
-
setNestedValue(doc, fieldPath, vector);
|
|
11175
|
-
}
|
|
11176
|
-
}
|
|
11859
|
+
if (doc) copyVectorsInto(doc, docId, vecIndexes, projection);
|
|
11177
11860
|
return doc;
|
|
11178
11861
|
},
|
|
11179
11862
|
getRef(docId) {
|
|
@@ -11181,6 +11864,10 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11181
11864
|
if (pid === void 0) return void 0;
|
|
11182
11865
|
return partitions[pid].getRef(docId);
|
|
11183
11866
|
},
|
|
11867
|
+
fieldValues(docId, fieldPaths) {
|
|
11868
|
+
const pid = locateDocument(docId);
|
|
11869
|
+
return pid === void 0 ? void 0 : partitions[pid].fieldValues(docId, fieldPaths);
|
|
11870
|
+
},
|
|
11184
11871
|
sortValues(docId, fields, fieldTypes, modes) {
|
|
11185
11872
|
const pid = locateDocument(docId);
|
|
11186
11873
|
if (pid === void 0) return fields.map(() => null);
|
|
@@ -11216,22 +11903,20 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11216
11903
|
},
|
|
11217
11904
|
attachFrozenSegment(partitionId, segment) {
|
|
11218
11905
|
validatePartitionId(partitionId);
|
|
11219
|
-
|
|
11220
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
11221
|
-
if (docId === void 0 || !docPartitionMap.has(docId)) continue;
|
|
11222
|
-
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
|
|
11223
|
-
docId,
|
|
11224
|
-
partitionId: docPartitionMap.get(docId)
|
|
11225
|
-
});
|
|
11226
|
-
}
|
|
11906
|
+
requireNewSegmentDocuments(segment, docPartitionMap);
|
|
11227
11907
|
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
11228
11908
|
registerSegmentDocuments(partitionId, segment);
|
|
11229
11909
|
writeLog.segmentWritten(segment);
|
|
11230
11910
|
},
|
|
11231
|
-
swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
|
|
11911
|
+
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
11232
11912
|
validatePartitionId(partitionId);
|
|
11233
|
-
asCompositePartition(partitionId).swapFrozenSegments(
|
|
11913
|
+
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
11914
|
+
dropSegmentIds,
|
|
11915
|
+
replacement,
|
|
11916
|
+
removedSpellings
|
|
11917
|
+
);
|
|
11234
11918
|
registerSegmentDocuments(partitionId, replacement);
|
|
11919
|
+
return applied;
|
|
11235
11920
|
},
|
|
11236
11921
|
freezeLiveTail(partitionId, freeze) {
|
|
11237
11922
|
validatePartitionId(partitionId);
|
|
@@ -11291,21 +11976,15 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
11291
11976
|
// src/core/partition/frozen/doc-table.ts
|
|
11292
11977
|
function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
11293
11978
|
const capacity = idTable.count;
|
|
11294
|
-
const storedCache = new Array(capacity);
|
|
11295
11979
|
let sortedIds = null;
|
|
11296
11980
|
let sortedRevision = -1;
|
|
11297
11981
|
function storedAt(ordinal) {
|
|
11298
|
-
|
|
11299
|
-
|
|
11300
|
-
const
|
|
11301
|
-
|
|
11302
|
-
const length = column[ordinal];
|
|
11303
|
-
if (length > 0) fieldLengths[fieldName] = length;
|
|
11304
|
-
}
|
|
11305
|
-
stored = { fields: documents.docAt(ordinal), fieldLengths };
|
|
11306
|
-
storedCache[ordinal] = stored;
|
|
11982
|
+
const fieldLengths = {};
|
|
11983
|
+
for (const [fieldName, column] of lengthColumns) {
|
|
11984
|
+
const length = column[ordinal];
|
|
11985
|
+
if (length > 0) fieldLengths[fieldName] = length;
|
|
11307
11986
|
}
|
|
11308
|
-
return
|
|
11987
|
+
return { fields: documents.docAt(ordinal), fieldLengths };
|
|
11309
11988
|
}
|
|
11310
11989
|
function liveOrdinal(docId) {
|
|
11311
11990
|
const ordinal = idTable.ordinalOf(docId);
|
|
@@ -11374,9 +12053,241 @@ function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
|
11374
12053
|
},
|
|
11375
12054
|
resolver() {
|
|
11376
12055
|
return resolver;
|
|
12056
|
+
},
|
|
12057
|
+
fieldReader(fieldPath) {
|
|
12058
|
+
const read = documents.fieldReader(fieldPath);
|
|
12059
|
+
return (internalId) => resolver.isLive(internalId) ? read(internalId) : void 0;
|
|
12060
|
+
},
|
|
12061
|
+
projectedDocument(docId, projection) {
|
|
12062
|
+
const ordinal = liveOrdinal(docId);
|
|
12063
|
+
return ordinal === void 0 ? void 0 : documents.projectedAt(ordinal, projection);
|
|
11377
12064
|
}
|
|
11378
12065
|
};
|
|
11379
12066
|
}
|
|
12067
|
+
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
12068
|
+
var keyDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
12069
|
+
function headerAt(view, offset) {
|
|
12070
|
+
if (offset >= view.byteLength) return null;
|
|
12071
|
+
const tag = view.getUint8(offset);
|
|
12072
|
+
if (tag <= 127 || tag >= 224) return { headerBytes: 1, bodyBytes: 0, children: 0 };
|
|
12073
|
+
if (tag <= 143) return { headerBytes: 1, bodyBytes: 0, children: (tag & 15) * 2 };
|
|
12074
|
+
if (tag <= 159) return { headerBytes: 1, bodyBytes: 0, children: tag & 15 };
|
|
12075
|
+
if (tag <= 191) return { headerBytes: 1, bodyBytes: tag & 31, children: 0 };
|
|
12076
|
+
const lengthBytes = LENGTH_BYTES_BY_TAG.get(tag);
|
|
12077
|
+
if (lengthBytes !== void 0) {
|
|
12078
|
+
if (offset + 1 + lengthBytes > view.byteLength) return null;
|
|
12079
|
+
const length = lengthBytes === 1 ? view.getUint8(offset + 1) : lengthBytes === 2 ? view.getUint16(offset + 1, false) : view.getUint32(offset + 1, false);
|
|
12080
|
+
const headerBytes = 1 + lengthBytes;
|
|
12081
|
+
if (ARRAY_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length };
|
|
12082
|
+
if (MAP_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length * 2 };
|
|
12083
|
+
return { headerBytes, bodyBytes: length + (EXT_TAGS.has(tag) ? 1 : 0), children: 0 };
|
|
12084
|
+
}
|
|
12085
|
+
const fixedBytes = FIXED_BODY_BYTES_BY_TAG.get(tag);
|
|
12086
|
+
return fixedBytes === void 0 ? null : { headerBytes: 1, bodyBytes: fixedBytes, children: 0 };
|
|
12087
|
+
}
|
|
12088
|
+
var LENGTH_BYTES_BY_TAG = /* @__PURE__ */ new Map([
|
|
12089
|
+
[196, 1],
|
|
12090
|
+
[197, 2],
|
|
12091
|
+
[198, 4],
|
|
12092
|
+
[199, 1],
|
|
12093
|
+
[200, 2],
|
|
12094
|
+
[201, 4],
|
|
12095
|
+
[217, 1],
|
|
12096
|
+
[218, 2],
|
|
12097
|
+
[219, 4],
|
|
12098
|
+
[220, 2],
|
|
12099
|
+
[221, 4],
|
|
12100
|
+
[222, 2],
|
|
12101
|
+
[223, 4]
|
|
12102
|
+
]);
|
|
12103
|
+
var ARRAY_TAGS = /* @__PURE__ */ new Set([220, 221]);
|
|
12104
|
+
var MAP_TAGS = /* @__PURE__ */ new Set([222, 223]);
|
|
12105
|
+
var EXT_TAGS = /* @__PURE__ */ new Set([199, 200, 201]);
|
|
12106
|
+
var FIXED_BODY_BYTES_BY_TAG = /* @__PURE__ */ new Map([
|
|
12107
|
+
[192, 0],
|
|
12108
|
+
[194, 0],
|
|
12109
|
+
[195, 0],
|
|
12110
|
+
[202, 4],
|
|
12111
|
+
[203, 8],
|
|
12112
|
+
[204, 1],
|
|
12113
|
+
[205, 2],
|
|
12114
|
+
[206, 4],
|
|
12115
|
+
[207, 8],
|
|
12116
|
+
[208, 1],
|
|
12117
|
+
[209, 2],
|
|
12118
|
+
[210, 4],
|
|
12119
|
+
[211, 8],
|
|
12120
|
+
[212, 2],
|
|
12121
|
+
[213, 3],
|
|
12122
|
+
[214, 5],
|
|
12123
|
+
[215, 9],
|
|
12124
|
+
[216, 17]
|
|
12125
|
+
]);
|
|
12126
|
+
function endOfValue(view, start) {
|
|
12127
|
+
let offset = start;
|
|
12128
|
+
let pending = 1;
|
|
12129
|
+
while (pending > 0) {
|
|
12130
|
+
const header = headerAt(view, offset);
|
|
12131
|
+
if (header === null) return null;
|
|
12132
|
+
offset += header.headerBytes + header.bodyBytes;
|
|
12133
|
+
if (offset > view.byteLength) return null;
|
|
12134
|
+
pending += header.children - 1;
|
|
12135
|
+
}
|
|
12136
|
+
return offset;
|
|
12137
|
+
}
|
|
12138
|
+
function stringKeyAt(bytes, view, offset) {
|
|
12139
|
+
const header = headerAt(view, offset);
|
|
12140
|
+
if (header === null || header.children !== 0) return null;
|
|
12141
|
+
const tag = view.getUint8(offset);
|
|
12142
|
+
const isString = tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
|
|
12143
|
+
const end = offset + header.headerBytes + header.bodyBytes;
|
|
12144
|
+
if (!isString || end > bytes.length) return null;
|
|
12145
|
+
try {
|
|
12146
|
+
return { key: keyDecoder.decode(bytes.subarray(offset + header.headerBytes, end)), end };
|
|
12147
|
+
} catch {
|
|
12148
|
+
return null;
|
|
12149
|
+
}
|
|
12150
|
+
}
|
|
12151
|
+
function topLevelMapKeys(bytes) {
|
|
12152
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
12153
|
+
const top = headerAt(view, 0);
|
|
12154
|
+
if (top === null || top.bodyBytes !== 0) return null;
|
|
12155
|
+
const tag = view.getUint8(0);
|
|
12156
|
+
if (!(tag >= 128 && tag <= 143 || MAP_TAGS.has(tag))) return null;
|
|
12157
|
+
const keys = /* @__PURE__ */ new Set();
|
|
12158
|
+
let offset = top.headerBytes;
|
|
12159
|
+
for (let pair = 0; pair < top.children / 2; pair += 1) {
|
|
12160
|
+
const key = stringKeyAt(bytes, view, offset);
|
|
12161
|
+
if (key === null) return null;
|
|
12162
|
+
const valueEnd = endOfValue(view, key.end);
|
|
12163
|
+
if (valueEnd === null) return null;
|
|
12164
|
+
keys.add(key.key);
|
|
12165
|
+
offset = valueEnd;
|
|
12166
|
+
}
|
|
12167
|
+
return keys;
|
|
12168
|
+
}
|
|
12169
|
+
function decodeMapWithoutFields(bytes, skippedFields) {
|
|
12170
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
12171
|
+
const top = headerAt(view, 0);
|
|
12172
|
+
if (top === null || top.children === 0 || top.bodyBytes !== 0) return null;
|
|
12173
|
+
const tag = view.getUint8(0);
|
|
12174
|
+
const isMap = tag >= 128 && tag <= 143 || MAP_TAGS.has(tag);
|
|
12175
|
+
if (!isMap) return null;
|
|
12176
|
+
const document = {};
|
|
12177
|
+
let offset = top.headerBytes;
|
|
12178
|
+
for (let pair = 0; pair < top.children / 2; pair += 1) {
|
|
12179
|
+
const key = stringKeyAt(bytes, view, offset);
|
|
12180
|
+
if (key === null || FORBIDDEN_KEYS.has(key.key)) return null;
|
|
12181
|
+
const valueEnd = endOfValue(view, key.end);
|
|
12182
|
+
if (valueEnd === null) return null;
|
|
12183
|
+
if (!skippedFields.has(key.key)) {
|
|
12184
|
+
try {
|
|
12185
|
+
document[key.key] = decode(bytes.subarray(key.end, valueEnd));
|
|
12186
|
+
} catch {
|
|
12187
|
+
return null;
|
|
12188
|
+
}
|
|
12189
|
+
}
|
|
12190
|
+
offset = valueEnd;
|
|
12191
|
+
}
|
|
12192
|
+
return offset === bytes.length ? document : null;
|
|
12193
|
+
}
|
|
12194
|
+
|
|
12195
|
+
// src/serialization/msgpack-path.ts
|
|
12196
|
+
var encoder2 = new TextEncoder();
|
|
12197
|
+
var PROTO_KEY_BYTES = encoder2.encode("__proto__");
|
|
12198
|
+
var MISSING = { found: false, value: void 0 };
|
|
12199
|
+
var pathDecoder = new Decoder();
|
|
12200
|
+
function encodePathSegments(segments) {
|
|
12201
|
+
if (segments.length === 0) return null;
|
|
12202
|
+
if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) return null;
|
|
12203
|
+
return segments.map((segment) => encoder2.encode(segment));
|
|
12204
|
+
}
|
|
12205
|
+
function isStringTag(tag) {
|
|
12206
|
+
return tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
|
|
12207
|
+
}
|
|
12208
|
+
function isMapTag(tag) {
|
|
12209
|
+
return tag >= 128 && tag <= 143 || tag === 222 || tag === 223;
|
|
12210
|
+
}
|
|
12211
|
+
function isArrayTag(tag) {
|
|
12212
|
+
return tag >= 144 && tag <= 159 || tag === 220 || tag === 221;
|
|
12213
|
+
}
|
|
12214
|
+
function isScalarTag(tag) {
|
|
12215
|
+
if (tag <= 127 || tag >= 224 || isStringTag(tag)) return true;
|
|
12216
|
+
return tag === 192 || tag === 194 || tag === 195 || tag >= 202 && tag <= 211;
|
|
12217
|
+
}
|
|
12218
|
+
function sameBytes(bytes, start, end, target) {
|
|
12219
|
+
if (end - start !== target.length) return false;
|
|
12220
|
+
for (let at = 0; at < target.length; at++) {
|
|
12221
|
+
if (bytes[start + at] !== target[at]) return false;
|
|
12222
|
+
}
|
|
12223
|
+
return true;
|
|
12224
|
+
}
|
|
12225
|
+
function keyRangeAt(view, offset) {
|
|
12226
|
+
if (!isStringTag(view.getUint8(offset))) return null;
|
|
12227
|
+
const header = headerAt(view, offset);
|
|
12228
|
+
if (header === null) return null;
|
|
12229
|
+
const start = offset + header.headerBytes;
|
|
12230
|
+
return { start, end: start + header.bodyBytes };
|
|
12231
|
+
}
|
|
12232
|
+
function readEncodedPath(bytes, segments, arrays) {
|
|
12233
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
12234
|
+
let start = 0;
|
|
12235
|
+
let end = bytes.length;
|
|
12236
|
+
for (let depth = 0; depth < segments.length; depth++) {
|
|
12237
|
+
if (start >= end) return null;
|
|
12238
|
+
const tag = view.getUint8(start);
|
|
12239
|
+
if (!isMapTag(tag)) {
|
|
12240
|
+
if (depth === 0) return null;
|
|
12241
|
+
if (isArrayTag(tag)) return arrays === "missing" ? MISSING : null;
|
|
12242
|
+
return isScalarTag(tag) ? MISSING : null;
|
|
12243
|
+
}
|
|
12244
|
+
const header = headerAt(view, start);
|
|
12245
|
+
if (header === null) return null;
|
|
12246
|
+
let offset = start + header.headerBytes;
|
|
12247
|
+
let valueStart = -1;
|
|
12248
|
+
let valueEnd = -1;
|
|
12249
|
+
for (let pair = 0; pair < header.children / 2; pair++) {
|
|
12250
|
+
if (offset >= end) return null;
|
|
12251
|
+
const key = keyRangeAt(view, offset);
|
|
12252
|
+
if (key === null || key.end > end) return null;
|
|
12253
|
+
if (sameBytes(bytes, key.start, key.end, PROTO_KEY_BYTES)) return null;
|
|
12254
|
+
const next = endOfValue(view, key.end);
|
|
12255
|
+
if (next === null || next > end) return null;
|
|
12256
|
+
if (sameBytes(bytes, key.start, key.end, segments[depth])) {
|
|
12257
|
+
valueStart = key.end;
|
|
12258
|
+
valueEnd = next;
|
|
12259
|
+
}
|
|
12260
|
+
offset = next;
|
|
12261
|
+
}
|
|
12262
|
+
if (offset !== end) return null;
|
|
12263
|
+
if (valueStart < 0) return MISSING;
|
|
12264
|
+
start = valueStart;
|
|
12265
|
+
end = valueEnd;
|
|
12266
|
+
}
|
|
12267
|
+
try {
|
|
12268
|
+
return { found: true, value: pathDecoder.decode(bytes.subarray(start, end)) };
|
|
12269
|
+
} catch {
|
|
12270
|
+
return null;
|
|
12271
|
+
}
|
|
12272
|
+
}
|
|
12273
|
+
|
|
12274
|
+
// src/core/partition/frozen/document-source.ts
|
|
12275
|
+
var encodedProjectionPaths = /* @__PURE__ */ new WeakMap();
|
|
12276
|
+
function encodedProjectionPathOf(path) {
|
|
12277
|
+
let segments = encodedProjectionPaths.get(path);
|
|
12278
|
+
if (segments === void 0) {
|
|
12279
|
+
segments = encodePathSegments(path);
|
|
12280
|
+
encodedProjectionPaths.set(path, segments);
|
|
12281
|
+
}
|
|
12282
|
+
return segments;
|
|
12283
|
+
}
|
|
12284
|
+
function sharesPrefix(left, right) {
|
|
12285
|
+
const shared2 = Math.min(left.length, right.length);
|
|
12286
|
+
for (let at = 0; at < shared2; at++) {
|
|
12287
|
+
if (left[at] !== right[at]) return false;
|
|
12288
|
+
}
|
|
12289
|
+
return true;
|
|
12290
|
+
}
|
|
11380
12291
|
function removeFieldAtPath(document, fieldPath) {
|
|
11381
12292
|
const segments = fieldPath.split(".");
|
|
11382
12293
|
let current = document;
|
|
@@ -11442,25 +12353,40 @@ function encodeDocumentTableData(documents) {
|
|
|
11442
12353
|
function wrapEncodedDocumentTable(data) {
|
|
11443
12354
|
const { blob, offsets } = data;
|
|
11444
12355
|
const count = offsets.length - 1;
|
|
11445
|
-
const
|
|
11446
|
-
const
|
|
11447
|
-
|
|
12356
|
+
const discarded = (data.discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
|
|
12357
|
+
const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
|
|
12358
|
+
const bytesAt = (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]);
|
|
12359
|
+
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
|
|
12360
|
+
function projectedPathAt(ordinal, path) {
|
|
12361
|
+
const segments = encodedProjectionPathOf(path);
|
|
12362
|
+
if (segments === null) return { found: false, value: void 0 };
|
|
12363
|
+
const read = touchesDiscarded(path) ? null : readEncodedPath(bytesAt(ordinal), segments, "missing");
|
|
12364
|
+
if (read === null) return readProjectedPath(docAt(ordinal), path);
|
|
12365
|
+
return read.found ? { found: true, value: valueWithOwnBuffers(read.value) } : read;
|
|
12366
|
+
}
|
|
11448
12367
|
return {
|
|
11449
12368
|
count,
|
|
11450
|
-
|
|
11451
|
-
|
|
11452
|
-
},
|
|
11453
|
-
docAt(ordinal) {
|
|
11454
|
-
let document = decoded[ordinal];
|
|
11455
|
-
if (document === void 0) {
|
|
11456
|
-
document = decodeStoredDocument(blob.slice(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
|
|
11457
|
-
decoded[ordinal] = document;
|
|
11458
|
-
decodedBytes += documentValueBytes(document);
|
|
11459
|
-
}
|
|
11460
|
-
return document;
|
|
11461
|
-
},
|
|
12369
|
+
byteLength: blob.byteLength + offsets.byteLength,
|
|
12370
|
+
docAt,
|
|
11462
12371
|
readAt(ordinal) {
|
|
11463
|
-
return
|
|
12372
|
+
return decodeStoredDocument(bytesAt(ordinal), data.discardedFields);
|
|
12373
|
+
},
|
|
12374
|
+
fieldReader(fieldPath) {
|
|
12375
|
+
const path = fieldPath.split(".");
|
|
12376
|
+
const segments = encodePathSegments(path);
|
|
12377
|
+
if (segments === null) return () => void 0;
|
|
12378
|
+
const decodesDocument = touchesDiscarded(path);
|
|
12379
|
+
return (ordinal) => {
|
|
12380
|
+
const read = decodesDocument ? null : readEncodedPath(bytesAt(ordinal), segments, "decode-document");
|
|
12381
|
+
if (read !== null) return valueWithOwnBuffers(read.value);
|
|
12382
|
+
return getNestedValue(docAt(ordinal), fieldPath);
|
|
12383
|
+
};
|
|
12384
|
+
},
|
|
12385
|
+
projectedAt(ordinal, projection) {
|
|
12386
|
+
if (projection === void 0 || projection.kind === "full") return docAt(ordinal);
|
|
12387
|
+
if (projection.kind === "none") return {};
|
|
12388
|
+
if (projection.include === null) return cloneProjected(docAt(ordinal), projection);
|
|
12389
|
+
return projectIncludedPaths(projection.include, projection.exclude, (path) => projectedPathAt(ordinal, path));
|
|
11464
12390
|
}
|
|
11465
12391
|
};
|
|
11466
12392
|
}
|
|
@@ -11477,6 +12403,12 @@ function wrapDocumentArray(documents) {
|
|
|
11477
12403
|
},
|
|
11478
12404
|
readAt(ordinal) {
|
|
11479
12405
|
return documents[ordinal];
|
|
12406
|
+
},
|
|
12407
|
+
fieldReader(fieldPath) {
|
|
12408
|
+
return (ordinal) => getNestedValue(documents[ordinal], fieldPath);
|
|
12409
|
+
},
|
|
12410
|
+
projectedAt(ordinal, projection) {
|
|
12411
|
+
return cloneProjected(documents[ordinal], projection);
|
|
11480
12412
|
}
|
|
11481
12413
|
};
|
|
11482
12414
|
}
|
|
@@ -11484,18 +12416,18 @@ function wrapDocumentArray(documents) {
|
|
|
11484
12416
|
// src/core/partition/frozen/string-blob.ts
|
|
11485
12417
|
var SURROGATE = /[\ud800-\udfff]/;
|
|
11486
12418
|
var UTF8_BYTES_PER_UTF16_UNIT = 3;
|
|
11487
|
-
var
|
|
12419
|
+
var encoder3 = new TextEncoder();
|
|
11488
12420
|
var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
|
|
11489
12421
|
function encodeLookupKey(text) {
|
|
11490
|
-
if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return
|
|
11491
|
-
const { written } =
|
|
12422
|
+
if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder3.encode(text);
|
|
12423
|
+
const { written } = encoder3.encodeInto(text, lookupKeyBytes);
|
|
11492
12424
|
return lookupKeyBytes.subarray(0, written);
|
|
11493
12425
|
}
|
|
11494
12426
|
function encodeOneByOne(strings) {
|
|
11495
12427
|
const encoded = new Array(strings.length);
|
|
11496
12428
|
let blobLength = 0;
|
|
11497
12429
|
for (let i = 0; i < strings.length; i++) {
|
|
11498
|
-
encoded[i] =
|
|
12430
|
+
encoded[i] = encoder3.encode(strings[i]);
|
|
11499
12431
|
blobLength += encoded[i].length;
|
|
11500
12432
|
}
|
|
11501
12433
|
const blob = new Uint8Array(blobLength);
|
|
@@ -11519,7 +12451,7 @@ function basicPlaneUtf8Bytes(text) {
|
|
|
11519
12451
|
function encodeStringBlob(strings) {
|
|
11520
12452
|
const joined2 = strings.join("");
|
|
11521
12453
|
if (SURROGATE.test(joined2)) return encodeOneByOne(strings);
|
|
11522
|
-
const blob =
|
|
12454
|
+
const blob = encoder3.encode(joined2);
|
|
11523
12455
|
const offsets = new Uint32Array(strings.length + 1);
|
|
11524
12456
|
let cursor = 0;
|
|
11525
12457
|
for (let i = 0; i < strings.length; i++) {
|
|
@@ -11854,9 +12786,13 @@ function createFrozenEnumReader(entry) {
|
|
|
11854
12786
|
}
|
|
11855
12787
|
|
|
11856
12788
|
// src/core/partition/frozen/inverted-reader.ts
|
|
11857
|
-
function createFrozenInvertedReader(table, views) {
|
|
12789
|
+
function createFrozenInvertedReader(table, views, counts) {
|
|
11858
12790
|
function viewAtSorted(sortedIndex) {
|
|
11859
|
-
return views.viewAt(table.payloadSlot(sortedIndex),
|
|
12791
|
+
return views.viewAt(table.payloadSlot(sortedIndex), sortedIndex);
|
|
12792
|
+
}
|
|
12793
|
+
function liveAt(token) {
|
|
12794
|
+
const at = table.find(token);
|
|
12795
|
+
return at < 0 || counts.documentFrequency(at) === 0 ? -1 : at;
|
|
11860
12796
|
}
|
|
11861
12797
|
function candidateRange(prefix) {
|
|
11862
12798
|
if (prefix.length === 0) return { start: 0, end: table.size };
|
|
@@ -11864,22 +12800,22 @@ function createFrozenInvertedReader(table, views) {
|
|
|
11864
12800
|
}
|
|
11865
12801
|
return {
|
|
11866
12802
|
lookup(token) {
|
|
11867
|
-
const at =
|
|
12803
|
+
const at = liveAt(token);
|
|
11868
12804
|
if (at < 0) return void 0;
|
|
11869
12805
|
return viewAtSorted(at);
|
|
11870
12806
|
},
|
|
11871
12807
|
lookupReadOnce(token) {
|
|
11872
|
-
const at =
|
|
12808
|
+
const at = liveAt(token);
|
|
11873
12809
|
if (at < 0) return void 0;
|
|
11874
|
-
return views.viewReadOnce(table.payloadSlot(at),
|
|
12810
|
+
return views.viewReadOnce(table.payloadSlot(at), at);
|
|
11875
12811
|
},
|
|
11876
12812
|
documentFrequency(token) {
|
|
11877
12813
|
const at = table.find(token);
|
|
11878
|
-
return at < 0 ? 0 :
|
|
12814
|
+
return at < 0 ? 0 : counts.documentFrequency(at);
|
|
11879
12815
|
},
|
|
11880
12816
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
11881
12817
|
if (tolerance === 0) {
|
|
11882
|
-
const at =
|
|
12818
|
+
const at = liveAt(token);
|
|
11883
12819
|
return at < 0 ? [] : [{ token, postingList: viewAtSorted(at) }];
|
|
11884
12820
|
}
|
|
11885
12821
|
const prefix = fuzzyPrefixOf(token, prefixLength);
|
|
@@ -11890,7 +12826,7 @@ function createFrozenInvertedReader(table, views) {
|
|
|
11890
12826
|
const candidate = table.tokenAt(at);
|
|
11891
12827
|
if (!candidate.startsWith(requiredPrefix)) continue;
|
|
11892
12828
|
const { withinTolerance } = boundedLevenshtein(token, candidate, tolerance);
|
|
11893
|
-
if (withinTolerance) {
|
|
12829
|
+
if (withinTolerance && counts.documentFrequency(at) > 0) {
|
|
11894
12830
|
results.push({ token: candidate, postingList: viewAtSorted(at) });
|
|
11895
12831
|
}
|
|
11896
12832
|
}
|
|
@@ -11901,14 +12837,15 @@ function createFrozenInvertedReader(table, views) {
|
|
|
11901
12837
|
const range = table.prefixRange(prefix);
|
|
11902
12838
|
const results = [];
|
|
11903
12839
|
for (let at = range.start; at < range.end; at++) {
|
|
11904
|
-
|
|
12840
|
+
const documentFrequency = counts.documentFrequency(at);
|
|
12841
|
+
if (documentFrequency > 0) results.push({ term: table.tokenAt(at), documentFrequency });
|
|
11905
12842
|
}
|
|
11906
12843
|
results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.term, b.term));
|
|
11907
12844
|
if (results.length > limit) results.length = limit;
|
|
11908
12845
|
return results;
|
|
11909
12846
|
},
|
|
11910
12847
|
has(token) {
|
|
11911
|
-
return
|
|
12848
|
+
return liveAt(token) >= 0;
|
|
11912
12849
|
},
|
|
11913
12850
|
*tokens() {
|
|
11914
12851
|
for (let at = 0; at < table.size; at++) {
|
|
@@ -11921,6 +12858,154 @@ function createFrozenInvertedReader(table, views) {
|
|
|
11921
12858
|
};
|
|
11922
12859
|
}
|
|
11923
12860
|
|
|
12861
|
+
// src/core/partition/frozen/live-counts.ts
|
|
12862
|
+
var UINT32_BYTES = 4;
|
|
12863
|
+
var FLOAT64_BYTES = 8;
|
|
12864
|
+
function removalIndexBytes(columns, terms) {
|
|
12865
|
+
const documentWords = 2 * (columns.documentCount + 1);
|
|
12866
|
+
return (columns.postingDocIds.length + documentWords + 2 * terms) * UINT32_BYTES + terms * FLOAT64_BYTES;
|
|
12867
|
+
}
|
|
12868
|
+
function processHasRoomFor(bytes) {
|
|
12869
|
+
return true;
|
|
12870
|
+
}
|
|
12871
|
+
function buildRemovalIndex(columns, table) {
|
|
12872
|
+
const { documentCount, postingDocIds } = columns;
|
|
12873
|
+
const starts = new Uint32Array(documentCount + 1);
|
|
12874
|
+
for (let row = 0; row < postingDocIds.length; row++) starts[postingDocIds[row] + 1]++;
|
|
12875
|
+
for (let ordinal = 0; ordinal < documentCount; ordinal++) starts[ordinal + 1] += starts[ordinal];
|
|
12876
|
+
const cursors = starts.slice(0, documentCount);
|
|
12877
|
+
const rows = new Uint32Array(postingDocIds.length);
|
|
12878
|
+
for (let row = 0; row < postingDocIds.length; row++) rows[cursors[postingDocIds[row]]++] = row;
|
|
12879
|
+
const termOfSlot = new Uint32Array(table.size);
|
|
12880
|
+
for (let sortedIndex = 0; sortedIndex < table.size; sortedIndex++) {
|
|
12881
|
+
termOfSlot[table.payloadSlot(sortedIndex)] = sortedIndex;
|
|
12882
|
+
}
|
|
12883
|
+
return {
|
|
12884
|
+
starts,
|
|
12885
|
+
rows,
|
|
12886
|
+
termOfSlot,
|
|
12887
|
+
documents: new Uint32Array(table.size),
|
|
12888
|
+
occurrences: new Float64Array(table.size)
|
|
12889
|
+
};
|
|
12890
|
+
}
|
|
12891
|
+
function tryBuildRemovalIndex(columns, table) {
|
|
12892
|
+
if (!processHasRoomFor(removalIndexBytes(columns, table.size))) return null;
|
|
12893
|
+
try {
|
|
12894
|
+
return buildRemovalIndex(columns, table);
|
|
12895
|
+
} catch (error) {
|
|
12896
|
+
if (error instanceof RangeError) return null;
|
|
12897
|
+
throw error;
|
|
12898
|
+
}
|
|
12899
|
+
}
|
|
12900
|
+
function slotOfRow(postingOffsets, slots, row) {
|
|
12901
|
+
let low = 0;
|
|
12902
|
+
let high = slots;
|
|
12903
|
+
while (low < high) {
|
|
12904
|
+
const middle = low + high >>> 1;
|
|
12905
|
+
if (postingOffsets[middle + 1] <= row) low = middle + 1;
|
|
12906
|
+
else high = middle;
|
|
12907
|
+
}
|
|
12908
|
+
return low;
|
|
12909
|
+
}
|
|
12910
|
+
function createFrozenLiveCounts(columns, table, tombstones, storedFrequencies) {
|
|
12911
|
+
const { postingOffsets, postingDocIds, postingFrequencies } = columns;
|
|
12912
|
+
let index = null;
|
|
12913
|
+
let lengths = null;
|
|
12914
|
+
let frequencies = null;
|
|
12915
|
+
let frequenciesRevision = -1;
|
|
12916
|
+
function countRemoval(built, ordinal) {
|
|
12917
|
+
const { starts, rows, termOfSlot, documents, occurrences } = built;
|
|
12918
|
+
let previousSlot = -1;
|
|
12919
|
+
for (let at = starts[ordinal]; at < starts[ordinal + 1]; at++) {
|
|
12920
|
+
const row = rows[at];
|
|
12921
|
+
const slot = slotOfRow(postingOffsets, table.size, row);
|
|
12922
|
+
const sortedIndex = termOfSlot[slot];
|
|
12923
|
+
if (slot !== previousSlot) documents[sortedIndex]++;
|
|
12924
|
+
occurrences[sortedIndex] += postingFrequencies[row];
|
|
12925
|
+
previousSlot = slot;
|
|
12926
|
+
}
|
|
12927
|
+
}
|
|
12928
|
+
function removedDocumentsOf(sortedIndex) {
|
|
12929
|
+
const slot = table.payloadSlot(sortedIndex);
|
|
12930
|
+
const end = postingOffsets[slot + 1];
|
|
12931
|
+
let removed = 0;
|
|
12932
|
+
let previous = -1;
|
|
12933
|
+
let ordered = true;
|
|
12934
|
+
for (let row = postingOffsets[slot]; row < end; row++) {
|
|
12935
|
+
const ordinal = postingDocIds[row];
|
|
12936
|
+
if (ordinal < previous) ordered = false;
|
|
12937
|
+
if (ordinal !== previous && tombstones.has(ordinal)) removed++;
|
|
12938
|
+
previous = ordinal;
|
|
12939
|
+
}
|
|
12940
|
+
if (ordered) return removed;
|
|
12941
|
+
const seen = /* @__PURE__ */ new Set();
|
|
12942
|
+
for (let row = postingOffsets[slot]; row < end; row++) {
|
|
12943
|
+
const ordinal = postingDocIds[row];
|
|
12944
|
+
if (tombstones.has(ordinal)) seen.add(ordinal);
|
|
12945
|
+
}
|
|
12946
|
+
return seen.size;
|
|
12947
|
+
}
|
|
12948
|
+
function removedOccurrencesOf(sortedIndex) {
|
|
12949
|
+
const slot = table.payloadSlot(sortedIndex);
|
|
12950
|
+
const end = postingOffsets[slot + 1];
|
|
12951
|
+
let removed = 0;
|
|
12952
|
+
for (let row = postingOffsets[slot]; row < end; row++) {
|
|
12953
|
+
if (tombstones.has(postingDocIds[row])) removed += postingFrequencies[row];
|
|
12954
|
+
}
|
|
12955
|
+
return removed;
|
|
12956
|
+
}
|
|
12957
|
+
function documentFrequency(sortedIndex) {
|
|
12958
|
+
const stored = table.documentFrequencyAt(sortedIndex);
|
|
12959
|
+
if (tombstones.size === 0) return stored;
|
|
12960
|
+
if (index !== null) return stored - index.documents[sortedIndex];
|
|
12961
|
+
return stored - removedDocumentsOf(sortedIndex);
|
|
12962
|
+
}
|
|
12963
|
+
return {
|
|
12964
|
+
totalDocuments() {
|
|
12965
|
+
return columns.documentCount - tombstones.size;
|
|
12966
|
+
},
|
|
12967
|
+
totalFieldLengths() {
|
|
12968
|
+
return lengths ?? columns.totalFieldLengths;
|
|
12969
|
+
},
|
|
12970
|
+
documentFrequency,
|
|
12971
|
+
removedOccurrences(sortedIndex) {
|
|
12972
|
+
if (tombstones.size === 0) return 0;
|
|
12973
|
+
if (index !== null) return index.occurrences[sortedIndex];
|
|
12974
|
+
return removedOccurrencesOf(sortedIndex);
|
|
12975
|
+
},
|
|
12976
|
+
docFrequencies() {
|
|
12977
|
+
if (tombstones.size === 0) return storedFrequencies();
|
|
12978
|
+
if (frequencies !== null && frequenciesRevision === tombstones.revision) return frequencies;
|
|
12979
|
+
const live = /* @__PURE__ */ Object.create(null);
|
|
12980
|
+
for (let at = 0; at < table.size; at++) {
|
|
12981
|
+
const documents = documentFrequency(at);
|
|
12982
|
+
if (documents > 0) live[table.tokenAt(at)] = documents;
|
|
12983
|
+
}
|
|
12984
|
+
frequencies = live;
|
|
12985
|
+
frequenciesRevision = tombstones.revision;
|
|
12986
|
+
return frequencies;
|
|
12987
|
+
},
|
|
12988
|
+
subtractRemoved(ordinal) {
|
|
12989
|
+
lengths ??= { ...columns.totalFieldLengths };
|
|
12990
|
+
for (let field = 0; field < columns.fieldLengthNames.length; field++) {
|
|
12991
|
+
lengths[columns.fieldLengthNames[field]] -= columns.fieldLengthColumns[field][ordinal];
|
|
12992
|
+
}
|
|
12993
|
+
if (index !== null) {
|
|
12994
|
+
countRemoval(index, ordinal);
|
|
12995
|
+
return;
|
|
12996
|
+
}
|
|
12997
|
+
const built = tryBuildRemovalIndex(columns, table);
|
|
12998
|
+
if (built === null) return;
|
|
12999
|
+
for (const removed of tombstones.values()) countRemoval(built, removed);
|
|
13000
|
+
index = built;
|
|
13001
|
+
},
|
|
13002
|
+
bytes() {
|
|
13003
|
+
if (index === null) return 0;
|
|
13004
|
+
return index.starts.byteLength + index.rows.byteLength + index.termOfSlot.byteLength + index.documents.byteLength + index.occurrences.byteLength;
|
|
13005
|
+
}
|
|
13006
|
+
};
|
|
13007
|
+
}
|
|
13008
|
+
|
|
11924
13009
|
// src/core/partition/frozen/posting-views.ts
|
|
11925
13010
|
function isOrdered(docIds) {
|
|
11926
13011
|
for (let i = 1; i < docIds.length; i++) {
|
|
@@ -11933,16 +13018,16 @@ function sumFrequencies(frequencies) {
|
|
|
11933
13018
|
for (let i = 0; i < frequencies.length; i++) total += frequencies[i];
|
|
11934
13019
|
return total;
|
|
11935
13020
|
}
|
|
11936
|
-
function createFrozenPostingViews(payload, tombstones) {
|
|
13021
|
+
function createFrozenPostingViews(payload, tombstones, counts) {
|
|
11937
13022
|
const views = /* @__PURE__ */ new Map();
|
|
11938
|
-
function buildView(payloadSlot,
|
|
13023
|
+
function buildView(payloadSlot, sortedIndex, keepsPositions) {
|
|
11939
13024
|
const start = payload.postingOffsets[payloadSlot];
|
|
11940
13025
|
const end = payload.postingOffsets[payloadSlot + 1];
|
|
11941
13026
|
const docIds = payload.postingDocIds.subarray(start, end);
|
|
11942
13027
|
const termFrequencies = payload.postingFrequencies.subarray(start, end);
|
|
11943
13028
|
const fieldNameIndices = payload.postingFieldIndices.subarray(start, end);
|
|
11944
13029
|
const ordered = isOrdered(docIds);
|
|
11945
|
-
const
|
|
13030
|
+
const storedTermFrequency = sumFrequencies(termFrequencies);
|
|
11946
13031
|
const { positionOffsets, positionValues } = payload;
|
|
11947
13032
|
let materializedPositions = null;
|
|
11948
13033
|
return {
|
|
@@ -11964,9 +13049,15 @@ function createFrozenPostingViews(payload, tombstones) {
|
|
|
11964
13049
|
if (positionOffsets === null) return 0;
|
|
11965
13050
|
return positionOffsets[start + row + 1] - positionOffsets[start + row];
|
|
11966
13051
|
},
|
|
11967
|
-
docIdSet: {
|
|
13052
|
+
docIdSet: {
|
|
13053
|
+
get size() {
|
|
13054
|
+
return counts.documentFrequency(sortedIndex);
|
|
13055
|
+
}
|
|
13056
|
+
},
|
|
11968
13057
|
deletedDocs: tombstones,
|
|
11969
|
-
totalTermFrequency
|
|
13058
|
+
get totalTermFrequency() {
|
|
13059
|
+
return storedTermFrequency - counts.removedOccurrences(sortedIndex);
|
|
13060
|
+
},
|
|
11970
13061
|
get structureRevision() {
|
|
11971
13062
|
return tombstones.revision;
|
|
11972
13063
|
},
|
|
@@ -11974,16 +13065,168 @@ function createFrozenPostingViews(payload, tombstones) {
|
|
|
11974
13065
|
};
|
|
11975
13066
|
}
|
|
11976
13067
|
return {
|
|
11977
|
-
viewAt(payloadSlot,
|
|
11978
|
-
let view = views.get(payloadSlot);
|
|
11979
|
-
if (view === void 0) {
|
|
11980
|
-
view = buildView(payloadSlot,
|
|
11981
|
-
views.set(payloadSlot, view);
|
|
13068
|
+
viewAt(payloadSlot, sortedIndex) {
|
|
13069
|
+
let view = views.get(payloadSlot);
|
|
13070
|
+
if (view === void 0) {
|
|
13071
|
+
view = buildView(payloadSlot, sortedIndex, true);
|
|
13072
|
+
views.set(payloadSlot, view);
|
|
13073
|
+
}
|
|
13074
|
+
return view;
|
|
13075
|
+
},
|
|
13076
|
+
viewReadOnce(payloadSlot, sortedIndex) {
|
|
13077
|
+
return buildView(payloadSlot, sortedIndex, false);
|
|
13078
|
+
}
|
|
13079
|
+
};
|
|
13080
|
+
}
|
|
13081
|
+
|
|
13082
|
+
// src/core/partition/partial-words/spellings.ts
|
|
13083
|
+
function textsOf(value) {
|
|
13084
|
+
if (typeof value === "string") return [value];
|
|
13085
|
+
if (!Array.isArray(value)) return [];
|
|
13086
|
+
return value.filter((element) => typeof element === "string");
|
|
13087
|
+
}
|
|
13088
|
+
function visitFieldTexts(document, fields, visit) {
|
|
13089
|
+
for (const field of fields) {
|
|
13090
|
+
for (const text of textsOf(getNestedValue(document, field))) visit(text);
|
|
13091
|
+
}
|
|
13092
|
+
}
|
|
13093
|
+
function visitDocumentSpellings(document, analysis, visit) {
|
|
13094
|
+
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
13095
|
+
const visitOccurrence = (token, _position, surface) => {
|
|
13096
|
+
visit(token, surface ?? token);
|
|
13097
|
+
};
|
|
13098
|
+
visitFieldTexts(document, analysis.fields, (text) => visitTokens(text, analysis.language, tokenizing, visitOccurrence));
|
|
13099
|
+
}
|
|
13100
|
+
function visitDocumentSurfaceSpellings(document, analysis, visit) {
|
|
13101
|
+
if (analysis.options?.customTokenizer !== void 0) return;
|
|
13102
|
+
visitFieldTexts(document, analysis.fields, (text) => visitSurfaceSpellings(text, analysis.language, visit));
|
|
13103
|
+
}
|
|
13104
|
+
function sameFields(left, right) {
|
|
13105
|
+
if (left.length !== right.length) return false;
|
|
13106
|
+
for (let at = 0; at < left.length; at++) {
|
|
13107
|
+
if (left[at] !== right[at]) return false;
|
|
13108
|
+
}
|
|
13109
|
+
return true;
|
|
13110
|
+
}
|
|
13111
|
+
|
|
13112
|
+
// src/core/partition/frozen/removal-pass.ts
|
|
13113
|
+
var NO_REMOVED_SPELLINGS = [];
|
|
13114
|
+
function createFrozenRemovalPass(documents, surfaces) {
|
|
13115
|
+
let bound = null;
|
|
13116
|
+
let pending = [];
|
|
13117
|
+
function heldSpellingsOf(ordinal, analysis) {
|
|
13118
|
+
const spellings = [];
|
|
13119
|
+
if (!surfaces.holdsAny()) return spellings;
|
|
13120
|
+
visitDocumentSurfaceSpellings(documents.documentAt(ordinal), analysis, (spelling) => {
|
|
13121
|
+
if (surfaces.holds(spelling)) spellings.push(spelling);
|
|
13122
|
+
});
|
|
13123
|
+
return spellings;
|
|
13124
|
+
}
|
|
13125
|
+
return {
|
|
13126
|
+
get settled() {
|
|
13127
|
+
return pending.length === 0;
|
|
13128
|
+
},
|
|
13129
|
+
bind(analysis) {
|
|
13130
|
+
if (bound !== null) return;
|
|
13131
|
+
const scoped = { ...analysis, fields: documents.textFields };
|
|
13132
|
+
bound = scoped;
|
|
13133
|
+
for (const ordinal of pending) surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, scoped));
|
|
13134
|
+
pending = [];
|
|
13135
|
+
},
|
|
13136
|
+
spellingsOf(ordinal) {
|
|
13137
|
+
return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
13138
|
+
},
|
|
13139
|
+
afterRemoval(ordinal, spellings) {
|
|
13140
|
+
if (spellings !== void 0) {
|
|
13141
|
+
surfaces.subtractSpellings(ordinal, spellings);
|
|
13142
|
+
} else if (bound !== null) {
|
|
13143
|
+
surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, bound));
|
|
13144
|
+
} else {
|
|
13145
|
+
pending.push(ordinal);
|
|
13146
|
+
}
|
|
13147
|
+
}
|
|
13148
|
+
};
|
|
13149
|
+
}
|
|
13150
|
+
|
|
13151
|
+
// src/core/partition/frozen/surface-slot.ts
|
|
13152
|
+
function applyDocumentSurfaces(target, document, analysis, direction) {
|
|
13153
|
+
if (direction === -1) {
|
|
13154
|
+
visitDocumentSurfaceSpellings(document, analysis, (spelling) => target.subtract(spelling, 1));
|
|
13155
|
+
return;
|
|
13156
|
+
}
|
|
13157
|
+
visitDocumentSpellings(document, analysis, (token, spelling) => {
|
|
13158
|
+
if (spelling !== token) target.add(spelling, token, 1);
|
|
13159
|
+
});
|
|
13160
|
+
}
|
|
13161
|
+
function addForms(target, forms) {
|
|
13162
|
+
for (const surface of Object.keys(forms)) {
|
|
13163
|
+
const value = forms[surface];
|
|
13164
|
+
if (Array.isArray(value)) target.add(surface, value[1], value[0]);
|
|
13165
|
+
}
|
|
13166
|
+
}
|
|
13167
|
+
function createFrozenSurfaceSlot(load, documents) {
|
|
13168
|
+
let registry = null;
|
|
13169
|
+
let uncounted = null;
|
|
13170
|
+
function loaded2() {
|
|
13171
|
+
if (registry === null) {
|
|
13172
|
+
const built = createSurfaceRegistry();
|
|
13173
|
+
const forms = load();
|
|
13174
|
+
if (forms !== null) built.deserialize(forms);
|
|
13175
|
+
registry = built;
|
|
13176
|
+
}
|
|
13177
|
+
return registry;
|
|
13178
|
+
}
|
|
13179
|
+
function isCounted(ordinal) {
|
|
13180
|
+
return uncounted === null || uncounted[ordinal] === 0;
|
|
13181
|
+
}
|
|
13182
|
+
return {
|
|
13183
|
+
reader: {
|
|
13184
|
+
candidatesForPrefix: (prefix) => loaded2().candidatesForPrefix(prefix),
|
|
13185
|
+
stemChangedTotalFor: (token) => loaded2().stemChangedTotalFor(token),
|
|
13186
|
+
size: () => loaded2().size(),
|
|
13187
|
+
serialize: () => loaded2().serialize()
|
|
13188
|
+
},
|
|
13189
|
+
holdsAny() {
|
|
13190
|
+
return loaded2().size() > 0;
|
|
13191
|
+
},
|
|
13192
|
+
holds(spelling) {
|
|
13193
|
+
return loaded2().holds(spelling);
|
|
13194
|
+
},
|
|
13195
|
+
subtractSpellings(ordinal, spellings) {
|
|
13196
|
+
uncounted ??= new Uint8Array(documents.count);
|
|
13197
|
+
uncounted[ordinal] = 1;
|
|
13198
|
+
if (spellings.length === 0) return;
|
|
13199
|
+
const target = loaded2();
|
|
13200
|
+
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
13201
|
+
},
|
|
13202
|
+
addKeptForms(target, mergedOrdinals, analysis) {
|
|
13203
|
+
const source = loaded2();
|
|
13204
|
+
if (analysis === null) {
|
|
13205
|
+
addForms(target, source.serialize());
|
|
13206
|
+
return;
|
|
11982
13207
|
}
|
|
11983
|
-
|
|
11984
|
-
|
|
11985
|
-
|
|
11986
|
-
|
|
13208
|
+
const scoped = { ...analysis, fields: documents.textFields };
|
|
13209
|
+
let kept = 0;
|
|
13210
|
+
let corrections = 0;
|
|
13211
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
13212
|
+
const keeps = mergedOrdinals[ordinal] >= 0;
|
|
13213
|
+
if (keeps) kept++;
|
|
13214
|
+
if (keeps !== isCounted(ordinal)) corrections++;
|
|
13215
|
+
}
|
|
13216
|
+
if (corrections > kept) {
|
|
13217
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
13218
|
+
if (mergedOrdinals[ordinal] >= 0) applyDocumentSurfaces(target, documents.documentAt(ordinal), scoped, 1);
|
|
13219
|
+
}
|
|
13220
|
+
return;
|
|
13221
|
+
}
|
|
13222
|
+
const corrected = createSurfaceRegistry();
|
|
13223
|
+
addForms(corrected, source.serialize());
|
|
13224
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
13225
|
+
const keeps = mergedOrdinals[ordinal] >= 0;
|
|
13226
|
+
if (keeps === isCounted(ordinal)) continue;
|
|
13227
|
+
applyDocumentSurfaces(corrected, documents.documentAt(ordinal), scoped, keeps ? 1 : -1);
|
|
13228
|
+
}
|
|
13229
|
+
addForms(target, corrected.serialize());
|
|
11987
13230
|
}
|
|
11988
13231
|
};
|
|
11989
13232
|
}
|
|
@@ -12011,27 +13254,9 @@ function decodeSurfaceTable(table) {
|
|
|
12011
13254
|
}
|
|
12012
13255
|
return forms;
|
|
12013
13256
|
}
|
|
12014
|
-
function createLazySurfaceReader(load) {
|
|
12015
|
-
let registry = null;
|
|
12016
|
-
function loaded2() {
|
|
12017
|
-
if (registry === null) {
|
|
12018
|
-
const built = createSurfaceRegistry();
|
|
12019
|
-
const forms = load();
|
|
12020
|
-
if (forms !== null) built.deserialize(forms);
|
|
12021
|
-
registry = built;
|
|
12022
|
-
}
|
|
12023
|
-
return registry;
|
|
12024
|
-
}
|
|
12025
|
-
return {
|
|
12026
|
-
candidatesForPrefix: (prefix) => loaded2().candidatesForPrefix(prefix),
|
|
12027
|
-
stemChangedTotalFor: (token) => loaded2().stemChangedTotalFor(token),
|
|
12028
|
-
size: () => loaded2().size(),
|
|
12029
|
-
serialize: () => loaded2().serialize()
|
|
12030
|
-
};
|
|
12031
|
-
}
|
|
12032
13257
|
|
|
12033
13258
|
// src/core/partition/frozen/token-table.ts
|
|
12034
|
-
var
|
|
13259
|
+
var encoder4 = new TextEncoder();
|
|
12035
13260
|
var decoder3 = new TextDecoder();
|
|
12036
13261
|
function compareBytes2(blob, start, end, query) {
|
|
12037
13262
|
const length = end - start;
|
|
@@ -12118,7 +13343,7 @@ function wrapFrozenTokenTable(data) {
|
|
|
12118
13343
|
return documentFrequencies[sortedIndex];
|
|
12119
13344
|
},
|
|
12120
13345
|
prefixRange(prefix) {
|
|
12121
|
-
const bytes =
|
|
13346
|
+
const bytes = encoder4.encode(prefix);
|
|
12122
13347
|
const start = lowerBound3(bytes);
|
|
12123
13348
|
return { start, end: prefixEnd(start, bytes) };
|
|
12124
13349
|
},
|
|
@@ -12136,8 +13361,8 @@ function wrapFrozenTokenTable(data) {
|
|
|
12136
13361
|
fromCodePoint = unit;
|
|
12137
13362
|
toCodePoint = nextCodePoint(unit);
|
|
12138
13363
|
}
|
|
12139
|
-
const from =
|
|
12140
|
-
const to =
|
|
13364
|
+
const from = encoder4.encode(String.fromCodePoint(fromCodePoint));
|
|
13365
|
+
const to = encoder4.encode(String.fromCodePoint(toCodePoint));
|
|
12141
13366
|
return { start: lowerBound3(from), end: lowerBound3(to) };
|
|
12142
13367
|
}
|
|
12143
13368
|
};
|
|
@@ -12149,6 +13374,7 @@ function buildFrozenTokenTable(tokens, docFrequencies) {
|
|
|
12149
13374
|
// src/core/partition/frozen/tombstones.ts
|
|
12150
13375
|
function createFrozenTombstones() {
|
|
12151
13376
|
const removed = /* @__PURE__ */ new Set();
|
|
13377
|
+
const arrivals = [];
|
|
12152
13378
|
let revision = 0;
|
|
12153
13379
|
return {
|
|
12154
13380
|
get size() {
|
|
@@ -12163,24 +13389,19 @@ function createFrozenTombstones() {
|
|
|
12163
13389
|
add(internalId) {
|
|
12164
13390
|
if (removed.has(internalId)) return false;
|
|
12165
13391
|
removed.add(internalId);
|
|
13392
|
+
arrivals.push(internalId);
|
|
12166
13393
|
revision++;
|
|
12167
13394
|
return true;
|
|
12168
13395
|
},
|
|
13396
|
+
at(arrival) {
|
|
13397
|
+
return arrivals[arrival];
|
|
13398
|
+
},
|
|
12169
13399
|
values() {
|
|
12170
13400
|
return removed.values();
|
|
12171
13401
|
}
|
|
12172
13402
|
};
|
|
12173
13403
|
}
|
|
12174
13404
|
|
|
12175
|
-
// src/core/partition/partial-words/spellings.ts
|
|
12176
|
-
function sameFields(left, right) {
|
|
12177
|
-
if (left.length !== right.length) return false;
|
|
12178
|
-
for (let at = 0; at < left.length; at++) {
|
|
12179
|
-
if (left[at] !== right[at]) return false;
|
|
12180
|
-
}
|
|
12181
|
-
return true;
|
|
12182
|
-
}
|
|
12183
|
-
|
|
12184
13405
|
// src/core/partition/frozen/shared-snapshot.ts
|
|
12185
13406
|
function sharedUint32(source) {
|
|
12186
13407
|
const copy = new Uint32Array(new SharedArrayBuffer(source.length * 4));
|
|
@@ -12345,26 +13566,36 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis =
|
|
|
12345
13566
|
}
|
|
12346
13567
|
|
|
12347
13568
|
// src/core/partition/frozen/index.ts
|
|
12348
|
-
function buildStatsView(
|
|
12349
|
-
const averageFieldLengths = {};
|
|
12350
|
-
if (source.documentCount > 0) {
|
|
12351
|
-
for (const [fieldName, total] of Object.entries(source.totalFieldLengths)) {
|
|
12352
|
-
averageFieldLengths[fieldName] = total / source.documentCount;
|
|
12353
|
-
}
|
|
12354
|
-
}
|
|
12355
|
-
let materializedFrequencies = null;
|
|
13569
|
+
function buildStatsView(counts) {
|
|
12356
13570
|
return {
|
|
12357
|
-
totalDocuments
|
|
12358
|
-
|
|
12359
|
-
|
|
12360
|
-
get
|
|
12361
|
-
|
|
12362
|
-
|
|
13571
|
+
get totalDocuments() {
|
|
13572
|
+
return counts.totalDocuments();
|
|
13573
|
+
},
|
|
13574
|
+
get totalFieldLengths() {
|
|
13575
|
+
return counts.totalFieldLengths();
|
|
13576
|
+
},
|
|
13577
|
+
get averageFieldLengths() {
|
|
13578
|
+
const documents = counts.totalDocuments();
|
|
13579
|
+
const averages = {};
|
|
13580
|
+
if (documents > 0) {
|
|
13581
|
+
for (const [fieldName, total] of Object.entries(counts.totalFieldLengths())) {
|
|
13582
|
+
averages[fieldName] = total / documents;
|
|
13583
|
+
}
|
|
12363
13584
|
}
|
|
12364
|
-
return
|
|
13585
|
+
return averages;
|
|
13586
|
+
},
|
|
13587
|
+
get docFrequencies() {
|
|
13588
|
+
return counts.docFrequencies();
|
|
12365
13589
|
}
|
|
12366
13590
|
};
|
|
12367
13591
|
}
|
|
13592
|
+
function storedFrequenciesOf(source) {
|
|
13593
|
+
let materialized = null;
|
|
13594
|
+
return () => {
|
|
13595
|
+
if (materialized === null) materialized = source.docFrequencies();
|
|
13596
|
+
return materialized;
|
|
13597
|
+
};
|
|
13598
|
+
}
|
|
12368
13599
|
function buildGeoReaders(entries) {
|
|
12369
13600
|
const readers = /* @__PURE__ */ new Map();
|
|
12370
13601
|
for (const entry of entries) {
|
|
@@ -12376,9 +13607,10 @@ function buildGeoReaders(entries) {
|
|
|
12376
13607
|
}
|
|
12377
13608
|
return readers;
|
|
12378
13609
|
}
|
|
12379
|
-
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
|
|
13610
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
|
|
12380
13611
|
const tombstones = createFrozenTombstones();
|
|
12381
|
-
const
|
|
13612
|
+
const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
13613
|
+
const postingViews = createFrozenPostingViews(source, tombstones, counts);
|
|
12382
13614
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
12383
13615
|
for (let i = 0; i < source.fieldLengthNames.length; i++) {
|
|
12384
13616
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
@@ -12389,6 +13621,14 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
12389
13621
|
documentAt: (ordinal) => documentSource.readAt(ordinal),
|
|
12390
13622
|
isTombstoned: (ordinal) => tombstones.has(ordinal)
|
|
12391
13623
|
});
|
|
13624
|
+
const textDocuments = {
|
|
13625
|
+
count: documentSource.count,
|
|
13626
|
+
textFields: source.fieldLengthNames,
|
|
13627
|
+
documentAt: (ordinal) => documentSource.readAt(ordinal)
|
|
13628
|
+
};
|
|
13629
|
+
const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
|
|
13630
|
+
const surfaces = createFrozenSurfaceSlot(loadSurfaceForms, textDocuments);
|
|
13631
|
+
const removals = createFrozenRemovalPass(textDocuments, surfaces);
|
|
12392
13632
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
12393
13633
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
12394
13634
|
const sizeIndexes = /* @__PURE__ */ new Map();
|
|
@@ -12410,10 +13650,12 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
12410
13650
|
documentSource,
|
|
12411
13651
|
sharedSnapshot,
|
|
12412
13652
|
arrays: source,
|
|
12413
|
-
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews),
|
|
13653
|
+
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
|
|
12414
13654
|
docStore,
|
|
12415
|
-
stats: buildStatsView(
|
|
12416
|
-
surfaceRegistry:
|
|
13655
|
+
stats: buildStatsView(counts),
|
|
13656
|
+
surfaceRegistry: surfaces.reader,
|
|
13657
|
+
surfaces,
|
|
13658
|
+
removals,
|
|
12417
13659
|
numericIndexes,
|
|
12418
13660
|
booleanIndexes,
|
|
12419
13661
|
enumIndexes,
|
|
@@ -12436,15 +13678,21 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
12436
13678
|
isTombstoned(ordinal) {
|
|
12437
13679
|
return tombstones.has(ordinal);
|
|
12438
13680
|
},
|
|
12439
|
-
|
|
13681
|
+
removedSpellingsOf(docId) {
|
|
13682
|
+
const ordinal = docStore.getInternalId(docId);
|
|
13683
|
+
if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
|
|
13684
|
+
return removals.spellingsOf(ordinal);
|
|
13685
|
+
},
|
|
13686
|
+
tombstoneDocument(docId, removedSpellings) {
|
|
12440
13687
|
const ordinal = docStore.getInternalId(docId);
|
|
12441
13688
|
if (ordinal === void 0) return false;
|
|
12442
13689
|
const added = tombstones.add(ordinal);
|
|
12443
13690
|
if (added) {
|
|
13691
|
+
counts.subtractRemoved(ordinal);
|
|
12444
13692
|
partialWords.afterRemoval(ordinal);
|
|
13693
|
+
removals.afterRemoval(ordinal, removedSpellings);
|
|
12445
13694
|
docStore.releaseSortedDocIds();
|
|
12446
|
-
segment.sortColumns
|
|
12447
|
-
segment.scoreBuffer = null;
|
|
13695
|
+
segment.sortColumns?.forget(ordinal);
|
|
12448
13696
|
}
|
|
12449
13697
|
return added;
|
|
12450
13698
|
},
|
|
@@ -12452,6 +13700,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
12452
13700
|
const docIds = [];
|
|
12453
13701
|
for (const ordinal of tombstones.values()) docIds.push(source.idTable.idAt(ordinal));
|
|
12454
13702
|
return docIds;
|
|
13703
|
+
},
|
|
13704
|
+
removalBytes() {
|
|
13705
|
+
return counts.bytes();
|
|
12455
13706
|
}
|
|
12456
13707
|
};
|
|
12457
13708
|
return segment;
|
|
@@ -12473,7 +13724,7 @@ function createFrozenSegment(payload, documents, segmentId) {
|
|
|
12473
13724
|
payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, tokenTable)
|
|
12474
13725
|
);
|
|
12475
13726
|
}
|
|
12476
|
-
function createSharedFrozenSegment(snapshot) {
|
|
13727
|
+
function createSharedFrozenSegment(snapshot, survivingSurfaceForms) {
|
|
12477
13728
|
const tokenTable = wrapFrozenTokenTable(snapshot.tokenTable);
|
|
12478
13729
|
return assembleFrozenSegment(
|
|
12479
13730
|
snapshot.segmentId,
|
|
@@ -12493,136 +13744,10 @@ function createSharedFrozenSegment(snapshot) {
|
|
|
12493
13744
|
},
|
|
12494
13745
|
wrapEncodedDocumentTable(snapshot.documentTable),
|
|
12495
13746
|
snapshot,
|
|
12496
|
-
snapshot.partialWords
|
|
13747
|
+
snapshot.partialWords,
|
|
13748
|
+
survivingSurfaceForms
|
|
12497
13749
|
);
|
|
12498
13750
|
}
|
|
12499
|
-
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
12500
|
-
var keyDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
12501
|
-
function headerAt(view, offset) {
|
|
12502
|
-
if (offset >= view.byteLength) return null;
|
|
12503
|
-
const tag = view.getUint8(offset);
|
|
12504
|
-
if (tag <= 127 || tag >= 224) return { headerBytes: 1, bodyBytes: 0, children: 0 };
|
|
12505
|
-
if (tag <= 143) return { headerBytes: 1, bodyBytes: 0, children: (tag & 15) * 2 };
|
|
12506
|
-
if (tag <= 159) return { headerBytes: 1, bodyBytes: 0, children: tag & 15 };
|
|
12507
|
-
if (tag <= 191) return { headerBytes: 1, bodyBytes: tag & 31, children: 0 };
|
|
12508
|
-
const lengthBytes = LENGTH_BYTES_BY_TAG.get(tag);
|
|
12509
|
-
if (lengthBytes !== void 0) {
|
|
12510
|
-
if (offset + 1 + lengthBytes > view.byteLength) return null;
|
|
12511
|
-
const length = lengthBytes === 1 ? view.getUint8(offset + 1) : lengthBytes === 2 ? view.getUint16(offset + 1, false) : view.getUint32(offset + 1, false);
|
|
12512
|
-
const headerBytes = 1 + lengthBytes;
|
|
12513
|
-
if (ARRAY_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length };
|
|
12514
|
-
if (MAP_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length * 2 };
|
|
12515
|
-
return { headerBytes, bodyBytes: length + (EXT_TAGS.has(tag) ? 1 : 0), children: 0 };
|
|
12516
|
-
}
|
|
12517
|
-
const fixedBytes = FIXED_BODY_BYTES_BY_TAG.get(tag);
|
|
12518
|
-
return fixedBytes === void 0 ? null : { headerBytes: 1, bodyBytes: fixedBytes, children: 0 };
|
|
12519
|
-
}
|
|
12520
|
-
var LENGTH_BYTES_BY_TAG = /* @__PURE__ */ new Map([
|
|
12521
|
-
[196, 1],
|
|
12522
|
-
[197, 2],
|
|
12523
|
-
[198, 4],
|
|
12524
|
-
[199, 1],
|
|
12525
|
-
[200, 2],
|
|
12526
|
-
[201, 4],
|
|
12527
|
-
[217, 1],
|
|
12528
|
-
[218, 2],
|
|
12529
|
-
[219, 4],
|
|
12530
|
-
[220, 2],
|
|
12531
|
-
[221, 4],
|
|
12532
|
-
[222, 2],
|
|
12533
|
-
[223, 4]
|
|
12534
|
-
]);
|
|
12535
|
-
var ARRAY_TAGS = /* @__PURE__ */ new Set([220, 221]);
|
|
12536
|
-
var MAP_TAGS = /* @__PURE__ */ new Set([222, 223]);
|
|
12537
|
-
var EXT_TAGS = /* @__PURE__ */ new Set([199, 200, 201]);
|
|
12538
|
-
var FIXED_BODY_BYTES_BY_TAG = /* @__PURE__ */ new Map([
|
|
12539
|
-
[192, 0],
|
|
12540
|
-
[194, 0],
|
|
12541
|
-
[195, 0],
|
|
12542
|
-
[202, 4],
|
|
12543
|
-
[203, 8],
|
|
12544
|
-
[204, 1],
|
|
12545
|
-
[205, 2],
|
|
12546
|
-
[206, 4],
|
|
12547
|
-
[207, 8],
|
|
12548
|
-
[208, 1],
|
|
12549
|
-
[209, 2],
|
|
12550
|
-
[210, 4],
|
|
12551
|
-
[211, 8],
|
|
12552
|
-
[212, 2],
|
|
12553
|
-
[213, 3],
|
|
12554
|
-
[214, 5],
|
|
12555
|
-
[215, 9],
|
|
12556
|
-
[216, 17]
|
|
12557
|
-
]);
|
|
12558
|
-
function endOfValue(view, start) {
|
|
12559
|
-
let offset = start;
|
|
12560
|
-
let pending = 1;
|
|
12561
|
-
while (pending > 0) {
|
|
12562
|
-
const header = headerAt(view, offset);
|
|
12563
|
-
if (header === null) return null;
|
|
12564
|
-
offset += header.headerBytes + header.bodyBytes;
|
|
12565
|
-
if (offset > view.byteLength) return null;
|
|
12566
|
-
pending += header.children - 1;
|
|
12567
|
-
}
|
|
12568
|
-
return offset;
|
|
12569
|
-
}
|
|
12570
|
-
function stringKeyAt(bytes, view, offset) {
|
|
12571
|
-
const header = headerAt(view, offset);
|
|
12572
|
-
if (header === null || header.children !== 0) return null;
|
|
12573
|
-
const tag = view.getUint8(offset);
|
|
12574
|
-
const isString = tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
|
|
12575
|
-
const end = offset + header.headerBytes + header.bodyBytes;
|
|
12576
|
-
if (!isString || end > bytes.length) return null;
|
|
12577
|
-
try {
|
|
12578
|
-
return { key: keyDecoder.decode(bytes.subarray(offset + header.headerBytes, end)), end };
|
|
12579
|
-
} catch {
|
|
12580
|
-
return null;
|
|
12581
|
-
}
|
|
12582
|
-
}
|
|
12583
|
-
function topLevelMapKeys(bytes) {
|
|
12584
|
-
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
12585
|
-
const top = headerAt(view, 0);
|
|
12586
|
-
if (top === null || top.bodyBytes !== 0) return null;
|
|
12587
|
-
const tag = view.getUint8(0);
|
|
12588
|
-
if (!(tag >= 128 && tag <= 143 || MAP_TAGS.has(tag))) return null;
|
|
12589
|
-
const keys = /* @__PURE__ */ new Set();
|
|
12590
|
-
let offset = top.headerBytes;
|
|
12591
|
-
for (let pair = 0; pair < top.children / 2; pair += 1) {
|
|
12592
|
-
const key = stringKeyAt(bytes, view, offset);
|
|
12593
|
-
if (key === null) return null;
|
|
12594
|
-
const valueEnd = endOfValue(view, key.end);
|
|
12595
|
-
if (valueEnd === null) return null;
|
|
12596
|
-
keys.add(key.key);
|
|
12597
|
-
offset = valueEnd;
|
|
12598
|
-
}
|
|
12599
|
-
return keys;
|
|
12600
|
-
}
|
|
12601
|
-
function decodeMapWithoutFields(bytes, skippedFields) {
|
|
12602
|
-
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
12603
|
-
const top = headerAt(view, 0);
|
|
12604
|
-
if (top === null || top.children === 0 || top.bodyBytes !== 0) return null;
|
|
12605
|
-
const tag = view.getUint8(0);
|
|
12606
|
-
const isMap = tag >= 128 && tag <= 143 || MAP_TAGS.has(tag);
|
|
12607
|
-
if (!isMap) return null;
|
|
12608
|
-
const document = {};
|
|
12609
|
-
let offset = top.headerBytes;
|
|
12610
|
-
for (let pair = 0; pair < top.children / 2; pair += 1) {
|
|
12611
|
-
const key = stringKeyAt(bytes, view, offset);
|
|
12612
|
-
if (key === null || FORBIDDEN_KEYS.has(key.key)) return null;
|
|
12613
|
-
const valueEnd = endOfValue(view, key.end);
|
|
12614
|
-
if (valueEnd === null) return null;
|
|
12615
|
-
if (!skippedFields.has(key.key)) {
|
|
12616
|
-
try {
|
|
12617
|
-
document[key.key] = decode(bytes.subarray(key.end, valueEnd));
|
|
12618
|
-
} catch {
|
|
12619
|
-
return null;
|
|
12620
|
-
}
|
|
12621
|
-
}
|
|
12622
|
-
offset = valueEnd;
|
|
12623
|
-
}
|
|
12624
|
-
return offset === bytes.length ? document : null;
|
|
12625
|
-
}
|
|
12626
13751
|
|
|
12627
13752
|
// src/core/partition/frozen/merge-fields.ts
|
|
12628
13753
|
function indexByFieldPath(entries) {
|
|
@@ -13014,16 +14139,9 @@ function mergeFieldLengths(inputs, documentCount) {
|
|
|
13014
14139
|
}
|
|
13015
14140
|
return { fieldLengthNames: names, fieldLengthColumns: columns, totalFieldLengths: totals };
|
|
13016
14141
|
}
|
|
13017
|
-
function mergeSurfaceForms2(inputs) {
|
|
14142
|
+
function mergeSurfaceForms2(inputs, analysis) {
|
|
13018
14143
|
const registry = createSurfaceRegistry();
|
|
13019
|
-
for (const input of inputs)
|
|
13020
|
-
const forms = input.segment.arrays.readSurfaceForms();
|
|
13021
|
-
if (forms === null) continue;
|
|
13022
|
-
for (const surface of Object.keys(forms)) {
|
|
13023
|
-
const value = forms[surface];
|
|
13024
|
-
if (Array.isArray(value)) registry.add(surface, value[1], value[0]);
|
|
13025
|
-
}
|
|
13026
|
-
}
|
|
14144
|
+
for (const input of inputs) input.segment.surfaces.addKeptForms(registry, input.remap, analysis);
|
|
13027
14145
|
return registry.size() === 0 ? null : registry.serialize();
|
|
13028
14146
|
}
|
|
13029
14147
|
function holdsADiscardedField(bytes, discardedFields) {
|
|
@@ -13093,15 +14211,16 @@ function mergePartialWords(inputs, mergedTokenCount, analysis) {
|
|
|
13093
14211
|
for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
|
|
13094
14212
|
return { spellings: encodePartialSpellingsOf(analysis.fields, counts), reused: null };
|
|
13095
14213
|
}
|
|
13096
|
-
function mergeFrozenSegments(segments, analysis) {
|
|
14214
|
+
function mergeFrozenSegments(segments, analysis, textAnalysis) {
|
|
13097
14215
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
13098
14216
|
const merged = mergeSegmentContents(
|
|
13099
14217
|
segments.map((segment) => ({ segment })),
|
|
13100
|
-
analysis
|
|
14218
|
+
analysis,
|
|
14219
|
+
textAnalysis
|
|
13101
14220
|
);
|
|
13102
14221
|
return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis, merged.reusedPartialWords);
|
|
13103
14222
|
}
|
|
13104
|
-
function mergeSegmentContents(selections, analysis = null) {
|
|
14223
|
+
function mergeSegmentContents(selections, analysis = null, textAnalysis = null) {
|
|
13105
14224
|
const docIds = [];
|
|
13106
14225
|
const fieldNames = [];
|
|
13107
14226
|
const inputs = selections.map((selection) => ({
|
|
@@ -13128,7 +14247,7 @@ function mergeSegmentContents(selections, analysis = null) {
|
|
|
13128
14247
|
fieldLengthColumns: lengths.fieldLengthColumns,
|
|
13129
14248
|
totalFieldLengths: lengths.totalFieldLengths,
|
|
13130
14249
|
docFrequencies: postings.docFrequencies,
|
|
13131
|
-
surfaceForms: mergeSurfaceForms2(inputs),
|
|
14250
|
+
surfaceForms: mergeSurfaceForms2(inputs, textAnalysis),
|
|
13132
14251
|
partialSpellings: partialWords.spellings,
|
|
13133
14252
|
...mergeFieldIndexes2(inputs)
|
|
13134
14253
|
};
|
|
@@ -13530,22 +14649,29 @@ function runSegmentAction(entry, action) {
|
|
|
13530
14649
|
case "attachSegments": {
|
|
13531
14650
|
for (const segment of action.segments) {
|
|
13532
14651
|
growPartitionsTo(entry.manager, segment.partitionId);
|
|
13533
|
-
const
|
|
13534
|
-
|
|
14652
|
+
const surfaceForms = segment.survivingSurfaceForms;
|
|
14653
|
+
const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
14654
|
+
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
14655
|
+
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
13535
14656
|
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
13536
14657
|
}
|
|
13537
14658
|
return void 0;
|
|
13538
14659
|
}
|
|
13539
14660
|
case "compactSegments": {
|
|
13540
14661
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
13541
|
-
return mergeFrozenSegments(
|
|
14662
|
+
return mergeFrozenSegments(
|
|
14663
|
+
partition.frozenSegmentsById(action.segmentIds),
|
|
14664
|
+
entry.manager.partialWordAnalysis,
|
|
14665
|
+
entry.manager.textAnalysis
|
|
14666
|
+
);
|
|
13542
14667
|
}
|
|
13543
14668
|
case "swapSegments": {
|
|
13544
14669
|
requireComposite(entry, action.indexName, action.partitionId, "swap");
|
|
13545
14670
|
entry.manager.swapFrozenSegments(
|
|
13546
14671
|
action.partitionId,
|
|
13547
14672
|
action.dropSegmentIds,
|
|
13548
|
-
createSharedFrozenSegment(action.snapshot)
|
|
14673
|
+
createSharedFrozenSegment(action.snapshot),
|
|
14674
|
+
action.removedSpellings
|
|
13549
14675
|
);
|
|
13550
14676
|
return void 0;
|
|
13551
14677
|
}
|
|
@@ -13829,12 +14955,12 @@ function createDirectExecutor(options) {
|
|
|
13829
14955
|
}
|
|
13830
14956
|
case "remove": {
|
|
13831
14957
|
const entry = requireIndex(action.indexName);
|
|
13832
|
-
entry.manager.remove(action.docId);
|
|
14958
|
+
entry.manager.remove(action.docId, action.removedSpellings);
|
|
13833
14959
|
return void 0;
|
|
13834
14960
|
}
|
|
13835
14961
|
case "update": {
|
|
13836
14962
|
const entry = requireIndex(action.indexName);
|
|
13837
|
-
entry.manager.update(action.docId, action.document);
|
|
14963
|
+
entry.manager.update(action.docId, action.document, void 0, action.removedSpellings);
|
|
13838
14964
|
return void 0;
|
|
13839
14965
|
}
|
|
13840
14966
|
case "query": {
|
|
@@ -15286,7 +16412,15 @@ function checkHeapAfterRecovery(core) {
|
|
|
15286
16412
|
// src/core/partition/frozen/share.ts
|
|
15287
16413
|
function shareFrozenSegment(segment, analysis) {
|
|
15288
16414
|
if (segment.sharedSnapshot !== null) {
|
|
15289
|
-
|
|
16415
|
+
const tombstonedDocIds = segment.tombstonedDocIds();
|
|
16416
|
+
if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
|
|
16417
|
+
return { snapshot: segment.sharedSnapshot, tombstonedDocIds };
|
|
16418
|
+
}
|
|
16419
|
+
return {
|
|
16420
|
+
snapshot: segment.sharedSnapshot,
|
|
16421
|
+
tombstonedDocIds,
|
|
16422
|
+
survivingSurfaceForms: segment.surfaces.reader.serialize()
|
|
16423
|
+
};
|
|
15290
16424
|
}
|
|
15291
16425
|
const payload = encodeSegmentState(segment);
|
|
15292
16426
|
const documents = payload.docIds.map((docId) => {
|
|
@@ -15304,6 +16438,35 @@ function shareFrozenSegment(segment, analysis) {
|
|
|
15304
16438
|
|
|
15305
16439
|
// src/engine/worker-resync.ts
|
|
15306
16440
|
var copyLoadSequence = 0;
|
|
16441
|
+
function emptySerializablePartition(manager, partitionId) {
|
|
16442
|
+
return {
|
|
16443
|
+
indexName: manager.indexName,
|
|
16444
|
+
partitionId,
|
|
16445
|
+
totalPartitions: manager.partitionCount,
|
|
16446
|
+
language: manager.language.name,
|
|
16447
|
+
schema: {},
|
|
16448
|
+
docCount: 0,
|
|
16449
|
+
avgDocLength: 0,
|
|
16450
|
+
documents: {},
|
|
16451
|
+
invertedIndex: {},
|
|
16452
|
+
fieldIndexes: { numeric: {}, boolean: {}, enum: {}, geopoint: {} },
|
|
16453
|
+
statistics: { totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} }
|
|
16454
|
+
};
|
|
16455
|
+
}
|
|
16456
|
+
function captureLiveTail(partition) {
|
|
16457
|
+
if (partition.count() === 0) return null;
|
|
16458
|
+
const payload = partition.encodeSegment();
|
|
16459
|
+
const documents = payload.docIds.map((docId) => {
|
|
16460
|
+
const document = partition.getRef(docId);
|
|
16461
|
+
if (document === void 0) {
|
|
16462
|
+
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while copying a partition`, {
|
|
16463
|
+
docId
|
|
16464
|
+
});
|
|
16465
|
+
}
|
|
16466
|
+
return document;
|
|
16467
|
+
});
|
|
16468
|
+
return { payload, documents };
|
|
16469
|
+
}
|
|
15307
16470
|
function captureComposite(manager, partitionId) {
|
|
15308
16471
|
const partition = manager.getPartition(partitionId);
|
|
15309
16472
|
if (!isCompositePartition(partition)) return null;
|
|
@@ -15314,13 +16477,7 @@ function captureComposite(manager, partitionId) {
|
|
|
15314
16477
|
if (shared2 === null) return null;
|
|
15315
16478
|
frozen.push(shared2);
|
|
15316
16479
|
}
|
|
15317
|
-
|
|
15318
|
-
manager.indexName,
|
|
15319
|
-
manager.partitionCount,
|
|
15320
|
-
manager.language.name,
|
|
15321
|
-
manager.schema
|
|
15322
|
-
);
|
|
15323
|
-
return { partitionId, live, frozen };
|
|
16480
|
+
return { partitionId, live: captureLiveTail(partition.live), frozen };
|
|
15324
16481
|
}
|
|
15325
16482
|
function capturePartitions(manager) {
|
|
15326
16483
|
const captured = [];
|
|
@@ -15328,30 +16485,45 @@ function capturePartitions(manager) {
|
|
|
15328
16485
|
captured.push(
|
|
15329
16486
|
captureComposite(manager, partitionId) ?? {
|
|
15330
16487
|
partitionId,
|
|
15331
|
-
live: manager.
|
|
16488
|
+
live: captureLiveTail(manager.getPartition(partitionId)),
|
|
15332
16489
|
frozen: []
|
|
15333
16490
|
}
|
|
15334
16491
|
);
|
|
15335
16492
|
}
|
|
15336
16493
|
return captured;
|
|
15337
16494
|
}
|
|
15338
|
-
async function sendPartition(
|
|
16495
|
+
async function sendPartition(manager, executors, captured) {
|
|
16496
|
+
const { indexName } = manager;
|
|
15339
16497
|
await Promise.all(
|
|
15340
16498
|
executors.map(
|
|
15341
16499
|
(workerExecutor) => workerExecutor.execute({
|
|
15342
16500
|
type: "deserialize",
|
|
15343
16501
|
indexName,
|
|
15344
16502
|
partitionId: captured.partitionId,
|
|
15345
|
-
data: captured.
|
|
16503
|
+
data: emptySerializablePartition(manager, captured.partitionId),
|
|
15346
16504
|
requestId: `resync-${indexName}-${captured.partitionId}`
|
|
15347
16505
|
})
|
|
15348
16506
|
)
|
|
15349
16507
|
);
|
|
16508
|
+
const live = captured.live;
|
|
16509
|
+
if (live !== null) {
|
|
16510
|
+
await Promise.all(
|
|
16511
|
+
executors.map(
|
|
16512
|
+
(workerExecutor) => workerExecutor.execute({
|
|
16513
|
+
type: "mergeSegments",
|
|
16514
|
+
indexName,
|
|
16515
|
+
segments: [{ partitionId: captured.partitionId, payload: live.payload, documents: live.documents }],
|
|
16516
|
+
requestId: `resync-live-${indexName}-${captured.partitionId}`
|
|
16517
|
+
})
|
|
16518
|
+
)
|
|
16519
|
+
);
|
|
16520
|
+
}
|
|
15350
16521
|
if (captured.frozen.length === 0) return;
|
|
15351
16522
|
const segments = captured.frozen.map((shared2) => ({
|
|
15352
16523
|
partitionId: captured.partitionId,
|
|
15353
16524
|
snapshot: shared2.snapshot,
|
|
15354
|
-
tombstonedDocIds: shared2.tombstonedDocIds
|
|
16525
|
+
tombstonedDocIds: shared2.tombstonedDocIds,
|
|
16526
|
+
survivingSurfaceForms: shared2.survivingSurfaceForms
|
|
15355
16527
|
}));
|
|
15356
16528
|
await Promise.all(
|
|
15357
16529
|
executors.map(
|
|
@@ -15397,7 +16569,7 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
|
|
|
15397
16569
|
)
|
|
15398
16570
|
);
|
|
15399
16571
|
for (const partition of captured) {
|
|
15400
|
-
await sendPartition(
|
|
16572
|
+
await sendPartition(manager, executors, partition);
|
|
15401
16573
|
}
|
|
15402
16574
|
return copyLoad;
|
|
15403
16575
|
}
|
|
@@ -16015,21 +17187,24 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
|
|
|
16015
17187
|
const mainCanMerge = composite !== null && mainHoldsAll(composite, segmentIds);
|
|
16016
17188
|
let snapshot = await compactOnWorker(state, indexName, partitionId, segmentIds);
|
|
16017
17189
|
if (snapshot === null && mainCanMerge && composite !== null) {
|
|
16018
|
-
snapshot = mergeFrozenSegments(
|
|
17190
|
+
snapshot = mergeFrozenSegments(
|
|
17191
|
+
composite.frozenSegmentsById(segmentIds),
|
|
17192
|
+
manager.partialWordAnalysis,
|
|
17193
|
+
manager.textAnalysis
|
|
17194
|
+
);
|
|
16019
17195
|
}
|
|
16020
17196
|
if (snapshot === null) {
|
|
16021
17197
|
if (!mainCanMerge) forgetLedgerSegments(state, indexName, partitionId, segmentIds);
|
|
16022
17198
|
return false;
|
|
16023
17199
|
}
|
|
16024
|
-
|
|
16025
|
-
composite.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot));
|
|
16026
|
-
}
|
|
17200
|
+
const removedSpellings = mainCanMerge && composite !== null ? composite.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot)) : void 0;
|
|
16027
17201
|
await replicateToWorkers(state, {
|
|
16028
17202
|
type: "swapSegments",
|
|
16029
17203
|
indexName,
|
|
16030
17204
|
partitionId,
|
|
16031
17205
|
dropSegmentIds: [...segmentIds],
|
|
16032
17206
|
snapshot,
|
|
17207
|
+
removedSpellings,
|
|
16033
17208
|
requestId: createRequestId()
|
|
16034
17209
|
});
|
|
16035
17210
|
return true;
|
|
@@ -17533,7 +18708,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
17533
18708
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
17534
18709
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
17535
18710
|
}
|
|
17536
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
18711
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-VSUBLUDK.mjs');
|
|
17537
18712
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
17538
18713
|
const executor = createDirectExecutor({
|
|
17539
18714
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19597,16 +20772,18 @@ async function prepareRemoval(ctx, indexName, docId) {
|
|
|
19597
20772
|
if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
|
|
19598
20773
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
19599
20774
|
}
|
|
19600
|
-
const
|
|
20775
|
+
const manager = ctx.requireManager(indexName);
|
|
20776
|
+
const restoreRef = ctx.durability && !rebalancing ? manager.getRef(docId) : void 0;
|
|
19601
20777
|
return {
|
|
19602
20778
|
docId,
|
|
19603
20779
|
document: null,
|
|
19604
20780
|
restoreDoc: restoreRef ? structuredClone(restoreRef) : void 0,
|
|
20781
|
+
removedSpellings: manager.removedSpellingsOf(docId),
|
|
19605
20782
|
buffered: false
|
|
19606
20783
|
};
|
|
19607
20784
|
}
|
|
19608
20785
|
function applyOfRemoval(ctx, indexName, prepared) {
|
|
19609
|
-
const { docId } = prepared;
|
|
20786
|
+
const { docId, removedSpellings } = prepared;
|
|
19610
20787
|
return async () => {
|
|
19611
20788
|
if (ctx.isRebalancing(indexName)) {
|
|
19612
20789
|
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
@@ -19619,11 +20796,12 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
19619
20796
|
prepared.buffered = true;
|
|
19620
20797
|
return;
|
|
19621
20798
|
}
|
|
19622
|
-
await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId });
|
|
20799
|
+
await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId, removedSpellings });
|
|
19623
20800
|
removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
|
|
19624
20801
|
};
|
|
19625
20802
|
}
|
|
19626
|
-
async function settleRemoval(ctx, indexName,
|
|
20803
|
+
async function settleRemoval(ctx, indexName, prepared) {
|
|
20804
|
+
const { docId, removedSpellings } = prepared;
|
|
19627
20805
|
try {
|
|
19628
20806
|
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
19629
20807
|
} catch (err) {
|
|
@@ -19633,7 +20811,8 @@ async function settleRemoval(ctx, indexName, docId) {
|
|
|
19633
20811
|
type: "remove",
|
|
19634
20812
|
indexName,
|
|
19635
20813
|
docId,
|
|
19636
|
-
requestId: `replicate-remove-${docId}
|
|
20814
|
+
requestId: `replicate-remove-${docId}`,
|
|
20815
|
+
removedSpellings
|
|
19637
20816
|
});
|
|
19638
20817
|
}
|
|
19639
20818
|
async function removeDocument(ctx, indexName, docId, options) {
|
|
@@ -19651,7 +20830,7 @@ async function removeDocument(ctx, indexName, docId, options) {
|
|
|
19651
20830
|
} else {
|
|
19652
20831
|
await apply();
|
|
19653
20832
|
}
|
|
19654
|
-
if (!prepared.buffered) await settleRemoval(ctx, indexName,
|
|
20833
|
+
if (!prepared.buffered) await settleRemoval(ctx, indexName, prepared);
|
|
19655
20834
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
19656
20835
|
}
|
|
19657
20836
|
async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
@@ -19687,7 +20866,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
19687
20866
|
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
19688
20867
|
continue;
|
|
19689
20868
|
}
|
|
19690
|
-
if (!removal.buffered) await settleRemoval(ctx, indexName, removal
|
|
20869
|
+
if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
|
|
19691
20870
|
succeeded.push(removal.docId);
|
|
19692
20871
|
}
|
|
19693
20872
|
if (chunkEnd < docIds.length) {
|
|
@@ -19735,7 +20914,8 @@ async function settleUpdate(ctx, indexName, prepared) {
|
|
|
19735
20914
|
indexName,
|
|
19736
20915
|
docId: prepared.docId,
|
|
19737
20916
|
document: prepared.partitionDoc,
|
|
19738
|
-
requestId: `replicate-update-${prepared.docId}
|
|
20917
|
+
requestId: `replicate-update-${prepared.docId}`,
|
|
20918
|
+
removedSpellings: prepared.removedSpellings
|
|
19739
20919
|
});
|
|
19740
20920
|
const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
|
|
19741
20921
|
for (const [fieldPath, vec] of prepared.extractedVectors) {
|
|
@@ -19802,12 +20982,13 @@ async function prepareUpdate(ctx, indexName, entry, docId, document) {
|
|
|
19802
20982
|
oldDocument,
|
|
19803
20983
|
rollbackDoc,
|
|
19804
20984
|
extractedVectors,
|
|
20985
|
+
removedSpellings: manager.removedSpellingsOf(docId),
|
|
19805
20986
|
buffered: false
|
|
19806
20987
|
};
|
|
19807
20988
|
}
|
|
19808
20989
|
function applyOfUpdate(ctx, indexName, prepared) {
|
|
19809
20990
|
const manager = ctx.requireManager(indexName);
|
|
19810
|
-
const { docId, document, partitionDoc, rollbackDoc, extractedVectors } = prepared;
|
|
20991
|
+
const { docId, document, partitionDoc, rollbackDoc, extractedVectors, removedSpellings } = prepared;
|
|
19811
20992
|
return async () => {
|
|
19812
20993
|
if (ctx.isRebalancing(indexName)) {
|
|
19813
20994
|
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
@@ -19820,7 +21001,14 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
19820
21001
|
prepared.buffered = true;
|
|
19821
21002
|
return;
|
|
19822
21003
|
}
|
|
19823
|
-
await ctx.executor.execute({
|
|
21004
|
+
await ctx.executor.execute({
|
|
21005
|
+
type: "update",
|
|
21006
|
+
indexName,
|
|
21007
|
+
docId,
|
|
21008
|
+
document: partitionDoc,
|
|
21009
|
+
requestId: docId,
|
|
21010
|
+
removedSpellings
|
|
21011
|
+
});
|
|
19824
21012
|
try {
|
|
19825
21013
|
updateDocumentVectors(docId, extractedVectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
|
|
19826
21014
|
} catch (err) {
|
|
@@ -20005,9 +21193,10 @@ async function replayEntry(manager, indexName, entry, ctx) {
|
|
|
20005
21193
|
vectorFieldPaths2
|
|
20006
21194
|
);
|
|
20007
21195
|
const overwrote = manager.has(entry.docId);
|
|
21196
|
+
const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
|
|
20008
21197
|
if (overwrote) {
|
|
20009
21198
|
removeDocumentVectors(entry.docId, vecIndexes);
|
|
20010
|
-
manager.remove(entry.docId);
|
|
21199
|
+
manager.remove(entry.docId, removedSpellings);
|
|
20011
21200
|
}
|
|
20012
21201
|
manager.insert(entry.docId, partitionDoc);
|
|
20013
21202
|
if (extractedVectors.size > 0) {
|
|
@@ -20021,17 +21210,20 @@ async function replayEntry(manager, indexName, entry, ctx) {
|
|
|
20021
21210
|
} catch (err) {
|
|
20022
21211
|
warnHookError("afterInsert", err);
|
|
20023
21212
|
}
|
|
20024
|
-
|
|
20025
|
-
type: overwrote ? "update" : "insert",
|
|
21213
|
+
const replicated = {
|
|
20026
21214
|
indexName,
|
|
20027
21215
|
docId: entry.docId,
|
|
20028
21216
|
document: entry.document,
|
|
20029
21217
|
requestId: `replicate-insert-${entry.docId}`
|
|
20030
|
-
}
|
|
21218
|
+
};
|
|
21219
|
+
await ctx.orchestrator.replicateToWorkers(
|
|
21220
|
+
overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
|
|
21221
|
+
);
|
|
20031
21222
|
return;
|
|
20032
21223
|
}
|
|
20033
21224
|
if (entry.action === "remove") {
|
|
20034
|
-
manager.
|
|
21225
|
+
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
21226
|
+
manager.remove(entry.docId, removedSpellings);
|
|
20035
21227
|
removeDocumentVectors(entry.docId, vecIndexes);
|
|
20036
21228
|
try {
|
|
20037
21229
|
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
|
|
@@ -20042,15 +21234,17 @@ async function replayEntry(manager, indexName, entry, ctx) {
|
|
|
20042
21234
|
type: "remove",
|
|
20043
21235
|
indexName,
|
|
20044
21236
|
docId: entry.docId,
|
|
20045
|
-
requestId: `replicate-remove-${entry.docId}
|
|
21237
|
+
requestId: `replicate-remove-${entry.docId}`,
|
|
21238
|
+
removedSpellings
|
|
20046
21239
|
});
|
|
20047
21240
|
return;
|
|
20048
21241
|
}
|
|
20049
21242
|
if (entry.action === "update" && entry.document) {
|
|
20050
21243
|
const oldDocument = manager.get(entry.docId);
|
|
20051
21244
|
const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths2);
|
|
21245
|
+
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
20052
21246
|
if (manager.has(entry.docId)) {
|
|
20053
|
-
manager.update(entry.docId, updateDoc);
|
|
21247
|
+
manager.update(entry.docId, updateDoc, void 0, removedSpellings);
|
|
20054
21248
|
} else {
|
|
20055
21249
|
manager.insert(entry.docId, updateDoc);
|
|
20056
21250
|
}
|
|
@@ -20087,7 +21281,8 @@ async function replayEntry(manager, indexName, entry, ctx) {
|
|
|
20087
21281
|
indexName,
|
|
20088
21282
|
docId: entry.docId,
|
|
20089
21283
|
document: entry.document,
|
|
20090
|
-
requestId: `replicate-update-${entry.docId}
|
|
21284
|
+
requestId: `replicate-update-${entry.docId}`,
|
|
21285
|
+
removedSpellings
|
|
20091
21286
|
});
|
|
20092
21287
|
}
|
|
20093
21288
|
}
|
|
@@ -20908,7 +22103,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
20908
22103
|
};
|
|
20909
22104
|
}
|
|
20910
22105
|
async function restoreIndex(indexName, data) {
|
|
20911
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
22106
|
+
const { restoreFromSnapshot } = await import('./snapshot-5MXBEDXF.mjs');
|
|
20912
22107
|
await restoreFromSnapshot(indexName, data, {
|
|
20913
22108
|
executor: core.executor,
|
|
20914
22109
|
indexRegistry: core.indexRegistry,
|
|
@@ -20928,7 +22123,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
20928
22123
|
return {
|
|
20929
22124
|
snapshot(indexName) {
|
|
20930
22125
|
return withOpenIndex(indexName, async () => {
|
|
20931
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22126
|
+
const { createSnapshot } = await import('./snapshot-5MXBEDXF.mjs');
|
|
20932
22127
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
20933
22128
|
});
|
|
20934
22129
|
},
|
|
@@ -20940,8 +22135,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
20940
22135
|
exportDocuments(indexName) {
|
|
20941
22136
|
return withOpenIndex(indexName, async () => {
|
|
20942
22137
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
20943
|
-
import('./snapshot-
|
|
20944
|
-
import('./write-
|
|
22138
|
+
import('./snapshot-5MXBEDXF.mjs'),
|
|
22139
|
+
import('./write-6SBYTS62.mjs')
|
|
20945
22140
|
]);
|
|
20946
22141
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
20947
22142
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -20965,7 +22160,7 @@ var LIST_TAG = 6;
|
|
|
20965
22160
|
var MAP_TAG = 7;
|
|
20966
22161
|
var LENGTH_BYTES = 4;
|
|
20967
22162
|
var LAST_ASCII_UNIT = 127;
|
|
20968
|
-
var
|
|
22163
|
+
var encoder5 = new TextEncoder();
|
|
20969
22164
|
function ensureCapacity(stream, extra) {
|
|
20970
22165
|
if (stream.length + extra <= stream.bytes.length) return;
|
|
20971
22166
|
const grown = new Uint8Array(Math.max(stream.bytes.length * 2, stream.length + extra));
|
|
@@ -20995,7 +22190,7 @@ function writeNumber(stream, value) {
|
|
|
20995
22190
|
stream.length += 8;
|
|
20996
22191
|
}
|
|
20997
22192
|
function writeEncodedString(stream, value) {
|
|
20998
|
-
const encoded =
|
|
22193
|
+
const encoded = encoder5.encode(value);
|
|
20999
22194
|
writeUint32(stream, encoded.length);
|
|
21000
22195
|
ensureCapacity(stream, encoded.length);
|
|
21001
22196
|
stream.bytes.set(encoded, stream.length);
|
|
@@ -21163,18 +22358,27 @@ function requireFacetableFields(facets, flatSchema, strict) {
|
|
|
21163
22358
|
}
|
|
21164
22359
|
}
|
|
21165
22360
|
}
|
|
21166
|
-
function
|
|
22361
|
+
function requireGroupableFields(fields, flatSchema, strict) {
|
|
21167
22362
|
for (const field of fields) {
|
|
21168
|
-
|
|
22363
|
+
const fieldType = flatSchema[field];
|
|
22364
|
+
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
|
|
22365
|
+
if (fieldType !== void 0 && VECTOR_PATTERN.test(fieldType)) {
|
|
22366
|
+
throw new NarsilError(
|
|
22367
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
22368
|
+
`The engine groups hits by a text, number, boolean, enum, or geopoint field, and the schema declares field "${field}" as "${fieldType}"`,
|
|
22369
|
+
{ field, fieldType }
|
|
22370
|
+
);
|
|
22371
|
+
}
|
|
21169
22372
|
}
|
|
21170
22373
|
}
|
|
21171
22374
|
function requireUsableQueryFields(params, schema, strict = false) {
|
|
21172
|
-
const
|
|
22375
|
+
const groupFields = params.group?.fields;
|
|
22376
|
+
const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
|
|
21173
22377
|
if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
|
|
21174
22378
|
const flatSchema = flattenSchema(schema);
|
|
21175
22379
|
if (params.boost !== void 0) requireBoostableFields(params.boost, flatSchema);
|
|
21176
22380
|
if (params.facets !== void 0) requireFacetableFields(params.facets, flatSchema, strict);
|
|
21177
|
-
if (checksGroup
|
|
22381
|
+
if (checksGroup) requireGroupableFields(groupFields, flatSchema, strict);
|
|
21178
22382
|
}
|
|
21179
22383
|
|
|
21180
22384
|
// src/search/typo-tolerance.ts
|
|
@@ -21420,6 +22624,7 @@ function executeListDocuments(params, context) {
|
|
|
21420
22624
|
const signature = sortSignatureOf(params.sort);
|
|
21421
22625
|
requireSortableFields(params.sort, schema, context.strict === true);
|
|
21422
22626
|
const filters = validQueryFilter(params.filters, schema);
|
|
22627
|
+
const projection = resolveProjection(params.document);
|
|
21423
22628
|
const binding = listBindingOf(params.filters);
|
|
21424
22629
|
let cursor = null;
|
|
21425
22630
|
if (params.cursor !== void 0) {
|
|
@@ -21430,7 +22635,6 @@ function executeListDocuments(params, context) {
|
|
|
21430
22635
|
const filtered = filters === void 0 ? null : collectFilterMatches(partitions, filters, schema, context.patternWorkCap);
|
|
21431
22636
|
const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
|
|
21432
22637
|
const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
|
|
21433
|
-
const projection = resolveProjection(params.document);
|
|
21434
22638
|
const documents = [];
|
|
21435
22639
|
for (const docId of page.ids) {
|
|
21436
22640
|
if (projection.kind === "none") {
|
|
@@ -21453,37 +22657,23 @@ function executeListDocuments(params, context) {
|
|
|
21453
22657
|
}
|
|
21454
22658
|
|
|
21455
22659
|
// src/search/grouping.ts
|
|
21456
|
-
function getNestedValue2(obj, path) {
|
|
21457
|
-
const segments = path.split(".");
|
|
21458
|
-
let current = obj;
|
|
21459
|
-
for (const segment of segments) {
|
|
21460
|
-
if (current === null || current === void 0 || typeof current !== "object") {
|
|
21461
|
-
return void 0;
|
|
21462
|
-
}
|
|
21463
|
-
if (!Object.hasOwn(current, segment)) {
|
|
21464
|
-
return void 0;
|
|
21465
|
-
}
|
|
21466
|
-
current = current[segment];
|
|
21467
|
-
}
|
|
21468
|
-
return current;
|
|
21469
|
-
}
|
|
21470
22660
|
function hitsKeptPerGroup(maxPerGroup) {
|
|
21471
22661
|
if (maxPerGroup === void 0 || !Number.isFinite(maxPerGroup) || maxPerGroup < 1) return DEFAULT_MAX_PER_GROUP;
|
|
21472
22662
|
return Math.floor(maxPerGroup);
|
|
21473
22663
|
}
|
|
21474
|
-
function applyGrouping(hits, group,
|
|
22664
|
+
function applyGrouping(hits, group, documents) {
|
|
21475
22665
|
if (!group.fields || group.fields.length === 0) {
|
|
21476
22666
|
return [{ values: {}, hits }];
|
|
21477
22667
|
}
|
|
21478
22668
|
const groupMap = /* @__PURE__ */ new Map();
|
|
21479
22669
|
for (const hit of hits) {
|
|
21480
|
-
const
|
|
22670
|
+
const stored = documents.fieldValues(hit.id, group.fields);
|
|
21481
22671
|
const fieldValues = [];
|
|
21482
22672
|
const valuesRecord = {};
|
|
21483
|
-
for (
|
|
21484
|
-
const value =
|
|
22673
|
+
for (let at = 0; at < group.fields.length; at++) {
|
|
22674
|
+
const value = stored?.[at];
|
|
21485
22675
|
fieldValues.push(value);
|
|
21486
|
-
valuesRecord[
|
|
22676
|
+
valuesRecord[group.fields[at]] = value;
|
|
21487
22677
|
}
|
|
21488
22678
|
const compositeKey = JSON.stringify(fieldValues);
|
|
21489
22679
|
const existing = groupMap.get(compositeKey);
|
|
@@ -21501,7 +22691,7 @@ function applyGrouping(hits, group, getDocument) {
|
|
|
21501
22691
|
if (group.reduce) {
|
|
21502
22692
|
const folded = foldGroupReducer(
|
|
21503
22693
|
group.reduce,
|
|
21504
|
-
everyHit.map((hit) => ({ document:
|
|
22694
|
+
everyHit.map((hit) => ({ document: documents.document(hit.id), score: hit.score ?? 0 }))
|
|
21505
22695
|
);
|
|
21506
22696
|
if (folded.reducerError !== void 0) {
|
|
21507
22697
|
result.reducerError = folded.reducerError;
|
|
@@ -22149,6 +23339,7 @@ async function executeQuery(request, callerContext) {
|
|
|
22149
23339
|
}
|
|
22150
23340
|
const params = validQueryParams(request, config);
|
|
22151
23341
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
23342
|
+
const projection = resolveProjection(params.document);
|
|
22152
23343
|
let paginated;
|
|
22153
23344
|
let nextCursor;
|
|
22154
23345
|
let count;
|
|
@@ -22232,7 +23423,10 @@ async function executeQuery(request, callerContext) {
|
|
|
22232
23423
|
hits = applySorting(hits, params.sort, sortKeyOf);
|
|
22233
23424
|
}
|
|
22234
23425
|
if (params.group) {
|
|
22235
|
-
groups = applyGrouping(hits, params.group,
|
|
23426
|
+
groups = applyGrouping(hits, params.group, {
|
|
23427
|
+
fieldValues: (docId, fieldPaths) => manager.fieldValues(docId, fieldPaths),
|
|
23428
|
+
document: (docId) => manager.getRef(docId)
|
|
23429
|
+
});
|
|
22236
23430
|
}
|
|
22237
23431
|
let pinsFromOutside = 0;
|
|
22238
23432
|
if (params.pinned && params.searchAfter === void 0 && context.partitionIds === void 0) {
|
|
@@ -22272,7 +23466,6 @@ async function executeQuery(request, callerContext) {
|
|
|
22272
23466
|
for (const hit of hits) hit.score = void 0;
|
|
22273
23467
|
}
|
|
22274
23468
|
}
|
|
22275
|
-
const projection = resolveProjection(params.document);
|
|
22276
23469
|
if (projection.kind === "none") {
|
|
22277
23470
|
for (const hit of paginated) {
|
|
22278
23471
|
hit.document = {};
|