@delali/narsil 0.3.1-next.69 → 0.3.1-next.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{capabilities-B1wz2Zl0.d.ts → capabilities-Dt-7B4Wz.d.ts} +2 -2
- package/dist/{chunk-DD5X23E6.mjs → chunk-2STVUTNA.mjs} +1 -1
- package/dist/chunk-3JGWULDF.mjs +256 -0
- package/dist/{chunk-6TGZG2TV.mjs → chunk-653BNMAF.mjs} +190 -58
- package/dist/{chunk-YA52AC2V.mjs → chunk-FVLQ63B7.mjs} +203 -9
- package/dist/{chunk-5JLSTECC.mjs → chunk-GIVAGD3Q.mjs} +1 -1
- package/dist/{chunk-AV4IJKD2.mjs → chunk-HWQLTLNG.mjs} +18 -1
- package/dist/{chunk-6IT3MH5M.mjs → chunk-IOR2K26O.mjs} +178 -238
- package/dist/{chunk-4VU5TWCR.mjs → chunk-LMO5TJUS.mjs} +40 -35
- package/dist/{chunk-BEWOA4VV.mjs → chunk-M76V3OGJ.mjs} +508 -401
- package/dist/{chunk-YMZCMH5M.mjs → chunk-OI42GWLR.mjs} +17 -1
- package/dist/{chunk-2BDG2IWP.mjs → chunk-P4DSQ6M4.mjs} +16 -6
- package/dist/{chunk-THYRHHO2.mjs → chunk-YJDQ7NLA.mjs} +1 -1
- package/dist/{client-xVrOCYTW.d.ts → client-COek5GXK.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/{config-CEeSV_Xo.d.ts → config-Dfanyqqt.d.ts} +2 -2
- package/dist/{config-CSvCVtB7.d.ts → config-rBpbgTw8.d.ts} +2 -2
- package/dist/direct-executor-GYJEKNV5.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +109 -69
- package/dist/{engine-77ErLy8P.d.ts → engine-igk0aMpQ.d.ts} +109 -60
- package/dist/index.browser.mjs +1167 -749
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +10 -10
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-CKF-9WLP.d.ts → memory-C7UMCMH3.d.ts} +35 -17
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +10 -10
- package/dist/react.d.ts +277 -182
- package/dist/react.mjs +189 -45
- package/dist/{schema-d5Afu03A.d.ts → schema-Bgzr4g7b.d.ts} +62 -10
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +11 -7
- package/dist/{tasks-BEiNxDRs.d.ts → tasks-BZt_zes6.d.ts} +3 -1
- package/dist/{thread-FLP45EUD.mjs → thread-JEVAYETR.mjs} +6 -6
- package/dist/{types-DAK7mnUZ.d.ts → types-gOUqhs2F.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +3 -3
- package/package.json +7 -7
- package/dist/chunk-VTPLKNWI.mjs +0 -135
- package/dist/direct-executor-3ZYXSGFX.mjs +0 -12
package/dist/index.browser.mjs
CHANGED
|
@@ -1904,23 +1904,14 @@ function generateId() {
|
|
|
1904
1904
|
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
1905
1905
|
}
|
|
1906
1906
|
|
|
1907
|
+
// src/schema/constants.ts
|
|
1908
|
+
var MAX_NESTING_DEPTH = 4;
|
|
1909
|
+
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
1910
|
+
var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
|
|
1911
|
+
var MIN_PATTERN_VALUE_LIMIT = 1;
|
|
1912
|
+
var MAX_PATTERN_VALUE_LIMIT = 65536;
|
|
1913
|
+
|
|
1907
1914
|
// src/schema/validator/shared.ts
|
|
1908
|
-
var SORTABLE_TEXT_FIELD_TYPE = "string:sortable";
|
|
1909
|
-
var SCALAR_FIELD_TYPES = /* @__PURE__ */ new Set([
|
|
1910
|
-
"string",
|
|
1911
|
-
SORTABLE_TEXT_FIELD_TYPE,
|
|
1912
|
-
"number",
|
|
1913
|
-
"boolean",
|
|
1914
|
-
"enum",
|
|
1915
|
-
"geopoint",
|
|
1916
|
-
"string[]",
|
|
1917
|
-
"number[]",
|
|
1918
|
-
"boolean[]",
|
|
1919
|
-
"enum[]"
|
|
1920
|
-
]);
|
|
1921
|
-
function isTextFieldType(fieldType) {
|
|
1922
|
-
return fieldType === "string" || fieldType === SORTABLE_TEXT_FIELD_TYPE;
|
|
1923
|
-
}
|
|
1924
1915
|
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
1925
1916
|
function isGeopointOrVectorType(fieldType) {
|
|
1926
1917
|
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
@@ -1932,6 +1923,144 @@ function isPlainObject(value) {
|
|
|
1932
1923
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1933
1924
|
}
|
|
1934
1925
|
|
|
1926
|
+
// src/schema/validator/field-type.ts
|
|
1927
|
+
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
1928
|
+
var OPTIONS_BY_BASE = {
|
|
1929
|
+
string: FIELD_OPTION_ORDER,
|
|
1930
|
+
"string[]": FIELD_OPTION_ORDER,
|
|
1931
|
+
verbatim: ["sortable"],
|
|
1932
|
+
"verbatim[]": ["sortable"],
|
|
1933
|
+
number: [],
|
|
1934
|
+
"number[]": [],
|
|
1935
|
+
boolean: [],
|
|
1936
|
+
"boolean[]": [],
|
|
1937
|
+
enum: [],
|
|
1938
|
+
"enum[]": [],
|
|
1939
|
+
geopoint: [],
|
|
1940
|
+
vector: []
|
|
1941
|
+
};
|
|
1942
|
+
var TYPE_SEPARATOR = ":";
|
|
1943
|
+
function isNamedBase(name) {
|
|
1944
|
+
return name !== "vector" && Object.hasOwn(OPTIONS_BY_BASE, name);
|
|
1945
|
+
}
|
|
1946
|
+
function describeAllowedOptions(baseName, allowed) {
|
|
1947
|
+
if (allowed.length === 0) return `the "${baseName}" type has no options`;
|
|
1948
|
+
const quoted = allowed.map((option) => `"${option}"`);
|
|
1949
|
+
if (quoted.length === 1) return `the only option of the "${baseName}" type is ${quoted[0]}`;
|
|
1950
|
+
return `the options of the "${baseName}" type are ${quoted.slice(0, -1).join(", ")}, and ${quoted[quoted.length - 1]}`;
|
|
1951
|
+
}
|
|
1952
|
+
function invalidType(path, type, message) {
|
|
1953
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, message, { field: path, type });
|
|
1954
|
+
}
|
|
1955
|
+
function parseBase(baseName, path, type) {
|
|
1956
|
+
if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
|
|
1957
|
+
const vectorMatch = VECTOR_PATTERN.exec(baseName);
|
|
1958
|
+
if (vectorMatch === null) {
|
|
1959
|
+
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
1960
|
+
}
|
|
1961
|
+
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
1962
|
+
if (dimension <= 0) {
|
|
1963
|
+
throw new NarsilError(
|
|
1964
|
+
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
1965
|
+
`Vector field "${path}" must have a positive dimension, got ${dimension}`,
|
|
1966
|
+
{ field: path, dimension }
|
|
1967
|
+
);
|
|
1968
|
+
}
|
|
1969
|
+
return { base: "vector", dimension };
|
|
1970
|
+
}
|
|
1971
|
+
function parseFieldType(type, path) {
|
|
1972
|
+
const separatorIndex = type.indexOf(TYPE_SEPARATOR);
|
|
1973
|
+
const baseName = separatorIndex === -1 ? type : type.slice(0, separatorIndex);
|
|
1974
|
+
const { base, dimension } = parseBase(baseName, path, type);
|
|
1975
|
+
const allowed = OPTIONS_BY_BASE[base];
|
|
1976
|
+
const options = /* @__PURE__ */ new Set();
|
|
1977
|
+
if (separatorIndex !== -1) {
|
|
1978
|
+
for (const option of type.slice(separatorIndex + 1).split(TYPE_SEPARATOR)) {
|
|
1979
|
+
if (!allowed.includes(option)) {
|
|
1980
|
+
invalidType(
|
|
1981
|
+
path,
|
|
1982
|
+
type,
|
|
1983
|
+
`Field "${path}" has type "${type}" with the option "${option}", but ${describeAllowedOptions(baseName, allowed)}`
|
|
1984
|
+
);
|
|
1985
|
+
}
|
|
1986
|
+
if (options.has(option)) {
|
|
1987
|
+
invalidType(path, type, `Field "${path}" has the "${option}" option twice in type "${type}"`);
|
|
1988
|
+
}
|
|
1989
|
+
options.add(option);
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
const ordered = FIELD_OPTION_ORDER.filter((option) => options.has(option));
|
|
1993
|
+
const verbatim = base === "verbatim" || base === "verbatim[]";
|
|
1994
|
+
return {
|
|
1995
|
+
type: [baseName, ...ordered].join(TYPE_SEPARATOR),
|
|
1996
|
+
base,
|
|
1997
|
+
sortable: options.has("sortable"),
|
|
1998
|
+
pattern: verbatim || options.has("pattern"),
|
|
1999
|
+
partial: options.has("partial"),
|
|
2000
|
+
dimension
|
|
2001
|
+
};
|
|
2002
|
+
}
|
|
2003
|
+
function parsedTypeOf(type) {
|
|
2004
|
+
if (type === void 0) return void 0;
|
|
2005
|
+
try {
|
|
2006
|
+
return parseFieldType(type, "");
|
|
2007
|
+
} catch {
|
|
2008
|
+
return void 0;
|
|
2009
|
+
}
|
|
2010
|
+
}
|
|
2011
|
+
function withSortableOption(parsed) {
|
|
2012
|
+
if (parsed.sortable) return parsed.type;
|
|
2013
|
+
const separatorIndex = parsed.type.indexOf(TYPE_SEPARATOR);
|
|
2014
|
+
const baseName = separatorIndex === -1 ? parsed.type : parsed.type.slice(0, separatorIndex);
|
|
2015
|
+
return parseFieldType(`${baseName}${TYPE_SEPARATOR}sortable${parsed.type.slice(baseName.length)}`, "").type;
|
|
2016
|
+
}
|
|
2017
|
+
function isListBase(base) {
|
|
2018
|
+
return base.endsWith("[]");
|
|
2019
|
+
}
|
|
2020
|
+
function isWordIndexedBase(base) {
|
|
2021
|
+
return base === "string" || base === "string[]";
|
|
2022
|
+
}
|
|
2023
|
+
function isSingleStringFieldType(fieldType) {
|
|
2024
|
+
return parsedTypeOf(fieldType)?.base === "string";
|
|
2025
|
+
}
|
|
2026
|
+
function isWordIndexedFieldType(fieldType) {
|
|
2027
|
+
const parsed = parsedTypeOf(fieldType);
|
|
2028
|
+
return parsed !== void 0 && isWordIndexedBase(parsed.base);
|
|
2029
|
+
}
|
|
2030
|
+
function isStringValuedBase(base) {
|
|
2031
|
+
return base === "string" || base === "string[]" || base === "verbatim" || base === "verbatim[]";
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
// src/core/ordering/code-points.ts
|
|
2035
|
+
function exceedsCodePoints(value, maximum) {
|
|
2036
|
+
if (value.length <= maximum) return false;
|
|
2037
|
+
let counted = 0;
|
|
2038
|
+
let i = 0;
|
|
2039
|
+
while (i < value.length) {
|
|
2040
|
+
const unit = value.charCodeAt(i);
|
|
2041
|
+
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
2042
|
+
const low = value.charCodeAt(i + 1);
|
|
2043
|
+
if (low >= 56320 && low < 57344) i++;
|
|
2044
|
+
}
|
|
2045
|
+
i++;
|
|
2046
|
+
counted++;
|
|
2047
|
+
if (counted > maximum) return true;
|
|
2048
|
+
}
|
|
2049
|
+
return false;
|
|
2050
|
+
}
|
|
2051
|
+
function adjustUtf16Unit(unit) {
|
|
2052
|
+
if (unit < 55296) return unit;
|
|
2053
|
+
if (unit < 57344) return unit + 8192;
|
|
2054
|
+
return unit - 2048;
|
|
2055
|
+
}
|
|
2056
|
+
function compareCodePoints(a, b) {
|
|
2057
|
+
const shorter = a.length < b.length ? a.length : b.length;
|
|
2058
|
+
let i = 0;
|
|
2059
|
+
while (i < shorter && a.charCodeAt(i) === b.charCodeAt(i)) i++;
|
|
2060
|
+
if (i === shorter) return a.length - b.length;
|
|
2061
|
+
return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
|
|
2062
|
+
}
|
|
2063
|
+
|
|
1935
2064
|
// src/schema/validator/field-values.ts
|
|
1936
2065
|
function validateGeopoint(path, value) {
|
|
1937
2066
|
if (!isPlainObject(value)) {
|
|
@@ -2024,10 +2153,19 @@ function validateTypedArray(path, value, elementType) {
|
|
|
2024
2153
|
}
|
|
2025
2154
|
}
|
|
2026
2155
|
}
|
|
2027
|
-
function
|
|
2028
|
-
|
|
2156
|
+
function validatePatternValue(path, value, patternValueLimit, index) {
|
|
2157
|
+
if (!exceedsCodePoints(value, patternValueLimit)) return;
|
|
2158
|
+
const subject = index === void 0 ? `The value of field "${path}"` : `Element ${index} of field "${path}"`;
|
|
2159
|
+
throw new NarsilError(
|
|
2160
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
2161
|
+
`${subject} is longer than ${patternValueLimit} code points, which is the pattern value limit of this index`,
|
|
2162
|
+
{ field: path, limit: patternValueLimit, ...index === void 0 ? {} : { index } }
|
|
2163
|
+
);
|
|
2164
|
+
}
|
|
2165
|
+
function validateFieldValue(path, value, type, patternValueLimit) {
|
|
2166
|
+
switch (type.base) {
|
|
2029
2167
|
case "string":
|
|
2030
|
-
case
|
|
2168
|
+
case "verbatim":
|
|
2031
2169
|
if (typeof value !== "string") {
|
|
2032
2170
|
throw new NarsilError(
|
|
2033
2171
|
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
@@ -2039,6 +2177,7 @@ function validateFieldValue(path, value, type) {
|
|
|
2039
2177
|
}
|
|
2040
2178
|
);
|
|
2041
2179
|
}
|
|
2180
|
+
if (type.pattern) validatePatternValue(path, value, patternValueLimit);
|
|
2042
2181
|
return;
|
|
2043
2182
|
case "number":
|
|
2044
2183
|
if (typeof value !== "number" || Number.isNaN(value)) {
|
|
@@ -2079,7 +2218,12 @@ function validateFieldValue(path, value, type) {
|
|
|
2079
2218
|
validateGeopoint(path, value);
|
|
2080
2219
|
return;
|
|
2081
2220
|
case "string[]":
|
|
2221
|
+
case "verbatim[]":
|
|
2082
2222
|
validateTypedArray(path, value, "string");
|
|
2223
|
+
if (type.pattern) {
|
|
2224
|
+
const elements = value;
|
|
2225
|
+
for (let i = 0; i < elements.length; i++) validatePatternValue(path, elements[i], patternValueLimit, i);
|
|
2226
|
+
}
|
|
2083
2227
|
return;
|
|
2084
2228
|
case "number[]":
|
|
2085
2229
|
validateTypedArray(path, value, "number");
|
|
@@ -2090,19 +2234,12 @@ function validateFieldValue(path, value, type) {
|
|
|
2090
2234
|
case "enum[]":
|
|
2091
2235
|
validateTypedArray(path, value, "string");
|
|
2092
2236
|
return;
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
validateVector(path, value, Number.parseInt(vectorMatch[1], 10));
|
|
2097
|
-
}
|
|
2098
|
-
}
|
|
2237
|
+
case "vector":
|
|
2238
|
+
validateVector(path, value, type.dimension);
|
|
2239
|
+
return;
|
|
2099
2240
|
}
|
|
2100
2241
|
}
|
|
2101
2242
|
|
|
2102
|
-
// src/schema/constants.ts
|
|
2103
|
-
var MAX_NESTING_DEPTH = 4;
|
|
2104
|
-
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
2105
|
-
|
|
2106
2243
|
// src/schema/validator/storable.ts
|
|
2107
2244
|
var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
2108
2245
|
function reject(path, reason) {
|
|
@@ -2180,32 +2317,44 @@ function vectorFieldPaths(schema) {
|
|
|
2180
2317
|
vectorPathCache.set(schema, paths);
|
|
2181
2318
|
return paths;
|
|
2182
2319
|
}
|
|
2183
|
-
|
|
2184
|
-
|
|
2320
|
+
var parsedLevelCache = /* @__PURE__ */ new WeakMap();
|
|
2321
|
+
function parsedLevel(schema, prefix) {
|
|
2322
|
+
let entries = parsedLevelCache.get(schema);
|
|
2323
|
+
if (entries !== void 0) return entries;
|
|
2324
|
+
entries = Object.entries(schema).map(([field, type]) => {
|
|
2325
|
+
if (isPlainObject(type)) return { field, nested: type, type: null };
|
|
2185
2326
|
const path = prefix ? `${prefix}.${field}` : field;
|
|
2327
|
+
return { field, nested: null, type: parseFieldType(type, path) };
|
|
2328
|
+
});
|
|
2329
|
+
parsedLevelCache.set(schema, entries);
|
|
2330
|
+
return entries;
|
|
2331
|
+
}
|
|
2332
|
+
function validateDocumentFields(doc, schema, prefix, patternValueLimit) {
|
|
2333
|
+
for (const { field, nested, type } of parsedLevel(schema, prefix)) {
|
|
2186
2334
|
const value = doc[field];
|
|
2187
2335
|
if (value === void 0 || value === null) continue;
|
|
2188
|
-
|
|
2336
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
2337
|
+
if (nested !== null) {
|
|
2189
2338
|
if (!isPlainObject(value)) {
|
|
2190
2339
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an object`, {
|
|
2191
2340
|
field: path,
|
|
2192
2341
|
received: Array.isArray(value) ? "array" : typeof value
|
|
2193
2342
|
});
|
|
2194
2343
|
}
|
|
2195
|
-
validateDocumentFields(value,
|
|
2344
|
+
validateDocumentFields(value, nested, path, patternValueLimit);
|
|
2196
2345
|
continue;
|
|
2197
2346
|
}
|
|
2198
|
-
validateFieldValue(path, value, type);
|
|
2347
|
+
if (type !== null) validateFieldValue(path, value, type, patternValueLimit);
|
|
2199
2348
|
}
|
|
2200
2349
|
}
|
|
2201
|
-
function validateDocument(document2, schema) {
|
|
2350
|
+
function validateDocument(document2, schema, patternValueLimit = DEFAULT_PATTERN_VALUE_LIMIT) {
|
|
2202
2351
|
if (!isPlainObject(document2)) {
|
|
2203
2352
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document must be a plain object", {
|
|
2204
2353
|
received: typeof document2
|
|
2205
2354
|
});
|
|
2206
2355
|
}
|
|
2207
2356
|
assertStorableDocument(document2, vectorFieldPaths(schema));
|
|
2208
|
-
validateDocumentFields(document2, schema, "");
|
|
2357
|
+
validateDocumentFields(document2, schema, "", patternValueLimit);
|
|
2209
2358
|
}
|
|
2210
2359
|
function collectExtraFields(doc, schema, prefix, extras) {
|
|
2211
2360
|
for (const key of Object.keys(doc)) {
|
|
@@ -2263,6 +2412,11 @@ function validateSchemaFields(schema, depth, prefix) {
|
|
|
2263
2412
|
{ path: prefix, maxDepth: MAX_NESTING_DEPTH }
|
|
2264
2413
|
);
|
|
2265
2414
|
}
|
|
2415
|
+
let canonical = null;
|
|
2416
|
+
const replace = (field, value) => {
|
|
2417
|
+
if (canonical === null) canonical = { ...schema };
|
|
2418
|
+
canonical[field] = value;
|
|
2419
|
+
};
|
|
2266
2420
|
for (const [field, type] of Object.entries(schema)) {
|
|
2267
2421
|
const path = prefix ? `${prefix}.${field}` : field;
|
|
2268
2422
|
if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
|
|
@@ -2286,7 +2440,8 @@ function validateSchemaFields(schema, depth, prefix) {
|
|
|
2286
2440
|
);
|
|
2287
2441
|
}
|
|
2288
2442
|
if (isPlainObject(type)) {
|
|
2289
|
-
validateSchemaFields(type, depth + 1, path);
|
|
2443
|
+
const nested = validateSchemaFields(type, depth + 1, path);
|
|
2444
|
+
if (nested !== type) replace(field, nested);
|
|
2290
2445
|
continue;
|
|
2291
2446
|
}
|
|
2292
2447
|
if (typeof type !== "string") {
|
|
@@ -2299,24 +2454,10 @@ function validateSchemaFields(schema, depth, prefix) {
|
|
|
2299
2454
|
}
|
|
2300
2455
|
);
|
|
2301
2456
|
}
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
if (vectorMatch) {
|
|
2305
|
-
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
2306
|
-
if (dimension <= 0) {
|
|
2307
|
-
throw new NarsilError(
|
|
2308
|
-
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
2309
|
-
`Vector field "${path}" must have a positive dimension, got ${dimension}`,
|
|
2310
|
-
{ field: path, dimension }
|
|
2311
|
-
);
|
|
2312
|
-
}
|
|
2313
|
-
continue;
|
|
2314
|
-
}
|
|
2315
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field "${path}" has an unsupported type: "${type}"`, {
|
|
2316
|
-
field: path,
|
|
2317
|
-
type
|
|
2318
|
-
});
|
|
2457
|
+
const parsed = parseFieldType(type, path);
|
|
2458
|
+
if (parsed.type !== type) replace(field, parsed.type);
|
|
2319
2459
|
}
|
|
2460
|
+
return canonical ?? schema;
|
|
2320
2461
|
}
|
|
2321
2462
|
function validateSchema(schema) {
|
|
2322
2463
|
if (!isPlainObject(schema)) {
|
|
@@ -2327,7 +2468,19 @@ function validateSchema(schema) {
|
|
|
2327
2468
|
if (Object.keys(schema).length === 0) {
|
|
2328
2469
|
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
2329
2470
|
}
|
|
2330
|
-
validateSchemaFields(schema, 1, "");
|
|
2471
|
+
return validateSchemaFields(schema, 1, "");
|
|
2472
|
+
}
|
|
2473
|
+
function requireValidFieldTypes(flatSchema) {
|
|
2474
|
+
for (const [path, type] of Object.entries(flatSchema)) {
|
|
2475
|
+
if (typeof type !== "string") {
|
|
2476
|
+
throw new NarsilError(
|
|
2477
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
2478
|
+
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
2479
|
+
{ field: path, type: String(type) }
|
|
2480
|
+
);
|
|
2481
|
+
}
|
|
2482
|
+
parseFieldType(type, path);
|
|
2483
|
+
}
|
|
2331
2484
|
}
|
|
2332
2485
|
function flattenRecursive(schema, prefix, result) {
|
|
2333
2486
|
for (const [field, type] of Object.entries(schema)) {
|
|
@@ -2348,14 +2501,22 @@ function flattenSchema(schema) {
|
|
|
2348
2501
|
flattenedSchemas.set(schema, Object.freeze(result));
|
|
2349
2502
|
return result;
|
|
2350
2503
|
}
|
|
2504
|
+
var parsedSchemas = /* @__PURE__ */ new WeakMap();
|
|
2505
|
+
function schemaFieldsOf(schema) {
|
|
2506
|
+
const cached = parsedSchemas.get(schema);
|
|
2507
|
+
if (cached !== void 0) return cached;
|
|
2508
|
+
const fields = [];
|
|
2509
|
+
for (const [path, type] of Object.entries(flattenSchema(schema))) {
|
|
2510
|
+
fields.push({ ...parseFieldType(type, path), path });
|
|
2511
|
+
}
|
|
2512
|
+
const frozen = Object.freeze(fields);
|
|
2513
|
+
parsedSchemas.set(schema, frozen);
|
|
2514
|
+
return frozen;
|
|
2515
|
+
}
|
|
2351
2516
|
function extractVectorFieldsFromSchema(schema) {
|
|
2352
|
-
const flat = flattenSchema(schema);
|
|
2353
2517
|
const result = /* @__PURE__ */ new Map();
|
|
2354
|
-
for (const
|
|
2355
|
-
|
|
2356
|
-
if (match) {
|
|
2357
|
-
result.set(fieldPath, Number.parseInt(match[1], 10));
|
|
2358
|
-
}
|
|
2518
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
2519
|
+
if (field.base === "vector") result.set(field.path, field.dimension);
|
|
2359
2520
|
}
|
|
2360
2521
|
return result;
|
|
2361
2522
|
}
|
|
@@ -2494,6 +2655,9 @@ function metadataToWire(meta) {
|
|
|
2494
2655
|
...promotion.storage !== void 0 ? { storage: promotion.storage } : {}
|
|
2495
2656
|
};
|
|
2496
2657
|
}
|
|
2658
|
+
if (meta.patternValueLimit !== void 0) {
|
|
2659
|
+
wire.pattern_value_limit = meta.patternValueLimit;
|
|
2660
|
+
}
|
|
2497
2661
|
if (meta.indexUuid !== void 0) {
|
|
2498
2662
|
wire.index_uuid = meta.indexUuid;
|
|
2499
2663
|
}
|
|
@@ -2615,6 +2779,16 @@ function wireToMetadata(raw) {
|
|
|
2615
2779
|
meta.vectorPromotion = restored;
|
|
2616
2780
|
}
|
|
2617
2781
|
}
|
|
2782
|
+
if (raw.pattern_value_limit !== void 0) {
|
|
2783
|
+
if (!isPositiveInteger2(raw.pattern_value_limit)) {
|
|
2784
|
+
throw new NarsilError(
|
|
2785
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2786
|
+
`The pattern value limit in the index metadata is ${String(raw.pattern_value_limit)}, but the limit must be a whole number of code points`,
|
|
2787
|
+
{ patternValueLimit: String(raw.pattern_value_limit) }
|
|
2788
|
+
);
|
|
2789
|
+
}
|
|
2790
|
+
meta.patternValueLimit = raw.pattern_value_limit;
|
|
2791
|
+
}
|
|
2618
2792
|
if (typeof raw.index_uuid === "string" && raw.index_uuid.length > 0) {
|
|
2619
2793
|
meta.indexUuid = raw.index_uuid;
|
|
2620
2794
|
}
|
|
@@ -2709,6 +2883,7 @@ function wireToPartition(raw) {
|
|
|
2709
2883
|
fieldLengths: doc.field_lengths ?? {}
|
|
2710
2884
|
};
|
|
2711
2885
|
}
|
|
2886
|
+
requireValidFieldTypes(raw.schema ?? {});
|
|
2712
2887
|
dropStoredVectorValues(documents, raw.schema ?? {});
|
|
2713
2888
|
const invertedIndex = {};
|
|
2714
2889
|
for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
|
|
@@ -2793,6 +2968,7 @@ function validateHnswMetric2(value) {
|
|
|
2793
2968
|
return void 0;
|
|
2794
2969
|
}
|
|
2795
2970
|
function storedDocumentsOf(raw) {
|
|
2971
|
+
requireValidFieldTypes(raw.schema ?? {});
|
|
2796
2972
|
const documents = {};
|
|
2797
2973
|
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
2798
2974
|
documents[docId] = {
|
|
@@ -2883,20 +3059,6 @@ function deserializePayloadV2(data) {
|
|
|
2883
3059
|
};
|
|
2884
3060
|
}
|
|
2885
3061
|
|
|
2886
|
-
// src/core/ordering/code-points.ts
|
|
2887
|
-
function adjustUtf16Unit(unit) {
|
|
2888
|
-
if (unit < 55296) return unit;
|
|
2889
|
-
if (unit < 57344) return unit + 8192;
|
|
2890
|
-
return unit - 2048;
|
|
2891
|
-
}
|
|
2892
|
-
function compareCodePoints(a, b) {
|
|
2893
|
-
const shorter = a.length < b.length ? a.length : b.length;
|
|
2894
|
-
let i = 0;
|
|
2895
|
-
while (i < shorter && a.charCodeAt(i) === b.charCodeAt(i)) i++;
|
|
2896
|
-
if (i === shorter) return a.length - b.length;
|
|
2897
|
-
return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
|
|
2898
|
-
}
|
|
2899
|
-
|
|
2900
3062
|
// src/core/ordering/fold-table.ts
|
|
2901
3063
|
var SINGLE_FOLDS = "1t,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2j,lj;b,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;y,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3d;1,1;2,1;2,1;2,-7g;2,5u;1,1;2,1;2,5q;1,1;2,5p;1,5p;1,1;3,27;1,5m;1,5n;1,1;2,5p;1,5r;2,5v;1,5t;1,1;4,5v;1,5x;2,5y;1,1;2,1;2,1;2,62;1,1;2,62;3,1;2,62;1,1;2,61;1,61;1,1;2,1;2,63;1,1;4,1;8,2;1,1;2,2;1,1;2,2;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,2;1,1;2,1;2,-2p;1,-1k;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;8,8bv;1,1;2,-4j;1,8bs;3,1;2,-5f;1,1x;1,1z;1,1;2,1;2,1;2,1;2,1;6v,38;17,1;2,1;4,1;9,38;7,12;2,11;1,11;1,11;2,1s;2,1r;1,1r;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;n,1;d,8;1,-u;1,-p;4,-f;1,-m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1i;1,-1c;3,-1o;1,-1s;2,1;2,-7;1,1;3,-3m;1,-3m;1,-3m;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1d,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,f;1,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;28a,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;2,5ls;6,5ls;mj,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1oj,-4su;1,-4st;1,-4sk;1,-4si;1,-4si;1,-4sj;1,-4sc;1,-4ro;1,r7n;1,1;7,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;3,-2bk;1,-2bk;1,-2bk;8x,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;7,-1m;5,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;b,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;c,-8;2,-8;2,-8;2,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;21,-8;1,-8;1,-22;1,-22;3,-5j9;a,-2e;1,-2e;1,-2e;1,-2e;d,-8;1,-8;1,-2s;1,-2s;d,-8;1,-8;1,-34;1,-34;1,-7;c,-3k;1,-3k;1,-3i;1,-3i;8b,-5st;4,-6gv;1,-6di;7,s;1a,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;k,1;mr,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1f5,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1d,1;2,-8af;1,-2xy;1,-89z;3,1;2,1;2,1;2,-8bg;1,-8al;1,-8bj;1,-8bi;2,1;3,1;9,-8cf;1,-8cf;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;9,1;2,1;5,1;nym,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;k,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3s,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;b,1;2,1;2,-r9g;1,1;2,1;2,1;2,1;2,1;5,1;2,-wmg;3,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wn8;1,-wnj;1,-wnf;1,-wn5;1,-wn8;2,-wlu;1,-wmi;1,-wlx;1,ps;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1c;1,-wn7;1,-raw;1,1;2,1;2,-wo7;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wu9;p,1;or,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;ggy,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;xy,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;3t,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;4d,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1d7,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;4e,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;27v,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;gw1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1t,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;o5k,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;";
|
|
2902
3064
|
var MULTI_FOLDS = "67,37,37;29,2x,lj;p,jg,32;4n,2y,lo;bk,qh,lk,ld;w,qt,lk,ld;d3,12d,136;4y7,2w,mp;1,38,lk;1,3b,lm;1,3d,lm;1,2p,ji;4,37,37;4y,qt,lv;2,qt,lv,lc;2,qt,lv,ld;2,qt,lv,n6;16,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,64g,qh;1,64h,qh;1,64i,qh;1,64j,qh;1,64k,qh;1,64l,qh;1,64m,qh;1,64n,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,65c,qh;1,65d,qh;1,65e,qh;1,65f,qh;1,65g,qh;1,65h,qh;1,65i,qh;1,65j,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;1,674,qh;1,675,qh;1,676,qh;1,677,qh;1,678,qh;1,679,qh;1,67a,qh;1,67b,qh;3,67k,qh;1,q9,qh;1,q4,qh;2,q9,n6;1,q9,n6,qh;5,q9,qh;6,67o,qh;1,qf,qh;1,q6,qh;2,qf,n6;1,qf,n6,qh;5,qf,qh;6,qh,lk,lc;1,qh,lk,ld;3,qh,n6;1,qh,lk,n6;b,qt,lk,lc;1,qt,lk,ld;1,qp,lv;2,qt,n6;1,qt,lk,n6;b,67w,qh;1,qx,qh;1,r2,qh;2,qx,n6;1,qx,n6,qh;5,qx,qh;179g,2u,2u;1,2u,2x;1,2u,30;1,2u,2u,2x;1,2u,2u,30;1,37,38;1,37,38;d,12s,12u;1,12s,12d;1,12s,12j;1,132,12u;1,12s,12l;";
|
|
@@ -4185,12 +4347,12 @@ function stringArraysEqual(a, b) {
|
|
|
4185
4347
|
}
|
|
4186
4348
|
return true;
|
|
4187
4349
|
}
|
|
4188
|
-
function textFieldsChanged(oldDoc, newDoc,
|
|
4189
|
-
for (const
|
|
4190
|
-
if (!
|
|
4350
|
+
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
4351
|
+
for (const { path, base } of fields) {
|
|
4352
|
+
if (!isWordIndexedBase(base)) continue;
|
|
4191
4353
|
const oldVal = getNestedValue(oldDoc, path);
|
|
4192
4354
|
const newVal = getNestedValue(newDoc, path);
|
|
4193
|
-
if (
|
|
4355
|
+
if (base === "string") {
|
|
4194
4356
|
if (oldVal !== newVal) return true;
|
|
4195
4357
|
} else {
|
|
4196
4358
|
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
@@ -4919,14 +5081,14 @@ function createNumericIndex() {
|
|
|
4919
5081
|
}
|
|
4920
5082
|
|
|
4921
5083
|
// src/core/partition/indexing.ts
|
|
4922
|
-
function ensureFieldIndex(state, fieldPath,
|
|
4923
|
-
if ((
|
|
5084
|
+
function ensureFieldIndex(state, fieldPath, base) {
|
|
5085
|
+
if ((base === "number" || base === "number[]") && !state.numericIndexes.has(fieldPath)) {
|
|
4924
5086
|
state.numericIndexes.set(fieldPath, createNumericIndex());
|
|
4925
|
-
} else if ((
|
|
5087
|
+
} else if ((base === "boolean" || base === "boolean[]") && !state.booleanIndexes.has(fieldPath)) {
|
|
4926
5088
|
state.booleanIndexes.set(fieldPath, createBooleanIndex());
|
|
4927
|
-
} else if ((
|
|
5089
|
+
} else if ((base === "enum" || base === "enum[]") && !state.enumIndexes.has(fieldPath)) {
|
|
4928
5090
|
state.enumIndexes.set(fieldPath, createEnumIndex());
|
|
4929
|
-
} else if (
|
|
5091
|
+
} else if (base === "geopoint" && !state.geoIndexes.has(fieldPath)) {
|
|
4930
5092
|
state.geoIndexes.set(fieldPath, createGeoIndex());
|
|
4931
5093
|
}
|
|
4932
5094
|
}
|
|
@@ -5052,26 +5214,26 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5052
5214
|
applySurfaceCounts(state, surfaceCounts, 1);
|
|
5053
5215
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5054
5216
|
}
|
|
5055
|
-
function indexDocument(state, docId, document2,
|
|
5217
|
+
function indexDocument(state, docId, document2, fields, language, options) {
|
|
5056
5218
|
const fieldLengths = {};
|
|
5057
5219
|
const tokensByField = {};
|
|
5058
5220
|
const internalId = resolveInternalId(state, docId);
|
|
5059
|
-
for (const
|
|
5221
|
+
for (const { path: fieldPath, base } of fields) {
|
|
5060
5222
|
const value = getNestedValue(document2, fieldPath);
|
|
5061
5223
|
if (value === void 0 || value === null) continue;
|
|
5062
|
-
ensureFieldIndex(state, fieldPath,
|
|
5063
|
-
if (
|
|
5224
|
+
ensureFieldIndex(state, fieldPath, base);
|
|
5225
|
+
if (base === "string") {
|
|
5064
5226
|
indexStringField(state, internalId, fieldPath, value, language, options, fieldLengths, tokensByField);
|
|
5065
|
-
} else if (
|
|
5227
|
+
} else if (base === "number") {
|
|
5066
5228
|
getOrCreateNumericIndex(state, fieldPath).insert(internalId, value);
|
|
5067
|
-
} else if (
|
|
5229
|
+
} else if (base === "boolean") {
|
|
5068
5230
|
getOrCreateBooleanIndex(state, fieldPath).insert(internalId, value);
|
|
5069
|
-
} else if (
|
|
5231
|
+
} else if (base === "enum") {
|
|
5070
5232
|
state.enumIndexes.get(fieldPath)?.insert(internalId, value);
|
|
5071
|
-
} else if (
|
|
5233
|
+
} else if (base === "geopoint") {
|
|
5072
5234
|
const geo = value;
|
|
5073
5235
|
state.geoIndexes.get(fieldPath)?.insert(internalId, geo.lat, geo.lon);
|
|
5074
|
-
} else if (
|
|
5236
|
+
} else if (base === "string[]") {
|
|
5075
5237
|
indexStringArrayField(
|
|
5076
5238
|
state,
|
|
5077
5239
|
internalId,
|
|
@@ -5082,13 +5244,13 @@ function indexDocument(state, docId, document2, flatSchema, language, options) {
|
|
|
5082
5244
|
fieldLengths,
|
|
5083
5245
|
tokensByField
|
|
5084
5246
|
);
|
|
5085
|
-
} else if (
|
|
5247
|
+
} else if (base === "number[]") {
|
|
5086
5248
|
const numIdx = getOrCreateNumericIndex(state, fieldPath);
|
|
5087
5249
|
for (const num of value) numIdx.insert(internalId, num);
|
|
5088
|
-
} else if (
|
|
5250
|
+
} else if (base === "boolean[]") {
|
|
5089
5251
|
const boolIdx = getOrCreateBooleanIndex(state, fieldPath);
|
|
5090
5252
|
for (const b of value) boolIdx.insert(internalId, b);
|
|
5091
|
-
} else if (
|
|
5253
|
+
} else if (base === "enum[]") {
|
|
5092
5254
|
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
5093
5255
|
if (enumIdx) {
|
|
5094
5256
|
for (const e of value) enumIdx.insert(internalId, e);
|
|
@@ -5097,16 +5259,16 @@ function indexDocument(state, docId, document2, flatSchema, language, options) {
|
|
|
5097
5259
|
}
|
|
5098
5260
|
return { fieldLengths, tokensByField };
|
|
5099
5261
|
}
|
|
5100
|
-
function removeFromIndexes(state, docId, storedDoc,
|
|
5262
|
+
function removeFromIndexes(state, docId, storedDoc, schemaFields, language, options) {
|
|
5101
5263
|
const fieldLengths = {};
|
|
5102
5264
|
const tokensByField = {};
|
|
5103
5265
|
const fields = storedDoc.fields;
|
|
5104
5266
|
const internalId = resolveInternalId(state, docId);
|
|
5105
5267
|
const opts = tokenizeOptions(options);
|
|
5106
|
-
for (const
|
|
5268
|
+
for (const { path: fieldPath, base } of schemaFields) {
|
|
5107
5269
|
const value = getNestedValue(fields, fieldPath);
|
|
5108
5270
|
if (value === void 0 || value === null) continue;
|
|
5109
|
-
if (
|
|
5271
|
+
if (base === "string") {
|
|
5110
5272
|
const result = tokenize(value, language, opts);
|
|
5111
5273
|
fieldLengths[fieldPath] = result.tokens.length;
|
|
5112
5274
|
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
@@ -5121,15 +5283,15 @@ function removeFromIndexes(state, docId, storedDoc, flatSchema, language, option
|
|
|
5121
5283
|
for (const token of uniqueTokens) state.invertedIdx.remove(token, internalId);
|
|
5122
5284
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
5123
5285
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5124
|
-
} else if (
|
|
5286
|
+
} else if (base === "number") {
|
|
5125
5287
|
state.numericIndexes.get(fieldPath)?.remove(internalId, value);
|
|
5126
|
-
} else if (
|
|
5288
|
+
} else if (base === "boolean") {
|
|
5127
5289
|
state.booleanIndexes.get(fieldPath)?.remove(internalId, value);
|
|
5128
|
-
} else if (
|
|
5290
|
+
} else if (base === "enum") {
|
|
5129
5291
|
state.enumIndexes.get(fieldPath)?.remove(internalId, value);
|
|
5130
|
-
} else if (
|
|
5292
|
+
} else if (base === "geopoint") {
|
|
5131
5293
|
state.geoIndexes.get(fieldPath)?.remove(internalId);
|
|
5132
|
-
} else if (
|
|
5294
|
+
} else if (base === "string[]") {
|
|
5133
5295
|
const arr = value;
|
|
5134
5296
|
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
5135
5297
|
const fieldTokenList = [];
|
|
@@ -5147,17 +5309,17 @@ function removeFromIndexes(state, docId, storedDoc, flatSchema, language, option
|
|
|
5147
5309
|
for (const token of uniqueTokens) state.invertedIdx.remove(token, internalId);
|
|
5148
5310
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
5149
5311
|
tokensByField[fieldPath] = fieldTokenList;
|
|
5150
|
-
} else if (
|
|
5312
|
+
} else if (base === "number[]") {
|
|
5151
5313
|
const numIdx = state.numericIndexes.get(fieldPath);
|
|
5152
5314
|
if (numIdx) {
|
|
5153
5315
|
for (const num of value) numIdx.remove(internalId, num);
|
|
5154
5316
|
}
|
|
5155
|
-
} else if (
|
|
5317
|
+
} else if (base === "boolean[]") {
|
|
5156
5318
|
const boolIdx = state.booleanIndexes.get(fieldPath);
|
|
5157
5319
|
if (boolIdx) {
|
|
5158
5320
|
for (const b of value) boolIdx.remove(internalId, b);
|
|
5159
5321
|
}
|
|
5160
|
-
} else if (
|
|
5322
|
+
} else if (base === "enum[]") {
|
|
5161
5323
|
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
5162
5324
|
if (enumIdx) {
|
|
5163
5325
|
for (const e of value) enumIdx.remove(internalId, e);
|
|
@@ -5175,10 +5337,10 @@ function resolveInternalId2(state, docId) {
|
|
|
5175
5337
|
}
|
|
5176
5338
|
return internalId;
|
|
5177
5339
|
}
|
|
5178
|
-
function updateFieldIndexOnly(state, docId, oldFields, newDoc,
|
|
5340
|
+
function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
|
|
5179
5341
|
const internalId = resolveInternalId2(state, docId);
|
|
5180
|
-
for (const
|
|
5181
|
-
if (
|
|
5342
|
+
for (const { path: fieldPath, base: fieldType } of fields) {
|
|
5343
|
+
if (isWordIndexedBase(fieldType)) continue;
|
|
5182
5344
|
const oldVal = getNestedValue(oldFields, fieldPath);
|
|
5183
5345
|
const newVal = getNestedValue(newDoc, fieldPath);
|
|
5184
5346
|
if (oldVal === newVal) continue;
|
|
@@ -5333,6 +5495,16 @@ function requireFieldKind(fieldPath, operator, type, accepted) {
|
|
|
5333
5495
|
operator
|
|
5334
5496
|
);
|
|
5335
5497
|
}
|
|
5498
|
+
function requireTextRangeField(fieldPath, operator, type) {
|
|
5499
|
+
const parsed = parsedTypeOf(type);
|
|
5500
|
+
if (parsed?.sortable && (parsed.base === "string" || parsed.base === "verbatim")) return;
|
|
5501
|
+
const declared = type === void 0 ? `the schema has no field "${fieldPath}"` : `field "${fieldPath}" has type "${type}"`;
|
|
5502
|
+
invalid(
|
|
5503
|
+
`The engine applies "${operator}" with a text bound to a "string" or "verbatim" field whose type includes "sortable", but ${declared}`,
|
|
5504
|
+
fieldPath,
|
|
5505
|
+
operator
|
|
5506
|
+
);
|
|
5507
|
+
}
|
|
5336
5508
|
function requireRadius(fieldPath, radius) {
|
|
5337
5509
|
const circle = radius;
|
|
5338
5510
|
if (circle === null || typeof circle !== "object") {
|
|
@@ -5393,22 +5565,32 @@ function requireValidFilter(expression, fieldTypes, declaredOnly = false) {
|
|
|
5393
5565
|
}
|
|
5394
5566
|
function requireValidOperands(fieldPath, filter, type) {
|
|
5395
5567
|
for (const operator of RANGE_OPERATORS) {
|
|
5396
|
-
|
|
5397
|
-
if (
|
|
5398
|
-
|
|
5568
|
+
const bound = filter[operator];
|
|
5569
|
+
if (bound === void 0) continue;
|
|
5570
|
+
if (typeof bound === "string") {
|
|
5571
|
+
requireTextRangeField(fieldPath, operator, type);
|
|
5572
|
+
continue;
|
|
5573
|
+
}
|
|
5574
|
+
if (!isFiniteNumber(bound)) {
|
|
5575
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a finite number or a string`, fieldPath, operator);
|
|
5399
5576
|
}
|
|
5400
5577
|
requireFieldKind(fieldPath, operator, type, ["number"]);
|
|
5401
5578
|
}
|
|
5402
5579
|
if (filter.between !== void 0) {
|
|
5403
5580
|
const bounds = filter.between;
|
|
5404
|
-
|
|
5405
|
-
|
|
5406
|
-
|
|
5407
|
-
|
|
5408
|
-
|
|
5409
|
-
|
|
5581
|
+
const isPair = Array.isArray(bounds) && bounds.length === 2;
|
|
5582
|
+
if (isPair && typeof bounds[0] === "string" && typeof bounds[1] === "string") {
|
|
5583
|
+
requireTextRangeField(fieldPath, "between", type);
|
|
5584
|
+
} else {
|
|
5585
|
+
if (!isPair || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
|
|
5586
|
+
invalid(
|
|
5587
|
+
`"between" on field "${fieldPath}" must be two finite numbers or two strings, the lower bound first`,
|
|
5588
|
+
fieldPath,
|
|
5589
|
+
"between"
|
|
5590
|
+
);
|
|
5591
|
+
}
|
|
5592
|
+
requireFieldKind(fieldPath, "between", type, ["number"]);
|
|
5410
5593
|
}
|
|
5411
|
-
requireFieldKind(fieldPath, "between", type, ["number"]);
|
|
5412
5594
|
}
|
|
5413
5595
|
for (const operator of TEXT_OPERATORS) {
|
|
5414
5596
|
if (filter[operator] === void 0) continue;
|
|
@@ -5668,6 +5850,14 @@ function applyGeoPolygonBitset(filter, geoIndex, capacity) {
|
|
|
5668
5850
|
}
|
|
5669
5851
|
|
|
5670
5852
|
// src/filters/evaluator.ts
|
|
5853
|
+
var UNBOUNDED = { lower: null, upper: null };
|
|
5854
|
+
function textBoundsOf(operator, bound) {
|
|
5855
|
+
if (operator === "gt" || operator === "gte") {
|
|
5856
|
+
return { ...UNBOUNDED, lower: bound, lowerInclusive: operator === "gte", upperInclusive: false };
|
|
5857
|
+
}
|
|
5858
|
+
return { ...UNBOUNDED, upper: bound, upperInclusive: operator === "lte", lowerInclusive: false };
|
|
5859
|
+
}
|
|
5860
|
+
var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
|
|
5671
5861
|
var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
|
|
5672
5862
|
var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
|
|
5673
5863
|
function requireKnownKeys2(value, known, describe) {
|
|
@@ -5739,20 +5929,33 @@ function evaluateFieldFilter(fieldPath, filter, context) {
|
|
|
5739
5929
|
if ("ne" in f && f.ne !== void 0) {
|
|
5740
5930
|
bitsets.push(applyNeBitset(f.ne, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5741
5931
|
}
|
|
5742
|
-
|
|
5932
|
+
for (const operator of TEXT_RANGE_OPERATORS) {
|
|
5933
|
+
const bound = f[operator];
|
|
5934
|
+
if (typeof bound === "string") bitsets.push(context.textRangeBitset(fieldPath, textBoundsOf(operator, bound)));
|
|
5935
|
+
}
|
|
5936
|
+
if (typeof f.gt === "number") {
|
|
5743
5937
|
bitsets.push(applyGtBitset(f.gt, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5744
5938
|
}
|
|
5745
|
-
if (
|
|
5939
|
+
if (typeof f.lt === "number") {
|
|
5746
5940
|
bitsets.push(applyLtBitset(f.lt, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5747
5941
|
}
|
|
5748
|
-
if (
|
|
5942
|
+
if (typeof f.gte === "number") {
|
|
5749
5943
|
bitsets.push(applyGteBitset(f.gte, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5750
5944
|
}
|
|
5751
|
-
if (
|
|
5945
|
+
if (typeof f.lte === "number") {
|
|
5752
5946
|
bitsets.push(applyLteBitset(f.lte, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5753
5947
|
}
|
|
5754
|
-
if (
|
|
5755
|
-
|
|
5948
|
+
if (Array.isArray(f.between)) {
|
|
5949
|
+
const [low, high] = f.between;
|
|
5950
|
+
if (typeof low === "string" && typeof high === "string") {
|
|
5951
|
+
bitsets.push(
|
|
5952
|
+
context.textRangeBitset(fieldPath, { lower: low, lowerInclusive: true, upper: high, upperInclusive: true })
|
|
5953
|
+
);
|
|
5954
|
+
} else {
|
|
5955
|
+
bitsets.push(
|
|
5956
|
+
applyBetweenBitset([low, high], getAllDocsBitset, capacity, getValue, fieldIndex)
|
|
5957
|
+
);
|
|
5958
|
+
}
|
|
5756
5959
|
}
|
|
5757
5960
|
if ("in" in f && f.in !== void 0) {
|
|
5758
5961
|
bitsets.push(applyInBitset(f.in, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
@@ -5814,266 +6017,32 @@ function resolveGeoIndex(fieldPath, fieldIndex) {
|
|
|
5814
6017
|
return fieldIndex.index;
|
|
5815
6018
|
}
|
|
5816
6019
|
|
|
5817
|
-
// src/core/partition/
|
|
5818
|
-
var
|
|
5819
|
-
|
|
5820
|
-
|
|
5821
|
-
|
|
5822
|
-
|
|
5823
|
-
|
|
5824
|
-
|
|
5825
|
-
|
|
5826
|
-
|
|
5827
|
-
|
|
5828
|
-
|
|
5829
|
-
|
|
5830
|
-
|
|
5831
|
-
|
|
5832
|
-
|
|
5833
|
-
|
|
5834
|
-
|
|
5835
|
-
|
|
5836
|
-
|
|
5837
|
-
|
|
5838
|
-
|
|
5839
|
-
|
|
5840
|
-
eqBitset: (v, cap) => numIdx.queryEqBitset(v, cap),
|
|
5841
|
-
gtBitset: (v, cap) => numIdx.queryGtBitset(v, cap),
|
|
5842
|
-
gteBitset: (v, cap) => numIdx.queryGteBitset(v, cap),
|
|
5843
|
-
ltBitset: (v, cap) => numIdx.queryLtBitset(v, cap),
|
|
5844
|
-
lteBitset: (v, cap) => numIdx.queryLteBitset(v, cap),
|
|
5845
|
-
betweenBitset: (min, max, cap) => numIdx.queryBetweenBitset(min, max, cap),
|
|
5846
|
-
allDocIdsBitset: (cap) => numIdx.getAllDocIdsBitset(cap)
|
|
5847
|
-
}
|
|
5848
|
-
};
|
|
5849
|
-
}
|
|
5850
|
-
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
5851
|
-
const boolIdx = state.booleanIndexes.get(fieldPath);
|
|
5852
|
-
if (boolIdx) {
|
|
5853
|
-
fieldIndexes[fieldPath] = {
|
|
5854
|
-
type: "boolean",
|
|
5855
|
-
index: {
|
|
5856
|
-
getTrue: () => boolIdx.queryEq(true),
|
|
5857
|
-
getFalse: () => boolIdx.queryEq(false),
|
|
5858
|
-
allDocIds: () => boolIdx.getAllDocIds(),
|
|
5859
|
-
getTrueBitset: (cap) => boolIdx.queryEqBitset(true, cap),
|
|
5860
|
-
getFalseBitset: (cap) => boolIdx.queryEqBitset(false, cap),
|
|
5861
|
-
allDocIdsBitset: (cap) => boolIdx.getAllDocIdsBitset(cap)
|
|
5862
|
-
}
|
|
5863
|
-
};
|
|
5864
|
-
}
|
|
5865
|
-
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
5866
|
-
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
5867
|
-
if (enumIdx) {
|
|
5868
|
-
fieldIndexes[fieldPath] = {
|
|
5869
|
-
type: "enum",
|
|
5870
|
-
index: {
|
|
5871
|
-
getDocIds: (v) => enumIdx.queryEq(v),
|
|
5872
|
-
allDocIds: () => enumIdx.getAllDocIds(),
|
|
5873
|
-
getDocIdsBitset: (v, cap) => enumIdx.queryEqBitset(v, cap),
|
|
5874
|
-
getDocIdsInBitset: (values, cap) => enumIdx.queryInBitset(values, cap),
|
|
5875
|
-
allDocIdsBitset: (cap) => enumIdx.getAllDocIdsBitset(cap)
|
|
5876
|
-
}
|
|
5877
|
-
};
|
|
5878
|
-
}
|
|
5879
|
-
} else if (fieldType === "geopoint") {
|
|
5880
|
-
const geoIdx = state.geoIndexes.get(fieldPath);
|
|
5881
|
-
fieldIndexes[fieldPath] = {
|
|
5882
|
-
type: "geopoint",
|
|
5883
|
-
index: geoIdx === void 0 ? EMPTY_GEO_INDEX : geoIdx
|
|
5884
|
-
};
|
|
6020
|
+
// src/core/partition/constants.ts
|
|
6021
|
+
var TARGET_BLOCK_ENTRIES = 128;
|
|
6022
|
+
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
6023
|
+
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
6024
|
+
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
6025
|
+
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
6026
|
+
|
|
6027
|
+
// src/core/partition/sort-columns/order.ts
|
|
6028
|
+
var MISSING_RANK = -1;
|
|
6029
|
+
var BYTES_PER_RANK = 4;
|
|
6030
|
+
var BYTES_PER_ORDERED_ENTRY = 4;
|
|
6031
|
+
function compareAscending(a, b) {
|
|
6032
|
+
return compareComparableValues(a, b, "asc");
|
|
6033
|
+
}
|
|
6034
|
+
function buildOrder(store, liveIds, capacity) {
|
|
6035
|
+
const counts = /* @__PURE__ */ new Map();
|
|
6036
|
+
const present = [];
|
|
6037
|
+
const missing2 = [];
|
|
6038
|
+
for (const internalId of liveIds) {
|
|
6039
|
+
const value = store.get(internalId);
|
|
6040
|
+
if (value === null) {
|
|
6041
|
+
missing2.push(internalId);
|
|
6042
|
+
continue;
|
|
5885
6043
|
}
|
|
5886
|
-
|
|
5887
|
-
|
|
5888
|
-
let cachedAllDocIdsBitset = null;
|
|
5889
|
-
return {
|
|
5890
|
-
fieldTypes: flat,
|
|
5891
|
-
fieldIndexes,
|
|
5892
|
-
getFieldValue: (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath),
|
|
5893
|
-
get allDocIds() {
|
|
5894
|
-
if (!cachedAllDocIds) {
|
|
5895
|
-
cachedAllDocIds = getAllInternalDocIds(state.docStore);
|
|
5896
|
-
}
|
|
5897
|
-
return cachedAllDocIds;
|
|
5898
|
-
},
|
|
5899
|
-
capacity,
|
|
5900
|
-
get allDocIdsBitset() {
|
|
5901
|
-
if (!cachedAllDocIdsBitset) {
|
|
5902
|
-
cachedAllDocIdsBitset = bitsetFromSet(this.allDocIds, capacity);
|
|
5903
|
-
}
|
|
5904
|
-
return cachedAllDocIdsBitset;
|
|
5905
|
-
}
|
|
5906
|
-
};
|
|
5907
|
-
}
|
|
5908
|
-
function applyPartitionFilters(state, filters, schema) {
|
|
5909
|
-
const context = buildFilterContext(state, schema);
|
|
5910
|
-
const resultBitset = evaluateFilters(filters, context);
|
|
5911
|
-
const resolver = state.docStore.resolver();
|
|
5912
|
-
const externalResult = /* @__PURE__ */ new Set();
|
|
5913
|
-
for (let wi = 0; wi < resultBitset.length; wi++) {
|
|
5914
|
-
let word = resultBitset[wi];
|
|
5915
|
-
if (word === 0) continue;
|
|
5916
|
-
const base = wi << 5;
|
|
5917
|
-
while (word !== 0) {
|
|
5918
|
-
const tz = Math.clz32(word & -word) ^ 31;
|
|
5919
|
-
const internalId = base + tz;
|
|
5920
|
-
const externalId = resolver.toExternal(internalId);
|
|
5921
|
-
if (externalId !== void 0) {
|
|
5922
|
-
externalResult.add(externalId);
|
|
5923
|
-
}
|
|
5924
|
-
word &= word - 1;
|
|
5925
|
-
}
|
|
5926
|
-
}
|
|
5927
|
-
return externalResult;
|
|
5928
|
-
}
|
|
5929
|
-
function applyPartitionFiltersBitset(state, filters, schema) {
|
|
5930
|
-
const context = buildFilterContext(state, schema);
|
|
5931
|
-
return evaluateFilters(filters, context);
|
|
5932
|
-
}
|
|
5933
|
-
function partitionFilterMatches(state, filters, schema) {
|
|
5934
|
-
const accepted = applyPartitionFiltersBitset(state, filters, schema);
|
|
5935
|
-
const docStore = state.docStore;
|
|
5936
|
-
const resolver = docStore.resolver();
|
|
5937
|
-
let count = 0;
|
|
5938
|
-
for (let wordIndex = 0; wordIndex < accepted.length; wordIndex++) {
|
|
5939
|
-
let word = accepted[wordIndex];
|
|
5940
|
-
if (word === 0) continue;
|
|
5941
|
-
const base = wordIndex << 5;
|
|
5942
|
-
while (word !== 0) {
|
|
5943
|
-
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
5944
|
-
if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
|
|
5945
|
-
word &= word - 1;
|
|
5946
|
-
}
|
|
5947
|
-
}
|
|
5948
|
-
return {
|
|
5949
|
-
count,
|
|
5950
|
-
hasExternal(docId) {
|
|
5951
|
-
const internalId = docStore.getInternalId(docId);
|
|
5952
|
-
return internalId !== void 0 && bitsetHas(accepted, internalId);
|
|
5953
|
-
},
|
|
5954
|
-
hasInternal(internalId) {
|
|
5955
|
-
return bitsetHas(accepted, internalId);
|
|
5956
|
-
}
|
|
5957
|
-
};
|
|
5958
|
-
}
|
|
5959
|
-
|
|
5960
|
-
// src/core/partition/posting-columns.ts
|
|
5961
|
-
function postingColumns(list) {
|
|
5962
|
-
const deletedDocs = list.deletedDocs;
|
|
5963
|
-
return {
|
|
5964
|
-
docIds: list.docIds,
|
|
5965
|
-
termFrequencies: list.termFrequencies,
|
|
5966
|
-
fieldNameIndices: list.fieldNameIndices,
|
|
5967
|
-
deletedDocs,
|
|
5968
|
-
hasDeleted: deletedDocs.size > 0,
|
|
5969
|
-
count: list.length
|
|
5970
|
-
};
|
|
5971
|
-
}
|
|
5972
|
-
|
|
5973
|
-
// src/core/partition/matches.ts
|
|
5974
|
-
function markPostingList(matched, list, fieldNames, fields, filterBitset) {
|
|
5975
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(list);
|
|
5976
|
-
for (let pi = 0; pi < count; pi++) {
|
|
5977
|
-
const internalId = docIds[pi];
|
|
5978
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5979
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
5980
|
-
if (fields && !fields.includes(fieldNames[fieldNameIndices[pi]])) continue;
|
|
5981
|
-
bitsetSet(matched, internalId);
|
|
5982
|
-
}
|
|
5983
|
-
}
|
|
5984
|
-
function searchFulltextMatches(state, params) {
|
|
5985
|
-
const { queryTokens, prefixExpansion, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
5986
|
-
const fieldNames = state.fieldNameTable.names;
|
|
5987
|
-
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
5988
|
-
for (const qt of queryTokens) {
|
|
5989
|
-
if (prefixExpansion && qt.token === prefixExpansion.token) {
|
|
5990
|
-
const seen = /* @__PURE__ */ new Set();
|
|
5991
|
-
for (const term of [qt.token, ...prefixExpansion.terms]) {
|
|
5992
|
-
if (seen.has(term)) continue;
|
|
5993
|
-
seen.add(term);
|
|
5994
|
-
const postingList = state.invertedIdx.lookup(term);
|
|
5995
|
-
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
5996
|
-
}
|
|
5997
|
-
continue;
|
|
5998
|
-
}
|
|
5999
|
-
if (exact || tolerance === 0) {
|
|
6000
|
-
const postingList = state.invertedIdx.lookup(qt.token);
|
|
6001
|
-
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
6002
|
-
continue;
|
|
6003
|
-
}
|
|
6004
|
-
for (const match of state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength)) {
|
|
6005
|
-
markPostingList(matched, match.postingList, fieldNames, fields, filterBitset);
|
|
6006
|
-
}
|
|
6007
|
-
}
|
|
6008
|
-
const docStore = state.docStore;
|
|
6009
|
-
const resolver = docStore.resolver();
|
|
6010
|
-
let count = 0;
|
|
6011
|
-
for (let wordIndex = 0; wordIndex < matched.length; wordIndex++) {
|
|
6012
|
-
let word = matched[wordIndex];
|
|
6013
|
-
if (word === 0) continue;
|
|
6014
|
-
const base = wordIndex << 5;
|
|
6015
|
-
while (word !== 0) {
|
|
6016
|
-
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
6017
|
-
if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
|
|
6018
|
-
word &= word - 1;
|
|
6019
|
-
}
|
|
6020
|
-
}
|
|
6021
|
-
return {
|
|
6022
|
-
count,
|
|
6023
|
-
hasExternal(docId) {
|
|
6024
|
-
const internalId = docStore.getInternalId(docId);
|
|
6025
|
-
return internalId !== void 0 && bitsetHas(matched, internalId);
|
|
6026
|
-
},
|
|
6027
|
-
hasInternal(internalId) {
|
|
6028
|
-
return bitsetHas(matched, internalId);
|
|
6029
|
-
},
|
|
6030
|
-
matchedDocIds() {
|
|
6031
|
-
const external = /* @__PURE__ */ new Set();
|
|
6032
|
-
for (let wordIndex = 0; wordIndex < matched.length; wordIndex++) {
|
|
6033
|
-
let word = matched[wordIndex];
|
|
6034
|
-
if (word === 0) continue;
|
|
6035
|
-
const base = wordIndex << 5;
|
|
6036
|
-
while (word !== 0) {
|
|
6037
|
-
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
6038
|
-
const externalId = resolver.toExternal(base + trailingZeros);
|
|
6039
|
-
if (externalId !== void 0) external.add(externalId);
|
|
6040
|
-
word &= word - 1;
|
|
6041
|
-
}
|
|
6042
|
-
}
|
|
6043
|
-
return external;
|
|
6044
|
-
},
|
|
6045
|
-
ordinalBitset() {
|
|
6046
|
-
return matched;
|
|
6047
|
-
}
|
|
6048
|
-
};
|
|
6049
|
-
}
|
|
6050
|
-
|
|
6051
|
-
// src/core/partition/constants.ts
|
|
6052
|
-
var TARGET_BLOCK_ENTRIES = 128;
|
|
6053
|
-
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
6054
|
-
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
6055
|
-
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
6056
|
-
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
6057
|
-
|
|
6058
|
-
// src/core/partition/sort-columns/order.ts
|
|
6059
|
-
var MISSING_RANK = -1;
|
|
6060
|
-
var BYTES_PER_RANK = 4;
|
|
6061
|
-
var BYTES_PER_ORDERED_ENTRY = 4;
|
|
6062
|
-
function compareAscending(a, b) {
|
|
6063
|
-
return compareComparableValues(a, b, "asc");
|
|
6064
|
-
}
|
|
6065
|
-
function buildOrder(store, liveIds, capacity) {
|
|
6066
|
-
const counts = /* @__PURE__ */ new Map();
|
|
6067
|
-
const present = [];
|
|
6068
|
-
const missing2 = [];
|
|
6069
|
-
for (const internalId of liveIds) {
|
|
6070
|
-
const value = store.get(internalId);
|
|
6071
|
-
if (value === null) {
|
|
6072
|
-
missing2.push(internalId);
|
|
6073
|
-
continue;
|
|
6074
|
-
}
|
|
6075
|
-
present.push(internalId);
|
|
6076
|
-
counts.set(value, (counts.get(value) ?? 0) + 1);
|
|
6044
|
+
present.push(internalId);
|
|
6045
|
+
counts.set(value, (counts.get(value) ?? 0) + 1);
|
|
6077
6046
|
}
|
|
6078
6047
|
const values = Array.from(counts.keys());
|
|
6079
6048
|
values.sort(compareAscending);
|
|
@@ -6156,8 +6125,9 @@ function grow(length, wanted) {
|
|
|
6156
6125
|
return next;
|
|
6157
6126
|
}
|
|
6158
6127
|
function kindForFieldType(fieldType) {
|
|
6159
|
-
|
|
6160
|
-
if (
|
|
6128
|
+
const base = parsedTypeOf(fieldType)?.base;
|
|
6129
|
+
if (base === "number") return "number";
|
|
6130
|
+
if (base === "boolean") return "boolean";
|
|
6161
6131
|
return "mixed";
|
|
6162
6132
|
}
|
|
6163
6133
|
function createValueStore(initialKind) {
|
|
@@ -6280,7 +6250,8 @@ function createValueStore(initialKind) {
|
|
|
6280
6250
|
|
|
6281
6251
|
// src/core/partition/sort-columns/index.ts
|
|
6282
6252
|
function mayHoldList(fieldType) {
|
|
6283
|
-
|
|
6253
|
+
const parsed = parsedTypeOf(fieldType);
|
|
6254
|
+
return parsed === void 0 || isListBase(parsed.base);
|
|
6284
6255
|
}
|
|
6285
6256
|
function columnKeyOf(field, fieldType, mode) {
|
|
6286
6257
|
return mayHoldList(fieldType) ? `${mode}:${field}` : `value:${field}`;
|
|
@@ -6416,6 +6387,48 @@ function createSortColumnSet(docStore) {
|
|
|
6416
6387
|
};
|
|
6417
6388
|
}
|
|
6418
6389
|
|
|
6390
|
+
// src/core/partition/sort-columns/range.ts
|
|
6391
|
+
function withinBounds(value, bounds) {
|
|
6392
|
+
if (value === null) return false;
|
|
6393
|
+
if (bounds.lower !== null) {
|
|
6394
|
+
const comparison = compareComparableValues(value, bounds.lower, "asc");
|
|
6395
|
+
if (comparison < 0 || comparison === 0 && !bounds.lowerInclusive) return false;
|
|
6396
|
+
}
|
|
6397
|
+
if (bounds.upper !== null) {
|
|
6398
|
+
const comparison = compareComparableValues(value, bounds.upper, "asc");
|
|
6399
|
+
if (comparison > 0 || comparison === 0 && !bounds.upperInclusive) return false;
|
|
6400
|
+
}
|
|
6401
|
+
return true;
|
|
6402
|
+
}
|
|
6403
|
+
function selectTextRange(column, requested, capacity, isLive2) {
|
|
6404
|
+
const bounds = {
|
|
6405
|
+
...requested,
|
|
6406
|
+
lower: requested.lower === null ? null : truncateSortString(requested.lower),
|
|
6407
|
+
upper: requested.upper === null ? null : truncateSortString(requested.upper)
|
|
6408
|
+
};
|
|
6409
|
+
const order = column.order;
|
|
6410
|
+
const ordered = order.ordered;
|
|
6411
|
+
const result = createBitSet(capacity);
|
|
6412
|
+
let start = 0;
|
|
6413
|
+
if (bounds.lower !== null) {
|
|
6414
|
+
const lowerRank = rankOfValue(order, bounds.lower);
|
|
6415
|
+
start = column.seek(bounds.lowerInclusive ? lowerRank : lowerRank + 1, "asc");
|
|
6416
|
+
}
|
|
6417
|
+
let end = ordered.length;
|
|
6418
|
+
if (bounds.upper !== null) {
|
|
6419
|
+
const upperRank = rankOfValue(order, bounds.upper);
|
|
6420
|
+
end = column.seek(bounds.upperInclusive ? upperRank + 1 : upperRank, "asc");
|
|
6421
|
+
}
|
|
6422
|
+
for (let position = start; position < end; position++) {
|
|
6423
|
+
const internalId = ordered[position];
|
|
6424
|
+
if (!column.isDirty(internalId) && isLive2(internalId)) bitsetSet(result, internalId);
|
|
6425
|
+
}
|
|
6426
|
+
for (const internalId of column.dirtyStream().present) {
|
|
6427
|
+
if (withinBounds(column.valueOf(internalId), bounds)) bitsetSet(result, internalId);
|
|
6428
|
+
}
|
|
6429
|
+
return result;
|
|
6430
|
+
}
|
|
6431
|
+
|
|
6419
6432
|
// src/core/partition/sort-columns/select.ts
|
|
6420
6433
|
function rankOrdersAfter(rank, other, direction) {
|
|
6421
6434
|
if (rank === MISSING_RANK) return other !== MISSING_RANK;
|
|
@@ -6573,48 +6586,289 @@ function selectSortedPage(state, request) {
|
|
|
6573
6586
|
if (compareCandidates(columns, directions, candidate, page[limit - 1]) >= 0) continue;
|
|
6574
6587
|
page.pop();
|
|
6575
6588
|
}
|
|
6576
|
-
insertOrdered(page, candidate, columns, directions);
|
|
6577
|
-
}
|
|
6578
|
-
return page.map((entry) => ({ id: entry.externalId, key: entry.key }));
|
|
6579
|
-
}
|
|
6580
|
-
|
|
6581
|
-
// src/core/partition/sorting.ts
|
|
6582
|
-
function ensureColumns(state) {
|
|
6583
|
-
if (state.sortColumns === null) {
|
|
6584
|
-
state.sortColumns = createSortColumnSet(state.docStore);
|
|
6589
|
+
insertOrdered(page, candidate, columns, directions);
|
|
6590
|
+
}
|
|
6591
|
+
return page.map((entry) => ({ id: entry.externalId, key: entry.key }));
|
|
6592
|
+
}
|
|
6593
|
+
|
|
6594
|
+
// src/core/partition/sorting.ts
|
|
6595
|
+
function ensureColumns(state) {
|
|
6596
|
+
if (state.sortColumns === null) {
|
|
6597
|
+
state.sortColumns = createSortColumnSet(state.docStore);
|
|
6598
|
+
}
|
|
6599
|
+
return state.sortColumns;
|
|
6600
|
+
}
|
|
6601
|
+
function textRangeBitsetOf(state, fieldPath, fieldType, bounds, capacity) {
|
|
6602
|
+
const column = ensureColumns(state).column(fieldPath, fieldType, "min");
|
|
6603
|
+
const resolver = state.docStore.resolver();
|
|
6604
|
+
return selectTextRange(column, bounds, capacity, (internalId) => resolver.toExternal(internalId) !== void 0);
|
|
6605
|
+
}
|
|
6606
|
+
function sortedPageOf(state, request) {
|
|
6607
|
+
ensureColumns(state);
|
|
6608
|
+
return selectSortedPage(state, request);
|
|
6609
|
+
}
|
|
6610
|
+
function sortValuesOf(state, docId, fields, fieldTypes, modes) {
|
|
6611
|
+
const set = state.sortColumns;
|
|
6612
|
+
const internalId = state.docStore.getInternalId(docId);
|
|
6613
|
+
const stored = state.docStore.get(docId);
|
|
6614
|
+
const key = new Array(fields.length);
|
|
6615
|
+
for (let i = 0; i < fields.length; i++) {
|
|
6616
|
+
const mode = modes[i] ?? "min";
|
|
6617
|
+
if (set !== null && internalId !== void 0 && set.holds(fields[i], fieldTypes[i], mode)) {
|
|
6618
|
+
key[i] = set.column(fields[i], fieldTypes[i], mode).valueOf(internalId);
|
|
6619
|
+
continue;
|
|
6620
|
+
}
|
|
6621
|
+
key[i] = stored === void 0 ? null : toReducedSortValue(readSortField(stored.fields, fields[i]), mode);
|
|
6622
|
+
}
|
|
6623
|
+
return key;
|
|
6624
|
+
}
|
|
6625
|
+
function recordSortValues(state, internalId, document2) {
|
|
6626
|
+
state.sortColumns?.record(internalId, document2);
|
|
6627
|
+
}
|
|
6628
|
+
function forgetSortValues(state, internalId) {
|
|
6629
|
+
if (internalId === void 0) return;
|
|
6630
|
+
state.sortColumns?.forget(internalId);
|
|
6631
|
+
}
|
|
6632
|
+
function refreshSortColumns(state) {
|
|
6633
|
+
state.sortColumns?.refresh();
|
|
6634
|
+
}
|
|
6635
|
+
function sortColumnBytes(state) {
|
|
6636
|
+
return state.sortColumns === null ? 0 : state.sortColumns.estimateBytes();
|
|
6637
|
+
}
|
|
6638
|
+
|
|
6639
|
+
// src/core/partition/filters.ts
|
|
6640
|
+
var EMPTY_GEO_INDEX = {
|
|
6641
|
+
radiusQuery: () => /* @__PURE__ */ new Set(),
|
|
6642
|
+
polygonQuery: () => /* @__PURE__ */ new Set()
|
|
6643
|
+
};
|
|
6644
|
+
function buildFilterContext(state, schema) {
|
|
6645
|
+
const flat = getFlatSchema(state, schema);
|
|
6646
|
+
const fieldIndexes = {};
|
|
6647
|
+
const capacity = state.docStore.internalIdCapacity();
|
|
6648
|
+
for (const { path: fieldPath, base: fieldType } of schemaFieldsOf(schema)) {
|
|
6649
|
+
if (fieldType === "number" || fieldType === "number[]") {
|
|
6650
|
+
const numIdx = state.numericIndexes.get(fieldPath);
|
|
6651
|
+
if (numIdx) {
|
|
6652
|
+
fieldIndexes[fieldPath] = {
|
|
6653
|
+
type: "numeric",
|
|
6654
|
+
index: {
|
|
6655
|
+
eq: (v) => numIdx.queryEq(v),
|
|
6656
|
+
gt: (v) => numIdx.queryGt(v),
|
|
6657
|
+
gte: (v) => numIdx.queryGte(v),
|
|
6658
|
+
lt: (v) => numIdx.queryLt(v),
|
|
6659
|
+
lte: (v) => numIdx.queryLte(v),
|
|
6660
|
+
between: (min, max) => numIdx.queryBetween(min, max),
|
|
6661
|
+
allDocIds: () => numIdx.getAllDocIds(),
|
|
6662
|
+
eqBitset: (v, cap) => numIdx.queryEqBitset(v, cap),
|
|
6663
|
+
gtBitset: (v, cap) => numIdx.queryGtBitset(v, cap),
|
|
6664
|
+
gteBitset: (v, cap) => numIdx.queryGteBitset(v, cap),
|
|
6665
|
+
ltBitset: (v, cap) => numIdx.queryLtBitset(v, cap),
|
|
6666
|
+
lteBitset: (v, cap) => numIdx.queryLteBitset(v, cap),
|
|
6667
|
+
betweenBitset: (min, max, cap) => numIdx.queryBetweenBitset(min, max, cap),
|
|
6668
|
+
allDocIdsBitset: (cap) => numIdx.getAllDocIdsBitset(cap)
|
|
6669
|
+
}
|
|
6670
|
+
};
|
|
6671
|
+
}
|
|
6672
|
+
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
6673
|
+
const boolIdx = state.booleanIndexes.get(fieldPath);
|
|
6674
|
+
if (boolIdx) {
|
|
6675
|
+
fieldIndexes[fieldPath] = {
|
|
6676
|
+
type: "boolean",
|
|
6677
|
+
index: {
|
|
6678
|
+
getTrue: () => boolIdx.queryEq(true),
|
|
6679
|
+
getFalse: () => boolIdx.queryEq(false),
|
|
6680
|
+
allDocIds: () => boolIdx.getAllDocIds(),
|
|
6681
|
+
getTrueBitset: (cap) => boolIdx.queryEqBitset(true, cap),
|
|
6682
|
+
getFalseBitset: (cap) => boolIdx.queryEqBitset(false, cap),
|
|
6683
|
+
allDocIdsBitset: (cap) => boolIdx.getAllDocIdsBitset(cap)
|
|
6684
|
+
}
|
|
6685
|
+
};
|
|
6686
|
+
}
|
|
6687
|
+
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
6688
|
+
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
6689
|
+
if (enumIdx) {
|
|
6690
|
+
fieldIndexes[fieldPath] = {
|
|
6691
|
+
type: "enum",
|
|
6692
|
+
index: {
|
|
6693
|
+
getDocIds: (v) => enumIdx.queryEq(v),
|
|
6694
|
+
allDocIds: () => enumIdx.getAllDocIds(),
|
|
6695
|
+
getDocIdsBitset: (v, cap) => enumIdx.queryEqBitset(v, cap),
|
|
6696
|
+
getDocIdsInBitset: (values, cap) => enumIdx.queryInBitset(values, cap),
|
|
6697
|
+
allDocIdsBitset: (cap) => enumIdx.getAllDocIdsBitset(cap)
|
|
6698
|
+
}
|
|
6699
|
+
};
|
|
6700
|
+
}
|
|
6701
|
+
} else if (fieldType === "geopoint") {
|
|
6702
|
+
const geoIdx = state.geoIndexes.get(fieldPath);
|
|
6703
|
+
fieldIndexes[fieldPath] = {
|
|
6704
|
+
type: "geopoint",
|
|
6705
|
+
index: geoIdx === void 0 ? EMPTY_GEO_INDEX : geoIdx
|
|
6706
|
+
};
|
|
6707
|
+
}
|
|
6708
|
+
}
|
|
6709
|
+
let cachedAllDocIds = null;
|
|
6710
|
+
let cachedAllDocIdsBitset = null;
|
|
6711
|
+
return {
|
|
6712
|
+
fieldTypes: flat,
|
|
6713
|
+
fieldIndexes,
|
|
6714
|
+
getFieldValue: (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath),
|
|
6715
|
+
textRangeBitset: (fieldPath, bounds) => textRangeBitsetOf(state, fieldPath, flat[fieldPath], bounds, capacity),
|
|
6716
|
+
get allDocIds() {
|
|
6717
|
+
if (!cachedAllDocIds) {
|
|
6718
|
+
cachedAllDocIds = getAllInternalDocIds(state.docStore);
|
|
6719
|
+
}
|
|
6720
|
+
return cachedAllDocIds;
|
|
6721
|
+
},
|
|
6722
|
+
capacity,
|
|
6723
|
+
get allDocIdsBitset() {
|
|
6724
|
+
if (!cachedAllDocIdsBitset) {
|
|
6725
|
+
cachedAllDocIdsBitset = bitsetFromSet(this.allDocIds, capacity);
|
|
6726
|
+
}
|
|
6727
|
+
return cachedAllDocIdsBitset;
|
|
6728
|
+
}
|
|
6729
|
+
};
|
|
6730
|
+
}
|
|
6731
|
+
function applyPartitionFilters(state, filters, schema) {
|
|
6732
|
+
const context = buildFilterContext(state, schema);
|
|
6733
|
+
const resultBitset = evaluateFilters(filters, context);
|
|
6734
|
+
const resolver = state.docStore.resolver();
|
|
6735
|
+
const externalResult = /* @__PURE__ */ new Set();
|
|
6736
|
+
for (let wi = 0; wi < resultBitset.length; wi++) {
|
|
6737
|
+
let word = resultBitset[wi];
|
|
6738
|
+
if (word === 0) continue;
|
|
6739
|
+
const base = wi << 5;
|
|
6740
|
+
while (word !== 0) {
|
|
6741
|
+
const tz = Math.clz32(word & -word) ^ 31;
|
|
6742
|
+
const internalId = base + tz;
|
|
6743
|
+
const externalId = resolver.toExternal(internalId);
|
|
6744
|
+
if (externalId !== void 0) {
|
|
6745
|
+
externalResult.add(externalId);
|
|
6746
|
+
}
|
|
6747
|
+
word &= word - 1;
|
|
6748
|
+
}
|
|
6749
|
+
}
|
|
6750
|
+
return externalResult;
|
|
6751
|
+
}
|
|
6752
|
+
function applyPartitionFiltersBitset(state, filters, schema) {
|
|
6753
|
+
const context = buildFilterContext(state, schema);
|
|
6754
|
+
return evaluateFilters(filters, context);
|
|
6755
|
+
}
|
|
6756
|
+
function partitionFilterMatches(state, filters, schema) {
|
|
6757
|
+
const accepted = applyPartitionFiltersBitset(state, filters, schema);
|
|
6758
|
+
const docStore = state.docStore;
|
|
6759
|
+
const resolver = docStore.resolver();
|
|
6760
|
+
let count = 0;
|
|
6761
|
+
for (let wordIndex = 0; wordIndex < accepted.length; wordIndex++) {
|
|
6762
|
+
let word = accepted[wordIndex];
|
|
6763
|
+
if (word === 0) continue;
|
|
6764
|
+
const base = wordIndex << 5;
|
|
6765
|
+
while (word !== 0) {
|
|
6766
|
+
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
6767
|
+
if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
|
|
6768
|
+
word &= word - 1;
|
|
6769
|
+
}
|
|
6770
|
+
}
|
|
6771
|
+
return {
|
|
6772
|
+
count,
|
|
6773
|
+
hasExternal(docId) {
|
|
6774
|
+
const internalId = docStore.getInternalId(docId);
|
|
6775
|
+
return internalId !== void 0 && bitsetHas(accepted, internalId);
|
|
6776
|
+
},
|
|
6777
|
+
hasInternal(internalId) {
|
|
6778
|
+
return bitsetHas(accepted, internalId);
|
|
6779
|
+
}
|
|
6780
|
+
};
|
|
6781
|
+
}
|
|
6782
|
+
|
|
6783
|
+
// src/core/partition/posting-columns.ts
|
|
6784
|
+
function postingColumns(list) {
|
|
6785
|
+
const deletedDocs = list.deletedDocs;
|
|
6786
|
+
return {
|
|
6787
|
+
docIds: list.docIds,
|
|
6788
|
+
termFrequencies: list.termFrequencies,
|
|
6789
|
+
fieldNameIndices: list.fieldNameIndices,
|
|
6790
|
+
deletedDocs,
|
|
6791
|
+
hasDeleted: deletedDocs.size > 0,
|
|
6792
|
+
count: list.length
|
|
6793
|
+
};
|
|
6794
|
+
}
|
|
6795
|
+
|
|
6796
|
+
// src/core/partition/matches.ts
|
|
6797
|
+
function markPostingList(matched, list, fieldNames, fields, filterBitset) {
|
|
6798
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(list);
|
|
6799
|
+
for (let pi = 0; pi < count; pi++) {
|
|
6800
|
+
const internalId = docIds[pi];
|
|
6801
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
6802
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
6803
|
+
if (fields && !fields.includes(fieldNames[fieldNameIndices[pi]])) continue;
|
|
6804
|
+
bitsetSet(matched, internalId);
|
|
6805
|
+
}
|
|
6806
|
+
}
|
|
6807
|
+
function searchFulltextMatches(state, params) {
|
|
6808
|
+
const { queryTokens, prefixExpansion, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
6809
|
+
const fieldNames = state.fieldNameTable.names;
|
|
6810
|
+
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
6811
|
+
for (const qt of queryTokens) {
|
|
6812
|
+
if (prefixExpansion && qt.token === prefixExpansion.token) {
|
|
6813
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6814
|
+
for (const term of [qt.token, ...prefixExpansion.terms]) {
|
|
6815
|
+
if (seen.has(term)) continue;
|
|
6816
|
+
seen.add(term);
|
|
6817
|
+
const postingList = state.invertedIdx.lookup(term);
|
|
6818
|
+
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
6819
|
+
}
|
|
6820
|
+
continue;
|
|
6821
|
+
}
|
|
6822
|
+
if (exact || tolerance === 0) {
|
|
6823
|
+
const postingList = state.invertedIdx.lookup(qt.token);
|
|
6824
|
+
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
6825
|
+
continue;
|
|
6826
|
+
}
|
|
6827
|
+
for (const match of state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength)) {
|
|
6828
|
+
markPostingList(matched, match.postingList, fieldNames, fields, filterBitset);
|
|
6829
|
+
}
|
|
6585
6830
|
}
|
|
6586
|
-
|
|
6587
|
-
|
|
6588
|
-
|
|
6589
|
-
|
|
6590
|
-
|
|
6591
|
-
|
|
6592
|
-
|
|
6593
|
-
|
|
6594
|
-
|
|
6595
|
-
|
|
6596
|
-
|
|
6597
|
-
const mode = modes[i] ?? "min";
|
|
6598
|
-
if (set !== null && internalId !== void 0 && set.holds(fields[i], fieldTypes[i], mode)) {
|
|
6599
|
-
key[i] = set.column(fields[i], fieldTypes[i], mode).valueOf(internalId);
|
|
6600
|
-
continue;
|
|
6831
|
+
const docStore = state.docStore;
|
|
6832
|
+
const resolver = docStore.resolver();
|
|
6833
|
+
let count = 0;
|
|
6834
|
+
for (let wordIndex = 0; wordIndex < matched.length; wordIndex++) {
|
|
6835
|
+
let word = matched[wordIndex];
|
|
6836
|
+
if (word === 0) continue;
|
|
6837
|
+
const base = wordIndex << 5;
|
|
6838
|
+
while (word !== 0) {
|
|
6839
|
+
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
6840
|
+
if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
|
|
6841
|
+
word &= word - 1;
|
|
6601
6842
|
}
|
|
6602
|
-
key[i] = stored === void 0 ? null : toReducedSortValue(readSortField(stored.fields, fields[i]), mode);
|
|
6603
6843
|
}
|
|
6604
|
-
return
|
|
6605
|
-
|
|
6606
|
-
|
|
6607
|
-
|
|
6608
|
-
|
|
6609
|
-
|
|
6610
|
-
|
|
6611
|
-
|
|
6612
|
-
}
|
|
6613
|
-
|
|
6614
|
-
|
|
6615
|
-
|
|
6616
|
-
|
|
6617
|
-
|
|
6844
|
+
return {
|
|
6845
|
+
count,
|
|
6846
|
+
hasExternal(docId) {
|
|
6847
|
+
const internalId = docStore.getInternalId(docId);
|
|
6848
|
+
return internalId !== void 0 && bitsetHas(matched, internalId);
|
|
6849
|
+
},
|
|
6850
|
+
hasInternal(internalId) {
|
|
6851
|
+
return bitsetHas(matched, internalId);
|
|
6852
|
+
},
|
|
6853
|
+
matchedDocIds() {
|
|
6854
|
+
const external = /* @__PURE__ */ new Set();
|
|
6855
|
+
for (let wordIndex = 0; wordIndex < matched.length; wordIndex++) {
|
|
6856
|
+
let word = matched[wordIndex];
|
|
6857
|
+
if (word === 0) continue;
|
|
6858
|
+
const base = wordIndex << 5;
|
|
6859
|
+
while (word !== 0) {
|
|
6860
|
+
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
6861
|
+
const externalId = resolver.toExternal(base + trailingZeros);
|
|
6862
|
+
if (externalId !== void 0) external.add(externalId);
|
|
6863
|
+
word &= word - 1;
|
|
6864
|
+
}
|
|
6865
|
+
}
|
|
6866
|
+
return external;
|
|
6867
|
+
},
|
|
6868
|
+
ordinalBitset() {
|
|
6869
|
+
return matched;
|
|
6870
|
+
}
|
|
6871
|
+
};
|
|
6618
6872
|
}
|
|
6619
6873
|
|
|
6620
6874
|
// src/core/partition/memory.ts
|
|
@@ -6789,11 +7043,11 @@ var EMPTY_STATS = {
|
|
|
6789
7043
|
averageFieldLengths: {},
|
|
6790
7044
|
docFrequencies: {}
|
|
6791
7045
|
};
|
|
6792
|
-
function textFieldsOf(
|
|
6793
|
-
return
|
|
7046
|
+
function textFieldsOf(fields) {
|
|
7047
|
+
return fields.filter((field) => isWordIndexedBase(field.base));
|
|
6794
7048
|
}
|
|
6795
7049
|
function rebuildTextIndex(state, schema, language, options) {
|
|
6796
|
-
const textFields = textFieldsOf(
|
|
7050
|
+
const textFields = textFieldsOf(schemaFieldsOf(schema));
|
|
6797
7051
|
state.invertedIdx.clear();
|
|
6798
7052
|
state.surfaceRegistry.clear();
|
|
6799
7053
|
state.stats.deserialize(EMPTY_STATS);
|
|
@@ -6805,10 +7059,10 @@ function rebuildTextIndex(state, schema, language, options) {
|
|
|
6805
7059
|
const fields = stored.fields;
|
|
6806
7060
|
const fieldLengths = {};
|
|
6807
7061
|
const tokensByField = {};
|
|
6808
|
-
for (const
|
|
7062
|
+
for (const { path: fieldPath, base } of textFields) {
|
|
6809
7063
|
const value = getNestedValue(fields, fieldPath);
|
|
6810
7064
|
if (value === void 0 || value === null) continue;
|
|
6811
|
-
if (
|
|
7065
|
+
if (base === "string") {
|
|
6812
7066
|
indexStringField(
|
|
6813
7067
|
state,
|
|
6814
7068
|
internalId,
|
|
@@ -8485,18 +8739,17 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8485
8739
|
);
|
|
8486
8740
|
}
|
|
8487
8741
|
if (options?.validate !== false) {
|
|
8488
|
-
validateDocument(document2, schema);
|
|
8742
|
+
validateDocument(document2, schema, options?.patternValueLimit);
|
|
8489
8743
|
if (options?.strict) {
|
|
8490
8744
|
validateDocumentStrict(document2, schema);
|
|
8491
8745
|
}
|
|
8492
8746
|
}
|
|
8493
8747
|
const internalId = state.docStore.ensureInternalId(docId);
|
|
8494
|
-
const flatSchema = getFlatSchema(state, schema);
|
|
8495
8748
|
const { fieldLengths, tokensByField } = indexDocument(
|
|
8496
8749
|
state,
|
|
8497
8750
|
docId,
|
|
8498
8751
|
document2,
|
|
8499
|
-
|
|
8752
|
+
schemaFieldsOf(schema),
|
|
8500
8753
|
language,
|
|
8501
8754
|
options
|
|
8502
8755
|
);
|
|
@@ -8516,8 +8769,14 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8516
8769
|
partitionId
|
|
8517
8770
|
});
|
|
8518
8771
|
}
|
|
8519
|
-
const
|
|
8520
|
-
|
|
8772
|
+
const { fieldLengths, tokensByField } = removeFromIndexes(
|
|
8773
|
+
state,
|
|
8774
|
+
docId,
|
|
8775
|
+
stored,
|
|
8776
|
+
schemaFieldsOf(schema),
|
|
8777
|
+
language,
|
|
8778
|
+
options
|
|
8779
|
+
);
|
|
8521
8780
|
const internalId = state.docStore.getInternalId(docId);
|
|
8522
8781
|
state.docStore.remove(docId);
|
|
8523
8782
|
state.stats.removeDocument(fieldLengths, tokensByField);
|
|
@@ -8548,19 +8807,19 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8548
8807
|
});
|
|
8549
8808
|
}
|
|
8550
8809
|
if (options?.validate !== false) {
|
|
8551
|
-
validateDocument(document2, schema);
|
|
8810
|
+
validateDocument(document2, schema, options?.patternValueLimit);
|
|
8552
8811
|
if (options?.strict) {
|
|
8553
8812
|
validateDocumentStrict(document2, schema);
|
|
8554
8813
|
}
|
|
8555
8814
|
}
|
|
8556
|
-
const
|
|
8557
|
-
const needsTextReindex = textFieldsChanged(stored.fields, document2,
|
|
8815
|
+
const fields = schemaFieldsOf(schema);
|
|
8816
|
+
const needsTextReindex = textFieldsChanged(stored.fields, document2, fields);
|
|
8558
8817
|
if (needsTextReindex) {
|
|
8559
8818
|
const { fieldLengths: oldFieldLengths, tokensByField: oldTokens } = removeFromIndexes(
|
|
8560
8819
|
state,
|
|
8561
8820
|
docId,
|
|
8562
8821
|
stored,
|
|
8563
|
-
|
|
8822
|
+
fields,
|
|
8564
8823
|
language,
|
|
8565
8824
|
options
|
|
8566
8825
|
);
|
|
@@ -8573,7 +8832,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8573
8832
|
state,
|
|
8574
8833
|
docId,
|
|
8575
8834
|
document2,
|
|
8576
|
-
|
|
8835
|
+
fields,
|
|
8577
8836
|
language,
|
|
8578
8837
|
options
|
|
8579
8838
|
);
|
|
@@ -8581,7 +8840,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8581
8840
|
state.stats.addDocument(newFieldLengths, newTokens);
|
|
8582
8841
|
recordSortValues(state, internalId, document2);
|
|
8583
8842
|
} else {
|
|
8584
|
-
updateFieldIndexOnly(state, docId, stored.fields, document2,
|
|
8843
|
+
updateFieldIndexOnly(state, docId, stored.fields, document2, fields);
|
|
8585
8844
|
state.docStore.store(docId, document2, stored.fieldLengths);
|
|
8586
8845
|
const internalId = state.docStore.getInternalId(docId);
|
|
8587
8846
|
if (internalId !== void 0) recordSortValues(state, internalId, document2);
|
|
@@ -8973,7 +9232,7 @@ function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
|
8973
9232
|
{ field: source, targetField }
|
|
8974
9233
|
);
|
|
8975
9234
|
}
|
|
8976
|
-
if (!
|
|
9235
|
+
if (!isSingleStringFieldType(sourceType)) {
|
|
8977
9236
|
throw new NarsilError(
|
|
8978
9237
|
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
8979
9238
|
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
@@ -9389,10 +9648,11 @@ function requireSortableFields(sort, schema, strict = false) {
|
|
|
9389
9648
|
);
|
|
9390
9649
|
}
|
|
9391
9650
|
requireSortMode(entry, fieldType);
|
|
9392
|
-
|
|
9651
|
+
const parsed = parsedTypeOf(fieldType);
|
|
9652
|
+
if (parsed !== void 0 && isStringValuedBase(parsed.base) && !parsed.sortable) {
|
|
9393
9653
|
throw new NarsilError(
|
|
9394
9654
|
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
9395
|
-
`The engine sorts by text field "${entry.field}" only where
|
|
9655
|
+
`The engine sorts by text field "${entry.field}" only where its type includes "sortable", as in "${withSortableOption(parsed)}", because it keeps up to the first ${SORT_VALUE_MAX_CODE_POINTS} code points of every document's value in memory to sort by a text field`,
|
|
9396
9656
|
{ field: entry.field, fieldType }
|
|
9397
9657
|
);
|
|
9398
9658
|
}
|
|
@@ -9426,22 +9686,6 @@ function invalidCursor(cursor, reason) {
|
|
|
9426
9686
|
const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
|
|
9427
9687
|
return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
|
|
9428
9688
|
}
|
|
9429
|
-
function exceedsCodePoints(value, maximum) {
|
|
9430
|
-
if (value.length <= maximum) return false;
|
|
9431
|
-
let counted = 0;
|
|
9432
|
-
let i = 0;
|
|
9433
|
-
while (i < value.length) {
|
|
9434
|
-
const unit = value.charCodeAt(i);
|
|
9435
|
-
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
9436
|
-
const low = value.charCodeAt(i + 1);
|
|
9437
|
-
if (low >= 56320 && low < 57344) i++;
|
|
9438
|
-
}
|
|
9439
|
-
i++;
|
|
9440
|
-
counted++;
|
|
9441
|
-
if (counted > maximum) return true;
|
|
9442
|
-
}
|
|
9443
|
-
return false;
|
|
9444
|
-
}
|
|
9445
9689
|
function sortSignatureOf(sort) {
|
|
9446
9690
|
const fields = normalizeSort(sort);
|
|
9447
9691
|
if (fields.length === 0) return null;
|
|
@@ -9770,7 +10014,7 @@ function validateSearchFields(fields, flatSchema) {
|
|
|
9770
10014
|
if (!fieldType) {
|
|
9771
10015
|
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FIELD, `Field "${field}" does not exist in the schema`, { field });
|
|
9772
10016
|
}
|
|
9773
|
-
if (!
|
|
10017
|
+
if (!isWordIndexedFieldType(fieldType)) {
|
|
9774
10018
|
throw new NarsilError(
|
|
9775
10019
|
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
9776
10020
|
`Field "${field}" has type "${fieldType}" which cannot be used for full-text search`,
|
|
@@ -10638,9 +10882,11 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
10638
10882
|
const applyStrict = config.strict === true;
|
|
10639
10883
|
const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
|
|
10640
10884
|
const applySurfaces = config.surfaceForms !== false;
|
|
10641
|
-
|
|
10885
|
+
const applyPatternLimit = config.patternValueLimit !== void 0;
|
|
10886
|
+
if (!applyStrict && !applyAnalyzer && !applySurfaces && !applyPatternLimit) return options;
|
|
10642
10887
|
const resolved = { ...options };
|
|
10643
10888
|
if (applyStrict) resolved.strict = true;
|
|
10889
|
+
if (applyPatternLimit) resolved.patternValueLimit = config.patternValueLimit;
|
|
10644
10890
|
if (applyAnalyzer) {
|
|
10645
10891
|
resolved.stopWordOverride = options?.stopWordOverride ?? analysis.stopWords;
|
|
10646
10892
|
resolved.customTokenizer = options?.customTokenizer ?? analysis.customTokenizer;
|
|
@@ -10649,6 +10895,44 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
10649
10895
|
return resolved;
|
|
10650
10896
|
}
|
|
10651
10897
|
|
|
10898
|
+
// src/partitioning/manager/capacity.ts
|
|
10899
|
+
function createCapacityChecks(source) {
|
|
10900
|
+
const { indexName, config } = source;
|
|
10901
|
+
return {
|
|
10902
|
+
assertCapacity(pendingWrites = 0, partitionCountCap) {
|
|
10903
|
+
const maxDocsPerPartition = config.partitions?.maxDocsPerPartition;
|
|
10904
|
+
if (maxDocsPerPartition === void 0) return;
|
|
10905
|
+
const partitionCount = source.partitionCount();
|
|
10906
|
+
const effectivePartitionCount = partitionCountCap === void 0 ? partitionCount : Math.min(partitionCount, partitionCountCap);
|
|
10907
|
+
const totalCapacity = maxDocsPerPartition * effectivePartitionCount;
|
|
10908
|
+
const currentCount = source.documentCount() + pendingWrites;
|
|
10909
|
+
if (currentCount < totalCapacity) return;
|
|
10910
|
+
throw new NarsilError(
|
|
10911
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
10912
|
+
`Index "${indexName}" has reached its capacity of ${totalCapacity} documents (${maxDocsPerPartition} per partition \xD7 ${effectivePartitionCount} partitions)`,
|
|
10913
|
+
{
|
|
10914
|
+
indexName,
|
|
10915
|
+
currentCount,
|
|
10916
|
+
totalCapacity,
|
|
10917
|
+
maxDocsPerPartition,
|
|
10918
|
+
partitionCount: effectivePartitionCount
|
|
10919
|
+
}
|
|
10920
|
+
);
|
|
10921
|
+
},
|
|
10922
|
+
assertPartitionCapacity(partitionId, pendingWrites = 0) {
|
|
10923
|
+
const maxDocsPerPartition = config.partitions?.maxDocsPerPartition;
|
|
10924
|
+
if (maxDocsPerPartition === void 0) return;
|
|
10925
|
+
const currentCount = source.getPartition(partitionId).count() + pendingWrites;
|
|
10926
|
+
if (currentCount < maxDocsPerPartition) return;
|
|
10927
|
+
throw new NarsilError(
|
|
10928
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
10929
|
+
`Partition ${partitionId} of index "${indexName}" has reached its capacity of ${maxDocsPerPartition} documents`,
|
|
10930
|
+
{ indexName, partitionId, currentCount, maxDocsPerPartition }
|
|
10931
|
+
);
|
|
10932
|
+
}
|
|
10933
|
+
};
|
|
10934
|
+
}
|
|
10935
|
+
|
|
10652
10936
|
// src/partitioning/manager/nested-values.ts
|
|
10653
10937
|
function setNestedValue(obj, path, value) {
|
|
10654
10938
|
if (!path.includes(".")) {
|
|
@@ -10795,27 +11079,17 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
10795
11079
|
}
|
|
10796
11080
|
partitions.length = newCount;
|
|
10797
11081
|
},
|
|
10798
|
-
|
|
10799
|
-
|
|
10800
|
-
|
|
10801
|
-
|
|
10802
|
-
|
|
10803
|
-
|
|
10804
|
-
|
|
10805
|
-
|
|
10806
|
-
|
|
10807
|
-
{
|
|
10808
|
-
indexName,
|
|
10809
|
-
currentCount: docPartitionMap.size + pendingWrites,
|
|
10810
|
-
totalCapacity,
|
|
10811
|
-
maxDocsPerPartition: currentMaxDocs,
|
|
10812
|
-
partitionCount: effectivePartitionCount
|
|
10813
|
-
}
|
|
10814
|
-
);
|
|
10815
|
-
}
|
|
11082
|
+
...createCapacityChecks({
|
|
11083
|
+
indexName,
|
|
11084
|
+
config,
|
|
11085
|
+
partitionCount: () => partitions.length,
|
|
11086
|
+
documentCount: () => manager.countDocuments(),
|
|
11087
|
+
getPartition: (partitionId) => manager.getPartition(partitionId)
|
|
11088
|
+
}),
|
|
11089
|
+
routePartition(docId) {
|
|
11090
|
+
return router.route(docId, partitions.length);
|
|
10816
11091
|
},
|
|
10817
11092
|
insert(docId, document2, options) {
|
|
10818
|
-
manager.assertCapacity();
|
|
10819
11093
|
const pid = router.route(docId, partitions.length);
|
|
10820
11094
|
const insertOpts = resolveInsertOptions(options);
|
|
10821
11095
|
partitions[pid].insert(docId, document2, config.schema, language, insertOpts);
|
|
@@ -18802,7 +19076,7 @@ function refuseRepeatedDocIds(documents) {
|
|
|
18802
19076
|
function validateAsAnInsertDoes(documents, schema, options) {
|
|
18803
19077
|
if (options?.validate === false) return;
|
|
18804
19078
|
for (const { document: document2 } of documents) {
|
|
18805
|
-
validateDocument(document2, schema);
|
|
19079
|
+
validateDocument(document2, schema, options?.patternValueLimit);
|
|
18806
19080
|
if (options?.strict) validateDocumentStrict(document2, schema);
|
|
18807
19081
|
}
|
|
18808
19082
|
}
|
|
@@ -19012,7 +19286,7 @@ var FieldLengths = class {
|
|
|
19012
19286
|
function buildSegmentPayload(documents, schema, language, options, trackPositions) {
|
|
19013
19287
|
refuseRepeatedDocIds(documents);
|
|
19014
19288
|
validateAsAnInsertDoes(documents, schema, options);
|
|
19015
|
-
const fields =
|
|
19289
|
+
const fields = schemaFieldsOf(schema);
|
|
19016
19290
|
const tokenizing = tokenizeOptions(options);
|
|
19017
19291
|
const postings = new TextPostings(trackPositions);
|
|
19018
19292
|
const surfaces = new SurfaceCounts();
|
|
@@ -19030,10 +19304,10 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
19030
19304
|
};
|
|
19031
19305
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
19032
19306
|
const document2 = documents[ordinal].document;
|
|
19033
|
-
for (const
|
|
19307
|
+
for (const { path: fieldPath, base: fieldType } of fields) {
|
|
19034
19308
|
const value = getNestedValue(document2, fieldPath);
|
|
19035
19309
|
if (value === void 0 || value === null) continue;
|
|
19036
|
-
if (fieldType
|
|
19310
|
+
if (!isWordIndexedBase(fieldType)) {
|
|
19037
19311
|
indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
|
|
19038
19312
|
continue;
|
|
19039
19313
|
}
|
|
@@ -20882,6 +21156,7 @@ async function decodeSnapshotBundle(data) {
|
|
|
20882
21156
|
if (!raw.schema || typeof raw.schema !== "object") {
|
|
20883
21157
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle missing schema");
|
|
20884
21158
|
}
|
|
21159
|
+
requireValidFieldTypes(raw.schema);
|
|
20885
21160
|
if (typeof raw.language !== "string") {
|
|
20886
21161
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle missing language");
|
|
20887
21162
|
}
|
|
@@ -21832,6 +22107,7 @@ function validateManifestCommon(raw) {
|
|
|
21832
22107
|
if (typeof raw.language !== "string") {
|
|
21833
22108
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest is missing language");
|
|
21834
22109
|
}
|
|
22110
|
+
requireValidFieldTypes(raw.schema);
|
|
21835
22111
|
return { schema: raw.schema, language: raw.language };
|
|
21836
22112
|
}
|
|
21837
22113
|
function normalizeCheckpoint2(raw) {
|
|
@@ -22252,6 +22528,9 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
22252
22528
|
if (metadata.vectorPromotion !== void 0) {
|
|
22253
22529
|
config.vectorPromotion = metadata.vectorPromotion;
|
|
22254
22530
|
}
|
|
22531
|
+
if (metadata.patternValueLimit !== void 0) {
|
|
22532
|
+
config.patternValueLimit = metadata.patternValueLimit;
|
|
22533
|
+
}
|
|
22255
22534
|
return config;
|
|
22256
22535
|
}
|
|
22257
22536
|
function isRecord4(value) {
|
|
@@ -24767,6 +25046,9 @@ function buildMetadata(indexName, hooks, documentCount) {
|
|
|
24767
25046
|
if (config.vectorPromotion !== void 0) {
|
|
24768
25047
|
metadata.vectorPromotion = config.vectorPromotion;
|
|
24769
25048
|
}
|
|
25049
|
+
if (config.patternValueLimit !== void 0) {
|
|
25050
|
+
metadata.patternValueLimit = config.patternValueLimit;
|
|
25051
|
+
}
|
|
24770
25052
|
if (config.indexUuid !== void 0) {
|
|
24771
25053
|
metadata.indexUuid = config.indexUuid;
|
|
24772
25054
|
}
|
|
@@ -24880,7 +25162,8 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
24880
25162
|
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
24881
25163
|
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
24882
25164
|
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
24883
|
-
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {}
|
|
25165
|
+
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
25166
|
+
...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {}
|
|
24884
25167
|
};
|
|
24885
25168
|
},
|
|
24886
25169
|
createIndexFromMetadata: wiring.createIndexFromMetadata,
|
|
@@ -24889,12 +25172,172 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
24889
25172
|
});
|
|
24890
25173
|
}
|
|
24891
25174
|
|
|
25175
|
+
// src/engine/constants.ts
|
|
25176
|
+
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
25177
|
+
var BATCH_CHUNK_SIZE = 1e3;
|
|
25178
|
+
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
25179
|
+
var DEFAULT_OFFSET = 0;
|
|
25180
|
+
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
25181
|
+
var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
|
|
25182
|
+
var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
|
|
25183
|
+
var MAX_INDEX_NAME_LENGTH = 255;
|
|
25184
|
+
var MAX_DOC_ID_LENGTH = 512;
|
|
25185
|
+
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
25186
|
+
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
25187
|
+
var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
|
|
25188
|
+
var HEAP_PRESSURE_REARM_FRACTION = 0.8;
|
|
25189
|
+
var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
|
|
25190
|
+
var FOREIGN_STATS_TTL_MS = 6e4;
|
|
25191
|
+
var DEFAULT_SUGGEST_LIMIT = 10;
|
|
25192
|
+
var MAX_SUGGEST_LIMIT = 100;
|
|
25193
|
+
|
|
25194
|
+
// src/engine/validation.ts
|
|
25195
|
+
function now() {
|
|
25196
|
+
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
25197
|
+
return performance.now();
|
|
25198
|
+
}
|
|
25199
|
+
return Date.now();
|
|
25200
|
+
}
|
|
25201
|
+
function validateIndexName(name) {
|
|
25202
|
+
const given = name;
|
|
25203
|
+
if (typeof given !== "string" || name.length === 0) {
|
|
25204
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index name must be a string of at least one character", {
|
|
25205
|
+
received: given === null ? "null" : typeof given
|
|
25206
|
+
});
|
|
25207
|
+
}
|
|
25208
|
+
if (name.length > MAX_INDEX_NAME_LENGTH) {
|
|
25209
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`, {
|
|
25210
|
+
indexName: name,
|
|
25211
|
+
length: name.length
|
|
25212
|
+
});
|
|
25213
|
+
}
|
|
25214
|
+
if (!INDEX_NAME_PATTERN.test(name)) {
|
|
25215
|
+
throw new NarsilError(
|
|
25216
|
+
ErrorCodes.CONFIG_INVALID,
|
|
25217
|
+
`Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
|
|
25218
|
+
{ indexName: name }
|
|
25219
|
+
);
|
|
25220
|
+
}
|
|
25221
|
+
if (name.includes("..")) {
|
|
25222
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name "${name}" must not contain ".."`, { indexName: name });
|
|
25223
|
+
}
|
|
25224
|
+
}
|
|
25225
|
+
function validateDocId(docId) {
|
|
25226
|
+
const given = docId;
|
|
25227
|
+
if (typeof given !== "string") {
|
|
25228
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must be a string", {
|
|
25229
|
+
received: given === null ? "null" : Array.isArray(given) ? "array" : typeof given
|
|
25230
|
+
});
|
|
25231
|
+
}
|
|
25232
|
+
if (docId.length === 0) {
|
|
25233
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
|
|
25234
|
+
}
|
|
25235
|
+
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
25236
|
+
throw new NarsilError(
|
|
25237
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
25238
|
+
`Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
|
|
25239
|
+
{
|
|
25240
|
+
docId,
|
|
25241
|
+
length: docId.length
|
|
25242
|
+
}
|
|
25243
|
+
);
|
|
25244
|
+
}
|
|
25245
|
+
if (docId.includes("\0")) {
|
|
25246
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
25247
|
+
}
|
|
25248
|
+
}
|
|
25249
|
+
function validatePartitionConfig(partitions) {
|
|
25250
|
+
const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
|
|
25251
|
+
if (maxDocsPerPartition !== void 0 && (!Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
|
|
25252
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxDocsPerPartition must be a positive integer", {
|
|
25253
|
+
maxDocsPerPartition
|
|
25254
|
+
});
|
|
25255
|
+
}
|
|
25256
|
+
if (maxPartitions !== void 0 && (!Number.isInteger(maxPartitions) || maxPartitions < 1)) {
|
|
25257
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxPartitions must be a positive integer", {
|
|
25258
|
+
maxPartitions
|
|
25259
|
+
});
|
|
25260
|
+
}
|
|
25261
|
+
if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
|
|
25262
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.watermark must be above 0 and at most 1", {
|
|
25263
|
+
watermark
|
|
25264
|
+
});
|
|
25265
|
+
}
|
|
25266
|
+
}
|
|
25267
|
+
function validatePatternValueLimit(limit) {
|
|
25268
|
+
if (limit === void 0) return;
|
|
25269
|
+
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < MIN_PATTERN_VALUE_LIMIT || limit > MAX_PATTERN_VALUE_LIMIT) {
|
|
25270
|
+
throw new NarsilError(
|
|
25271
|
+
ErrorCodes.CONFIG_INVALID,
|
|
25272
|
+
`patternValueLimit must be a whole number of code points from ${MIN_PATTERN_VALUE_LIMIT} to ${MAX_PATTERN_VALUE_LIMIT}`,
|
|
25273
|
+
{ patternValueLimit: typeof limit === "number" ? limit : String(limit) }
|
|
25274
|
+
);
|
|
25275
|
+
}
|
|
25276
|
+
}
|
|
25277
|
+
function requirePositiveSafeInteger(field, value) {
|
|
25278
|
+
if (value === void 0) return;
|
|
25279
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
25280
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${field} must be a positive safe integer`, { field, value });
|
|
25281
|
+
}
|
|
25282
|
+
}
|
|
25283
|
+
function validateBootstrapModule(bootstrapModule) {
|
|
25284
|
+
if (bootstrapModule === void 0) return;
|
|
25285
|
+
const absolute = bootstrapModule.startsWith("/") || ABSOLUTE_WINDOWS_PATH_PATTERN.test(bootstrapModule) || MODULE_URL_SCHEME_PATTERN.test(bootstrapModule);
|
|
25286
|
+
if (absolute) return;
|
|
25287
|
+
throw new NarsilError(
|
|
25288
|
+
ErrorCodes.CONFIG_INVALID,
|
|
25289
|
+
`workers.bootstrapModule is "${bootstrapModule}", which a worker would resolve against Narsil's own directory, so pass an absolute path or a URL such as new URL('./bootstrap.js', import.meta.url).href`,
|
|
25290
|
+
{ bootstrapModule }
|
|
25291
|
+
);
|
|
25292
|
+
}
|
|
25293
|
+
function validateWorkerConfig(workers, lifecycle) {
|
|
25294
|
+
if (workers === void 0) return;
|
|
25295
|
+
requirePositiveSafeInteger("workers.count", workers.count);
|
|
25296
|
+
requirePositiveSafeInteger("workers.promotionThreshold", workers.promotionThreshold);
|
|
25297
|
+
requirePositiveSafeInteger("workers.idleTimeoutMs", workers.idleTimeoutMs);
|
|
25298
|
+
if (workers.mainCopyQueries !== void 0 && workers.mainCopyQueries !== "lone" && workers.mainCopyQueries !== "none") {
|
|
25299
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "workers.mainCopyQueries must be 'lone' or 'none'", {
|
|
25300
|
+
mainCopyQueries: workers.mainCopyQueries
|
|
25301
|
+
});
|
|
25302
|
+
}
|
|
25303
|
+
validateBootstrapModule(workers.bootstrapModule);
|
|
25304
|
+
const closeAfter = lifecycle?.idleTimeoutMs;
|
|
25305
|
+
if (workers.idleTimeoutMs !== void 0 && closeAfter !== void 0 && workers.idleTimeoutMs > closeAfter) {
|
|
25306
|
+
throw new NarsilError(
|
|
25307
|
+
ErrorCodes.CONFIG_INVALID,
|
|
25308
|
+
"workers.idleTimeoutMs must be at most lifecycle.idleTimeoutMs, so that an index drops its copies before it closes",
|
|
25309
|
+
{ workersIdleTimeoutMs: workers.idleTimeoutMs, lifecycleIdleTimeoutMs: closeAfter }
|
|
25310
|
+
);
|
|
25311
|
+
}
|
|
25312
|
+
}
|
|
25313
|
+
function validateBM25Params(bm25) {
|
|
25314
|
+
if (bm25 === void 0) return;
|
|
25315
|
+
const { k1, b } = bm25;
|
|
25316
|
+
if (k1 !== void 0 && (!Number.isFinite(k1) || k1 < 0)) {
|
|
25317
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.k1 must be a finite number at or above 0", { k1 });
|
|
25318
|
+
}
|
|
25319
|
+
if (b !== void 0 && (!Number.isFinite(b) || b < 0 || b > 1)) {
|
|
25320
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.b must be between 0 and 1", { b });
|
|
25321
|
+
}
|
|
25322
|
+
}
|
|
25323
|
+
function clampLimit(limit) {
|
|
25324
|
+
return clampRowCount(limit, DEFAULT_LIMIT);
|
|
25325
|
+
}
|
|
25326
|
+
function clampOffset(offset) {
|
|
25327
|
+
return clampRowCount(offset, DEFAULT_OFFSET);
|
|
25328
|
+
}
|
|
25329
|
+
|
|
24892
25330
|
// src/engine/vector-fields.ts
|
|
24893
25331
|
function getVectorFieldPaths(schema) {
|
|
24894
25332
|
return new Set(extractVectorFieldsFromSchema(schema).keys());
|
|
24895
25333
|
}
|
|
24896
25334
|
|
|
24897
25335
|
// src/engine/index-from-metadata.ts
|
|
25336
|
+
var RECOVERY_CONFIG_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
25337
|
+
ErrorCodes.CONFIG_INVALID,
|
|
25338
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
25339
|
+
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION
|
|
25340
|
+
]);
|
|
24898
25341
|
function resolveEmbeddingAdapter(deps, metadata, schema) {
|
|
24899
25342
|
const name = metadata.embedding?.adapter ?? null;
|
|
24900
25343
|
if (!metadata.embedding) return { adapter: null, name };
|
|
@@ -24913,6 +25356,15 @@ function resolveEmbeddingAdapter(deps, metadata, schema) {
|
|
|
24913
25356
|
}
|
|
24914
25357
|
return { adapter: candidate, name };
|
|
24915
25358
|
}
|
|
25359
|
+
function recoveryFailure(metadata, err) {
|
|
25360
|
+
if (!(err instanceof NarsilError) || !RECOVERY_CONFIG_ERROR_CODES.has(err.code)) return err;
|
|
25361
|
+
return new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
|
|
25362
|
+
...err.details,
|
|
25363
|
+
indexName: metadata.indexName,
|
|
25364
|
+
tokenizer: metadata.tokenizer,
|
|
25365
|
+
stopWords: metadata.stopWords
|
|
25366
|
+
});
|
|
25367
|
+
}
|
|
24916
25368
|
async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
24917
25369
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
24918
25370
|
if (existing !== void 0) {
|
|
@@ -24923,19 +25375,18 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
24923
25375
|
}
|
|
24924
25376
|
const indexConfig = reconstructSchemaFromMetadata(metadata);
|
|
24925
25377
|
const language = getLanguage(indexConfig.language ?? "english");
|
|
25378
|
+
try {
|
|
25379
|
+
indexConfig.schema = validateSchema(indexConfig.schema);
|
|
25380
|
+
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
25381
|
+
} catch (err) {
|
|
25382
|
+
throw recoveryFailure(metadata, err);
|
|
25383
|
+
}
|
|
24926
25384
|
const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
|
|
24927
25385
|
try {
|
|
24928
25386
|
validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
|
|
24929
25387
|
if (loadData) deps.executor.createIndex(metadata.indexName, indexConfig, language);
|
|
24930
25388
|
} catch (err) {
|
|
24931
|
-
|
|
24932
|
-
throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
|
|
24933
|
-
indexName: metadata.indexName,
|
|
24934
|
-
tokenizer: metadata.tokenizer,
|
|
24935
|
-
stopWords: metadata.stopWords
|
|
24936
|
-
});
|
|
24937
|
-
}
|
|
24938
|
-
throw err;
|
|
25389
|
+
throw recoveryFailure(metadata, err);
|
|
24939
25390
|
}
|
|
24940
25391
|
deps.indexRegistry.set(metadata.indexName, {
|
|
24941
25392
|
config: indexConfig,
|
|
@@ -24960,25 +25411,6 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
24960
25411
|
}
|
|
24961
25412
|
}
|
|
24962
25413
|
|
|
24963
|
-
// src/engine/constants.ts
|
|
24964
|
-
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
24965
|
-
var BATCH_CHUNK_SIZE = 1e3;
|
|
24966
|
-
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
24967
|
-
var DEFAULT_OFFSET = 0;
|
|
24968
|
-
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
24969
|
-
var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
|
|
24970
|
-
var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
|
|
24971
|
-
var MAX_INDEX_NAME_LENGTH = 255;
|
|
24972
|
-
var MAX_DOC_ID_LENGTH = 512;
|
|
24973
|
-
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
24974
|
-
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
24975
|
-
var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
|
|
24976
|
-
var HEAP_PRESSURE_REARM_FRACTION = 0.8;
|
|
24977
|
-
var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
|
|
24978
|
-
var FOREIGN_STATS_TTL_MS = 6e4;
|
|
24979
|
-
var DEFAULT_SUGGEST_LIMIT = 10;
|
|
24980
|
-
var MAX_SUGGEST_LIMIT = 100;
|
|
24981
|
-
|
|
24982
25414
|
// src/engine/index-state/constants.ts
|
|
24983
25415
|
var DEFAULT_MAX_REOPEN_WAITERS = 64;
|
|
24984
25416
|
var MAX_REOPEN_FAILURES = 5;
|
|
@@ -25561,21 +25993,10 @@ function createHeapPressureNotifier(deps) {
|
|
|
25561
25993
|
}
|
|
25562
25994
|
|
|
25563
25995
|
// src/engine/watermark.ts
|
|
25564
|
-
function createWatermarkNotifier(deps) {
|
|
25996
|
+
function createWatermarkNotifier(deps, scope = "index") {
|
|
25565
25997
|
const latchedCapacity = /* @__PURE__ */ new Map();
|
|
25566
|
-
|
|
25567
|
-
|
|
25568
|
-
const maxDocs = partitionConfig?.maxDocsPerPartition;
|
|
25569
|
-
const watermark = partitionConfig?.watermark;
|
|
25570
|
-
if (maxDocs === void 0 || watermark === void 0) {
|
|
25571
|
-
latchedCapacity.delete(indexName);
|
|
25572
|
-
return;
|
|
25573
|
-
}
|
|
25574
|
-
const manager = deps.getManager(indexName);
|
|
25575
|
-
if (!manager) {
|
|
25576
|
-
latchedCapacity.delete(indexName);
|
|
25577
|
-
return;
|
|
25578
|
-
}
|
|
25998
|
+
const latchedPartitions = /* @__PURE__ */ new Map();
|
|
25999
|
+
function checkIndex(indexName, manager, maxDocs, watermark) {
|
|
25579
26000
|
const partitionCount = manager.partitionCount;
|
|
25580
26001
|
const capacity = maxDocs * partitionCount;
|
|
25581
26002
|
const documentCount = manager.countDocuments();
|
|
@@ -25588,19 +26009,63 @@ function createWatermarkNotifier(deps) {
|
|
|
25588
26009
|
latchedCapacity.set(indexName, capacity);
|
|
25589
26010
|
deps.emit({ indexName, documentCount, capacity, partitionCount });
|
|
25590
26011
|
}
|
|
26012
|
+
function checkPartitions(indexName, manager, maxDocs, watermark, insertedDocIds) {
|
|
26013
|
+
const threshold = watermark * maxDocs;
|
|
26014
|
+
const latched = latchedPartitions.get(indexName) ?? /* @__PURE__ */ new Set();
|
|
26015
|
+
latchedPartitions.set(indexName, latched);
|
|
26016
|
+
const checked = /* @__PURE__ */ new Set();
|
|
26017
|
+
for (const docId of insertedDocIds) {
|
|
26018
|
+
const partitionId = manager.partitionIdOf(docId) ?? manager.routePartition(docId);
|
|
26019
|
+
if (checked.has(partitionId)) continue;
|
|
26020
|
+
checked.add(partitionId);
|
|
26021
|
+
const documentCount = manager.getPartition(partitionId).count();
|
|
26022
|
+
if (documentCount < threshold) {
|
|
26023
|
+
latched.delete(partitionId);
|
|
26024
|
+
continue;
|
|
26025
|
+
}
|
|
26026
|
+
if (latched.has(partitionId)) continue;
|
|
26027
|
+
latched.add(partitionId);
|
|
26028
|
+
deps.emit({ indexName, documentCount, capacity: maxDocs, partitionCount: manager.partitionCount, partitionId });
|
|
26029
|
+
}
|
|
26030
|
+
}
|
|
26031
|
+
function check(indexName, inserted) {
|
|
26032
|
+
const partitionConfig = deps.getPartitionConfig(indexName);
|
|
26033
|
+
const maxDocs = partitionConfig?.maxDocsPerPartition;
|
|
26034
|
+
const watermark = partitionConfig?.watermark;
|
|
26035
|
+
if (maxDocs === void 0 || watermark === void 0) {
|
|
26036
|
+
forget(indexName);
|
|
26037
|
+
return;
|
|
26038
|
+
}
|
|
26039
|
+
const manager = deps.getManager(indexName);
|
|
26040
|
+
if (manager === void 0) {
|
|
26041
|
+
forget(indexName);
|
|
26042
|
+
return;
|
|
26043
|
+
}
|
|
26044
|
+
if (scope === "index") {
|
|
26045
|
+
checkIndex(indexName, manager, maxDocs, watermark);
|
|
26046
|
+
return;
|
|
26047
|
+
}
|
|
26048
|
+
if (inserted !== void 0) {
|
|
26049
|
+
checkPartitions(indexName, manager, maxDocs, watermark, typeof inserted === "string" ? [inserted] : inserted);
|
|
26050
|
+
}
|
|
26051
|
+
}
|
|
25591
26052
|
function forget(indexName) {
|
|
25592
26053
|
latchedCapacity.delete(indexName);
|
|
26054
|
+
latchedPartitions.delete(indexName);
|
|
25593
26055
|
}
|
|
25594
26056
|
return { check, forget };
|
|
25595
26057
|
}
|
|
25596
26058
|
|
|
25597
26059
|
// src/engine/notifiers.ts
|
|
25598
26060
|
function wireWatermarkNotifier(wiring) {
|
|
25599
|
-
return createWatermarkNotifier(
|
|
25600
|
-
|
|
25601
|
-
|
|
25602
|
-
|
|
25603
|
-
|
|
26061
|
+
return createWatermarkNotifier(
|
|
26062
|
+
{
|
|
26063
|
+
getManager: wiring.getManager,
|
|
26064
|
+
getPartitionConfig: (indexName) => wiring.indexRegistry.get(indexName)?.config.partitions,
|
|
26065
|
+
emit: (payload) => emitEngineEvent(wiring.eventHandlers, "partitionWatermark", payload)
|
|
26066
|
+
},
|
|
26067
|
+
wiring.capacityScope
|
|
26068
|
+
);
|
|
25604
26069
|
}
|
|
25605
26070
|
function warnUnheardHeapPressure(payload) {
|
|
25606
26071
|
console.warn(
|
|
@@ -27575,134 +28040,32 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
27575
28040
|
};
|
|
27576
28041
|
}
|
|
27577
28042
|
|
|
27578
|
-
// src/engine/
|
|
27579
|
-
function
|
|
27580
|
-
|
|
27581
|
-
|
|
27582
|
-
}
|
|
27583
|
-
return Date.now();
|
|
27584
|
-
}
|
|
27585
|
-
function validateIndexName(name) {
|
|
27586
|
-
const given = name;
|
|
27587
|
-
if (typeof given !== "string" || name.length === 0) {
|
|
27588
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index name must be a string of at least one character", {
|
|
27589
|
-
received: given === null ? "null" : typeof given
|
|
27590
|
-
});
|
|
27591
|
-
}
|
|
27592
|
-
if (name.length > MAX_INDEX_NAME_LENGTH) {
|
|
27593
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`, {
|
|
27594
|
-
indexName: name,
|
|
27595
|
-
length: name.length
|
|
27596
|
-
});
|
|
27597
|
-
}
|
|
27598
|
-
if (!INDEX_NAME_PATTERN.test(name)) {
|
|
27599
|
-
throw new NarsilError(
|
|
27600
|
-
ErrorCodes.CONFIG_INVALID,
|
|
27601
|
-
`Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
|
|
27602
|
-
{ indexName: name }
|
|
27603
|
-
);
|
|
27604
|
-
}
|
|
27605
|
-
if (name.includes("..")) {
|
|
27606
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name "${name}" must not contain ".."`, { indexName: name });
|
|
27607
|
-
}
|
|
27608
|
-
}
|
|
27609
|
-
function validateDocId(docId) {
|
|
27610
|
-
const given = docId;
|
|
27611
|
-
if (typeof given !== "string") {
|
|
27612
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must be a string", {
|
|
27613
|
-
received: given === null ? "null" : Array.isArray(given) ? "array" : typeof given
|
|
27614
|
-
});
|
|
27615
|
-
}
|
|
27616
|
-
if (docId.length === 0) {
|
|
27617
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
|
|
27618
|
-
}
|
|
27619
|
-
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
27620
|
-
throw new NarsilError(
|
|
27621
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
27622
|
-
`Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
|
|
27623
|
-
{
|
|
27624
|
-
docId,
|
|
27625
|
-
length: docId.length
|
|
27626
|
-
}
|
|
27627
|
-
);
|
|
27628
|
-
}
|
|
27629
|
-
if (docId.includes("\0")) {
|
|
27630
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
27631
|
-
}
|
|
27632
|
-
}
|
|
27633
|
-
function validatePartitionConfig(partitions) {
|
|
27634
|
-
const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
|
|
27635
|
-
if (maxDocsPerPartition !== void 0 && (!Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
|
|
27636
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxDocsPerPartition must be a positive integer", {
|
|
27637
|
-
maxDocsPerPartition
|
|
27638
|
-
});
|
|
27639
|
-
}
|
|
27640
|
-
if (maxPartitions !== void 0 && (!Number.isInteger(maxPartitions) || maxPartitions < 1)) {
|
|
27641
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxPartitions must be a positive integer", {
|
|
27642
|
-
maxPartitions
|
|
27643
|
-
});
|
|
27644
|
-
}
|
|
27645
|
-
if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
|
|
27646
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.watermark must be above 0 and at most 1", {
|
|
27647
|
-
watermark
|
|
27648
|
-
});
|
|
27649
|
-
}
|
|
27650
|
-
}
|
|
27651
|
-
function requirePositiveSafeInteger(field, value) {
|
|
27652
|
-
if (value === void 0) return;
|
|
27653
|
-
if (!Number.isSafeInteger(value) || value < 1) {
|
|
27654
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${field} must be a positive safe integer`, { field, value });
|
|
27655
|
-
}
|
|
27656
|
-
}
|
|
27657
|
-
function validateBootstrapModule(bootstrapModule) {
|
|
27658
|
-
if (bootstrapModule === void 0) return;
|
|
27659
|
-
const absolute = bootstrapModule.startsWith("/") || ABSOLUTE_WINDOWS_PATH_PATTERN.test(bootstrapModule) || MODULE_URL_SCHEME_PATTERN.test(bootstrapModule);
|
|
27660
|
-
if (absolute) return;
|
|
27661
|
-
throw new NarsilError(
|
|
27662
|
-
ErrorCodes.CONFIG_INVALID,
|
|
27663
|
-
`workers.bootstrapModule is "${bootstrapModule}", which a worker would resolve against Narsil's own directory, so pass an absolute path or a URL such as new URL('./bootstrap.js', import.meta.url).href`,
|
|
27664
|
-
{ bootstrapModule }
|
|
27665
|
-
);
|
|
28043
|
+
// src/engine/mutations/write-visibility.ts
|
|
28044
|
+
async function awaitWriteVisibility(ctx, indexName) {
|
|
28045
|
+
await ctx.awaitRebalanceReplay(indexName);
|
|
28046
|
+
await ctx.orchestrator.awaitWrites(indexName);
|
|
27666
28047
|
}
|
|
27667
|
-
|
|
27668
|
-
|
|
27669
|
-
|
|
27670
|
-
|
|
27671
|
-
|
|
27672
|
-
if (
|
|
27673
|
-
|
|
27674
|
-
|
|
27675
|
-
|
|
27676
|
-
|
|
27677
|
-
validateBootstrapModule(workers.bootstrapModule);
|
|
27678
|
-
const closeAfter = lifecycle?.idleTimeoutMs;
|
|
27679
|
-
if (workers.idleTimeoutMs !== void 0 && closeAfter !== void 0 && workers.idleTimeoutMs > closeAfter) {
|
|
27680
|
-
throw new NarsilError(
|
|
27681
|
-
ErrorCodes.CONFIG_INVALID,
|
|
27682
|
-
"workers.idleTimeoutMs must be at most lifecycle.idleTimeoutMs, so that an index drops its copies before it closes",
|
|
27683
|
-
{ workersIdleTimeoutMs: workers.idleTimeoutMs, lifecycleIdleTimeoutMs: closeAfter }
|
|
27684
|
-
);
|
|
27685
|
-
}
|
|
28048
|
+
|
|
28049
|
+
// src/engine/write-events.ts
|
|
28050
|
+
var WRITE_EVENT = "write";
|
|
28051
|
+
function emitWriteEvent(source, indexName) {
|
|
28052
|
+
const handlers = source.eventHandlers.get(WRITE_EVENT);
|
|
28053
|
+
if (handlers === void 0 || handlers.size === 0) return;
|
|
28054
|
+
const emit = () => {
|
|
28055
|
+
emitEngineEvent(source.eventHandlers, WRITE_EVENT, { indexName });
|
|
28056
|
+
};
|
|
28057
|
+
awaitWriteVisibility(source.mutationCtx, indexName).then(emit, emit);
|
|
27686
28058
|
}
|
|
27687
|
-
function
|
|
27688
|
-
|
|
27689
|
-
|
|
27690
|
-
|
|
27691
|
-
|
|
27692
|
-
}
|
|
27693
|
-
if (b !== void 0 && (!Number.isFinite(b) || b < 0 || b > 1)) {
|
|
27694
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.b must be between 0 and 1", { b });
|
|
28059
|
+
async function emitWriteEventAfter(source, indexName, write) {
|
|
28060
|
+
try {
|
|
28061
|
+
return await write;
|
|
28062
|
+
} finally {
|
|
28063
|
+
emitWriteEvent(source, indexName);
|
|
27695
28064
|
}
|
|
27696
28065
|
}
|
|
27697
|
-
function clampLimit(limit) {
|
|
27698
|
-
return clampRowCount(limit, DEFAULT_LIMIT);
|
|
27699
|
-
}
|
|
27700
|
-
function clampOffset(offset) {
|
|
27701
|
-
return clampRowCount(offset, DEFAULT_OFFSET);
|
|
27702
|
-
}
|
|
27703
28066
|
|
|
27704
28067
|
// src/engine/core.ts
|
|
27705
|
-
function createEngineCore(config, hooks) {
|
|
28068
|
+
function createEngineCore(config, hooks, capacityScope = "index") {
|
|
27706
28069
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
27707
28070
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
27708
28071
|
const vectorCopyPolicy = {
|
|
@@ -27840,7 +28203,11 @@ function createEngineCore(config, hooks) {
|
|
|
27840
28203
|
}
|
|
27841
28204
|
return names;
|
|
27842
28205
|
},
|
|
27843
|
-
reloadIndex: (indexName) =>
|
|
28206
|
+
reloadIndex: (indexName) => emitWriteEventAfter(
|
|
28207
|
+
{ eventHandlers, mutationCtx },
|
|
28208
|
+
indexName,
|
|
28209
|
+
durability?.manager.reloadIndex?.(indexName) ?? Promise.resolve()
|
|
28210
|
+
),
|
|
27844
28211
|
onError(error) {
|
|
27845
28212
|
if (emitEngineEvent(eventHandlers, "invalidationError", { error }) === 0) {
|
|
27846
28213
|
console.warn("Invalidation error:", error);
|
|
@@ -27850,7 +28217,8 @@ function createEngineCore(config, hooks) {
|
|
|
27850
28217
|
const notifierWiring = {
|
|
27851
28218
|
eventHandlers,
|
|
27852
28219
|
indexRegistry,
|
|
27853
|
-
getManager: (indexName) => executor.getManager(indexName)
|
|
28220
|
+
getManager: (indexName) => executor.getManager(indexName),
|
|
28221
|
+
capacityScope
|
|
27854
28222
|
};
|
|
27855
28223
|
const watermarkNotifier = wireWatermarkNotifier(notifierWiring);
|
|
27856
28224
|
const heapPressureNotifier = wireHeapPressureNotifier(notifierWiring);
|
|
@@ -27894,6 +28262,7 @@ function createEngineCore(config, hooks) {
|
|
|
27894
28262
|
pendingRebalanceWrites: (indexName) => waqMap.get(indexName)?.size ?? 0,
|
|
27895
28263
|
rebalanceTargetPartitionCount: (indexName) => rebalanceTargets.get(indexName),
|
|
27896
28264
|
bufferedDocState: (indexName, docId) => waqMap.get(indexName)?.bufferedDocState(docId),
|
|
28265
|
+
capacityScope,
|
|
27897
28266
|
checkWatermark: watermarkNotifier.check,
|
|
27898
28267
|
checkHeapPressure: heapPressureNotifier.check
|
|
27899
28268
|
};
|
|
@@ -28026,13 +28395,16 @@ async function runLifecycleHook(core, hookName, context) {
|
|
|
28026
28395
|
async function announceIndexCreated(core, name, config) {
|
|
28027
28396
|
await runLifecycleHook(core, "onIndexCreate", { indexName: name, config });
|
|
28028
28397
|
}
|
|
28029
|
-
async function createEngineIndex(core, config, name,
|
|
28398
|
+
async function createEngineIndex(core, config, name, requestedConfig, creation = {}) {
|
|
28399
|
+
const { indexUuid, adapterBinding = "required" } = creation;
|
|
28030
28400
|
core.guardShutdown();
|
|
28031
28401
|
validateIndexName(name);
|
|
28032
28402
|
if (core.indexRegistry.has(name)) {
|
|
28033
28403
|
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
|
|
28034
28404
|
}
|
|
28035
|
-
validateSchema(
|
|
28405
|
+
const schema = validateSchema(requestedConfig.schema);
|
|
28406
|
+
const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
28407
|
+
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
28036
28408
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
28037
28409
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
28038
28410
|
validateBM25Params(indexConfig.bm25);
|
|
@@ -28058,24 +28430,31 @@ async function createEngineIndex(core, config, name, indexConfig, indexUuid) {
|
|
|
28058
28430
|
let resolvedEmbeddingAdapter = null;
|
|
28059
28431
|
let embeddingAdapterName = null;
|
|
28060
28432
|
if (indexConfig.embedding) {
|
|
28061
|
-
|
|
28433
|
+
const configuredAdapter = indexConfig.embedding.adapter;
|
|
28434
|
+
const fields = indexConfig.embedding.fields;
|
|
28062
28435
|
if (typeof configuredAdapter === "string") {
|
|
28063
28436
|
embeddingAdapterName = configuredAdapter;
|
|
28064
28437
|
const registered = core.embeddingAdapters.get(configuredAdapter);
|
|
28065
|
-
if (
|
|
28438
|
+
if (registered !== void 0) {
|
|
28439
|
+
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
28440
|
+
{ fields, adapter: registered },
|
|
28441
|
+
indexConfig.schema,
|
|
28442
|
+
config?.embedding
|
|
28443
|
+
);
|
|
28444
|
+
} else if (adapterBinding === "required") {
|
|
28066
28445
|
throw new NarsilError(
|
|
28067
28446
|
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
28068
28447
|
`Embedding adapter "${configuredAdapter}" is not registered on this engine`,
|
|
28069
28448
|
{ adapter: configuredAdapter, available: [...core.embeddingAdapters.keys()] }
|
|
28070
28449
|
);
|
|
28071
28450
|
}
|
|
28072
|
-
|
|
28451
|
+
} else if (adapterBinding === "required" || configuredAdapter !== void 0 || config?.embedding !== void 0) {
|
|
28452
|
+
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
28453
|
+
{ fields, adapter: configuredAdapter },
|
|
28454
|
+
indexConfig.schema,
|
|
28455
|
+
config?.embedding
|
|
28456
|
+
);
|
|
28073
28457
|
}
|
|
28074
|
-
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
28075
|
-
{ fields: indexConfig.embedding.fields, adapter: configuredAdapter },
|
|
28076
|
-
indexConfig.schema,
|
|
28077
|
-
config?.embedding
|
|
28078
|
-
);
|
|
28079
28458
|
}
|
|
28080
28459
|
if (indexConfig.required && indexConfig.required.length > 0) {
|
|
28081
28460
|
validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
|
|
@@ -28088,7 +28467,7 @@ async function createEngineIndex(core, config, name, indexConfig, indexUuid) {
|
|
|
28088
28467
|
embeddingAdapter: resolvedEmbeddingAdapter,
|
|
28089
28468
|
embeddingAdapterName,
|
|
28090
28469
|
vectorFieldPaths: getVectorFieldPaths(indexConfig.schema),
|
|
28091
|
-
indexUuid: null,
|
|
28470
|
+
indexUuid: indexUuid ?? null,
|
|
28092
28471
|
heldPartitions: null,
|
|
28093
28472
|
documentCount: 0,
|
|
28094
28473
|
partitionCount: indexConfig.partitions?.maxPartitions ?? 1
|
|
@@ -28418,15 +28797,43 @@ function providedDocId(document2) {
|
|
|
28418
28797
|
function asBatchInsertError(err) {
|
|
28419
28798
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
28420
28799
|
}
|
|
28421
|
-
function
|
|
28422
|
-
|
|
28423
|
-
|
|
28800
|
+
function createPendingAdmissions() {
|
|
28801
|
+
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
28802
|
+
}
|
|
28803
|
+
function capsEachPartition(ctx, manager) {
|
|
28804
|
+
return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
|
|
28805
|
+
}
|
|
28806
|
+
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
28807
|
+
pending.total += 1;
|
|
28808
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
28809
|
+
const partitionId = manager.routePartition(docId);
|
|
28810
|
+
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
28811
|
+
}
|
|
28812
|
+
function admitInsert(ctx, indexName, manager, docId, pending) {
|
|
28813
|
+
const rebalancing = ctx.isRebalancing(indexName);
|
|
28814
|
+
if (rebalancing) {
|
|
28815
|
+
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
28816
|
+
const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
|
|
28817
|
+
if (exists) {
|
|
28818
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
28819
|
+
}
|
|
28820
|
+
}
|
|
28821
|
+
assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing);
|
|
28822
|
+
}
|
|
28823
|
+
function admitBufferedUpsert(ctx, indexName, manager, docId) {
|
|
28824
|
+
assertRoomFor(ctx, indexName, manager, docId, void 0, true);
|
|
28825
|
+
}
|
|
28826
|
+
function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
|
|
28827
|
+
if (ctx.capacityScope === "partition") {
|
|
28828
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
28829
|
+
const partitionId = manager.routePartition(docId);
|
|
28830
|
+
manager.assertPartitionCapacity(partitionId, pending?.byPartition.get(partitionId) ?? 0);
|
|
28424
28831
|
return;
|
|
28425
28832
|
}
|
|
28426
|
-
const
|
|
28427
|
-
|
|
28428
|
-
|
|
28429
|
-
|
|
28833
|
+
const pendingAdmitted = pending?.total ?? 0;
|
|
28834
|
+
if (!rebalancing) {
|
|
28835
|
+
manager.assertCapacity(pendingAdmitted);
|
|
28836
|
+
return;
|
|
28430
28837
|
}
|
|
28431
28838
|
manager.assertCapacity(
|
|
28432
28839
|
ctx.pendingRebalanceWrites(indexName) + pendingAdmitted,
|
|
@@ -28493,12 +28900,6 @@ async function embedChunkDocuments(entry, documents, chunkStart, chunkEnd, signa
|
|
|
28493
28900
|
}
|
|
28494
28901
|
}
|
|
28495
28902
|
|
|
28496
|
-
// src/engine/mutations/write-visibility.ts
|
|
28497
|
-
async function awaitWriteVisibility(ctx, indexName) {
|
|
28498
|
-
await ctx.awaitRebalanceReplay(indexName);
|
|
28499
|
-
await ctx.orchestrator.awaitWrites(indexName);
|
|
28500
|
-
}
|
|
28501
|
-
|
|
28502
28903
|
// src/engine/mutations/insert.ts
|
|
28503
28904
|
async function insertDocument(ctx, indexName, document2, docId, options) {
|
|
28504
28905
|
ctx.guardShutdown();
|
|
@@ -28591,7 +28992,7 @@ async function insertDocument(ctx, indexName, document2, docId, options) {
|
|
|
28591
28992
|
await applyInsert();
|
|
28592
28993
|
}
|
|
28593
28994
|
if (buffered) {
|
|
28594
|
-
ctx.checkWatermark(indexName);
|
|
28995
|
+
ctx.checkWatermark(indexName, resolvedDocId);
|
|
28595
28996
|
ctx.checkHeapPressure(indexName);
|
|
28596
28997
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
28597
28998
|
return resolvedDocId;
|
|
@@ -28615,7 +29016,7 @@ async function insertDocument(ctx, indexName, document2, docId, options) {
|
|
|
28615
29016
|
vecIndex.scheduleBuild();
|
|
28616
29017
|
}
|
|
28617
29018
|
}
|
|
28618
|
-
ctx.checkWatermark(indexName);
|
|
29019
|
+
ctx.checkWatermark(indexName, resolvedDocId);
|
|
28619
29020
|
ctx.checkHeapPressure(indexName);
|
|
28620
29021
|
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
28621
29022
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
@@ -28726,6 +29127,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
|
|
|
28726
29127
|
const required = entry.config.required;
|
|
28727
29128
|
const admitted = [];
|
|
28728
29129
|
const admittedIds = /* @__PURE__ */ new Set();
|
|
29130
|
+
const pending = createPendingAdmissions();
|
|
28729
29131
|
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
28730
29132
|
if (ctx.abortController.signal.aborted) break;
|
|
28731
29133
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
@@ -28758,15 +29160,16 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
|
|
|
28758
29160
|
if (extractedVectors.size > 0) {
|
|
28759
29161
|
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
28760
29162
|
}
|
|
28761
|
-
admitInsert(ctx, indexName, manager, docId,
|
|
29163
|
+
admitInsert(ctx, indexName, manager, docId, pending);
|
|
28762
29164
|
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
28763
29165
|
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
28764
29166
|
}
|
|
28765
|
-
validateDocument(partitionDoc, entry.config.schema);
|
|
29167
|
+
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
28766
29168
|
if (strict) {
|
|
28767
29169
|
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
28768
29170
|
}
|
|
28769
29171
|
admittedIds.add(docId);
|
|
29172
|
+
recordPendingAdmission(ctx, pending, manager, docId);
|
|
28770
29173
|
admitted.push({ docId, document: documents[i], partitionDoc, extractedVectors });
|
|
28771
29174
|
} catch (err) {
|
|
28772
29175
|
failed.push({ docId, error: asBatchInsertError(err) });
|
|
@@ -29165,7 +29568,7 @@ async function insertBatchViaSegments(ctx, indexName, documents, options) {
|
|
|
29165
29568
|
for (const fieldPath of outcome.touchedVectorFields) {
|
|
29166
29569
|
vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
29167
29570
|
}
|
|
29168
|
-
ctx.checkWatermark(indexName);
|
|
29571
|
+
ctx.checkWatermark(indexName, outcome.succeeded);
|
|
29169
29572
|
ctx.checkHeapPressure(indexName);
|
|
29170
29573
|
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
29171
29574
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
@@ -29298,7 +29701,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
29298
29701
|
vecIndex.scheduleBuild();
|
|
29299
29702
|
}
|
|
29300
29703
|
}
|
|
29301
|
-
ctx.checkWatermark(indexName);
|
|
29704
|
+
ctx.checkWatermark(indexName, succeeded);
|
|
29302
29705
|
ctx.checkHeapPressure(indexName);
|
|
29303
29706
|
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
29304
29707
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
@@ -29531,7 +29934,7 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
29531
29934
|
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
29532
29935
|
const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
|
|
29533
29936
|
if (!exists) {
|
|
29534
|
-
|
|
29937
|
+
admitBufferedUpsert(ctx, indexName, manager, docId);
|
|
29535
29938
|
}
|
|
29536
29939
|
}
|
|
29537
29940
|
if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document: document2, indexName })) {
|
|
@@ -29891,40 +30294,40 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
29891
30294
|
}
|
|
29892
30295
|
|
|
29893
30296
|
// src/engine/snapshot/restore-config.ts
|
|
29894
|
-
function
|
|
30297
|
+
function rejectSnapshot(message) {
|
|
29895
30298
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
|
|
29896
30299
|
}
|
|
29897
30300
|
function isPlainObject3(value) {
|
|
29898
30301
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
29899
30302
|
}
|
|
29900
|
-
function restoredBm25(raw) {
|
|
30303
|
+
function restoredBm25(raw, reject2) {
|
|
29901
30304
|
if (raw === void 0) {
|
|
29902
30305
|
return void 0;
|
|
29903
30306
|
}
|
|
29904
30307
|
if (!isPlainObject3(raw)) {
|
|
29905
|
-
|
|
30308
|
+
reject2("bm25 must be an object");
|
|
29906
30309
|
}
|
|
29907
30310
|
const { k1, b } = raw;
|
|
29908
30311
|
if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
|
|
29909
|
-
|
|
30312
|
+
reject2("bm25 k1 must be a finite number");
|
|
29910
30313
|
}
|
|
29911
30314
|
if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
|
|
29912
|
-
|
|
30315
|
+
reject2("bm25 b must be a finite number");
|
|
29913
30316
|
}
|
|
29914
30317
|
return {
|
|
29915
30318
|
...k1 !== void 0 ? { k1 } : {},
|
|
29916
30319
|
...b !== void 0 ? { b } : {}
|
|
29917
30320
|
};
|
|
29918
30321
|
}
|
|
29919
|
-
function restoredStopWords(name, list) {
|
|
30322
|
+
function restoredStopWords(name, list, reject2) {
|
|
29920
30323
|
if (name !== void 0 && typeof name !== "string") {
|
|
29921
|
-
|
|
30324
|
+
reject2("stopWords must be a name");
|
|
29922
30325
|
}
|
|
29923
30326
|
if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
|
|
29924
|
-
|
|
30327
|
+
reject2("stopWordList must be a list of words");
|
|
29925
30328
|
}
|
|
29926
30329
|
if (typeof name === "string" && Array.isArray(list)) {
|
|
29927
|
-
|
|
30330
|
+
reject2("stopWords and stopWordList cannot both be present");
|
|
29928
30331
|
}
|
|
29929
30332
|
if (typeof name === "string") {
|
|
29930
30333
|
return name;
|
|
@@ -29934,22 +30337,22 @@ function restoredStopWords(name, list) {
|
|
|
29934
30337
|
}
|
|
29935
30338
|
return void 0;
|
|
29936
30339
|
}
|
|
29937
|
-
function restoredPartitionConfig(raw) {
|
|
30340
|
+
function restoredPartitionConfig(raw, reject2 = rejectSnapshot) {
|
|
29938
30341
|
if (raw === void 0) {
|
|
29939
30342
|
return void 0;
|
|
29940
30343
|
}
|
|
29941
30344
|
if (!isPlainObject3(raw)) {
|
|
29942
|
-
|
|
30345
|
+
reject2("partitionConfig must be an object");
|
|
29943
30346
|
}
|
|
29944
30347
|
const { maxDocsPerPartition, maxPartitions, watermark } = raw;
|
|
29945
30348
|
if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
|
|
29946
|
-
|
|
30349
|
+
reject2("partitionConfig maxDocsPerPartition must be a positive integer");
|
|
29947
30350
|
}
|
|
29948
30351
|
if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
|
|
29949
|
-
|
|
30352
|
+
reject2("partitionConfig maxPartitions must be a positive integer");
|
|
29950
30353
|
}
|
|
29951
30354
|
if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
|
|
29952
|
-
|
|
30355
|
+
reject2("partitionConfig watermark must be above 0 and at most 1");
|
|
29953
30356
|
}
|
|
29954
30357
|
return {
|
|
29955
30358
|
...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
|
|
@@ -29957,61 +30360,61 @@ function restoredPartitionConfig(raw) {
|
|
|
29957
30360
|
...typeof watermark === "number" ? { watermark } : {}
|
|
29958
30361
|
};
|
|
29959
30362
|
}
|
|
29960
|
-
function restoredScoring(raw) {
|
|
30363
|
+
function restoredScoring(raw, reject2) {
|
|
29961
30364
|
if (raw === void 0) {
|
|
29962
30365
|
return void 0;
|
|
29963
30366
|
}
|
|
29964
30367
|
if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
|
|
29965
|
-
|
|
30368
|
+
reject2("defaultScoring must be 'local', 'dfs', or 'broadcast'");
|
|
29966
30369
|
}
|
|
29967
30370
|
return raw;
|
|
29968
30371
|
}
|
|
29969
|
-
function restoredBoolean(raw, field) {
|
|
30372
|
+
function restoredBoolean(raw, field, reject2) {
|
|
29970
30373
|
if (raw === void 0) {
|
|
29971
30374
|
return void 0;
|
|
29972
30375
|
}
|
|
29973
30376
|
if (typeof raw !== "boolean") {
|
|
29974
|
-
|
|
30377
|
+
reject2(`${field} must be a boolean`);
|
|
29975
30378
|
}
|
|
29976
30379
|
return raw;
|
|
29977
30380
|
}
|
|
29978
|
-
function restoredRequired(raw) {
|
|
30381
|
+
function restoredRequired(raw, reject2) {
|
|
29979
30382
|
if (raw === void 0) {
|
|
29980
30383
|
return void 0;
|
|
29981
30384
|
}
|
|
29982
30385
|
if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
|
|
29983
|
-
|
|
30386
|
+
reject2("required must be a list of field paths");
|
|
29984
30387
|
}
|
|
29985
30388
|
return raw;
|
|
29986
30389
|
}
|
|
29987
|
-
function restoredVectorPromotion(raw) {
|
|
30390
|
+
function restoredVectorPromotion(raw, reject2) {
|
|
29988
30391
|
if (raw === void 0) {
|
|
29989
30392
|
return void 0;
|
|
29990
30393
|
}
|
|
29991
30394
|
if (!isPlainObject3(raw) || raw.hnswConfig !== void 0 && !isPlainObject3(raw.hnswConfig)) {
|
|
29992
|
-
|
|
30395
|
+
reject2("vectorPromotion must be an object");
|
|
29993
30396
|
}
|
|
29994
30397
|
return raw;
|
|
29995
30398
|
}
|
|
29996
|
-
function restoredEmbedding(raw) {
|
|
30399
|
+
function restoredEmbedding(raw, reject2 = rejectSnapshot) {
|
|
29997
30400
|
if (raw === void 0) {
|
|
29998
30401
|
return void 0;
|
|
29999
30402
|
}
|
|
30000
30403
|
if (!isPlainObject3(raw)) {
|
|
30001
|
-
|
|
30404
|
+
reject2("embedding must be an object");
|
|
30002
30405
|
}
|
|
30003
30406
|
const { fields, adapter } = raw;
|
|
30004
30407
|
if (adapter !== void 0 && typeof adapter !== "string") {
|
|
30005
|
-
|
|
30408
|
+
reject2("embedding adapter must be a name");
|
|
30006
30409
|
}
|
|
30007
30410
|
if (!isPlainObject3(fields)) {
|
|
30008
|
-
|
|
30411
|
+
reject2("embedding fields must be an object");
|
|
30009
30412
|
}
|
|
30010
30413
|
for (const value of Object.values(fields)) {
|
|
30011
30414
|
const isPath = typeof value === "string";
|
|
30012
30415
|
const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
|
|
30013
30416
|
if (!isPath && !isPathList) {
|
|
30014
|
-
|
|
30417
|
+
reject2("embedding fields must map vector fields to source paths");
|
|
30015
30418
|
}
|
|
30016
30419
|
}
|
|
30017
30420
|
return {
|
|
@@ -30019,30 +30422,40 @@ function restoredEmbedding(raw) {
|
|
|
30019
30422
|
...adapter !== void 0 ? { adapter } : {}
|
|
30020
30423
|
};
|
|
30021
30424
|
}
|
|
30022
|
-
function
|
|
30023
|
-
if (
|
|
30024
|
-
|
|
30425
|
+
function readSharedConfigFields(source, reject2) {
|
|
30426
|
+
if (source.tokenizer !== void 0 && typeof source.tokenizer !== "string") {
|
|
30427
|
+
reject2("tokenizer must be a name");
|
|
30428
|
+
}
|
|
30429
|
+
const stopWords2 = restoredStopWords(source.stopWords, source.stopWordList, reject2);
|
|
30430
|
+
const bm25 = restoredBm25(source.bm25, reject2);
|
|
30431
|
+
const surfaceForms = restoredBoolean(source.surfaceForms, "surfaceForms", reject2);
|
|
30432
|
+
const defaultScoring = restoredScoring(source.defaultScoring, reject2);
|
|
30433
|
+
const trackPositions = restoredBoolean(source.trackPositions, "trackPositions", reject2);
|
|
30434
|
+
const strict = restoredBoolean(source.strict, "strict", reject2);
|
|
30435
|
+
const required = restoredRequired(source.required, reject2);
|
|
30436
|
+
const vectorPromotion = restoredVectorPromotion(source.vectorPromotion, reject2);
|
|
30437
|
+
if (source.patternValueLimit !== void 0 && typeof source.patternValueLimit !== "number") {
|
|
30438
|
+
reject2("patternValueLimit must be a number");
|
|
30025
30439
|
}
|
|
30026
|
-
const stopWords2 = restoredStopWords(envelope.stopWords, envelope.stopWordList);
|
|
30027
|
-
const bm25 = restoredBm25(envelope.bm25);
|
|
30028
|
-
const surfaceForms = restoredBoolean(envelope.surfaceForms, "surfaceForms");
|
|
30029
|
-
const partitions = restoredPartitionConfig(envelope.partitionConfig);
|
|
30030
|
-
const defaultScoring = restoredScoring(envelope.defaultScoring);
|
|
30031
|
-
const trackPositions = restoredBoolean(envelope.trackPositions, "trackPositions");
|
|
30032
|
-
const strict = restoredBoolean(envelope.strict, "strict");
|
|
30033
|
-
const required = restoredRequired(envelope.required);
|
|
30034
|
-
const vectorPromotion = restoredVectorPromotion(envelope.vectorPromotion);
|
|
30035
30440
|
return {
|
|
30036
|
-
...typeof
|
|
30441
|
+
...typeof source.tokenizer === "string" ? { tokenizer: source.tokenizer } : {},
|
|
30037
30442
|
...stopWords2 !== void 0 ? { stopWords: stopWords2 } : {},
|
|
30038
30443
|
...bm25 !== void 0 ? { bm25 } : {},
|
|
30039
30444
|
...surfaceForms !== void 0 ? { surfaceForms } : {},
|
|
30040
|
-
...partitions !== void 0 ? { partitions } : {},
|
|
30041
30445
|
...defaultScoring !== void 0 ? { defaultScoring } : {},
|
|
30042
30446
|
...trackPositions !== void 0 ? { trackPositions } : {},
|
|
30043
30447
|
...strict !== void 0 ? { strict } : {},
|
|
30044
30448
|
...required !== void 0 ? { required } : {},
|
|
30045
|
-
...vectorPromotion !== void 0 ? { vectorPromotion } : {}
|
|
30449
|
+
...vectorPromotion !== void 0 ? { vectorPromotion } : {},
|
|
30450
|
+
...typeof source.patternValueLimit === "number" ? { patternValueLimit: source.patternValueLimit } : {}
|
|
30451
|
+
};
|
|
30452
|
+
}
|
|
30453
|
+
function restoredConfigFields(envelope) {
|
|
30454
|
+
const shared2 = readSharedConfigFields(envelope, rejectSnapshot);
|
|
30455
|
+
const partitions = restoredPartitionConfig(envelope.partitionConfig);
|
|
30456
|
+
return {
|
|
30457
|
+
...shared2,
|
|
30458
|
+
...partitions !== void 0 ? { partitions } : {}
|
|
30046
30459
|
};
|
|
30047
30460
|
}
|
|
30048
30461
|
|
|
@@ -30101,6 +30514,7 @@ async function createSnapshot(manager, entry) {
|
|
|
30101
30514
|
...config.strict !== void 0 ? { strict: config.strict } : {},
|
|
30102
30515
|
...config.required !== void 0 ? { required: config.required } : {},
|
|
30103
30516
|
...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
|
|
30517
|
+
...config.patternValueLimit !== void 0 ? { patternValueLimit: config.patternValueLimit } : {},
|
|
30104
30518
|
...config.embedding !== void 0 ? {
|
|
30105
30519
|
embedding: {
|
|
30106
30520
|
fields: config.embedding.fields,
|
|
@@ -30153,8 +30567,8 @@ async function restoreFromSnapshot(indexName, data, deps) {
|
|
|
30153
30567
|
const configFields = restoredConfigFields(envelope);
|
|
30154
30568
|
const embedding = restoredEmbedding(envelope.embedding);
|
|
30155
30569
|
const language = getLanguage(envelope.language);
|
|
30156
|
-
const schema = envelope.schema;
|
|
30157
|
-
|
|
30570
|
+
const schema = validateSchema(envelope.schema);
|
|
30571
|
+
validatePatternValueLimit(configFields.patternValueLimit);
|
|
30158
30572
|
validateVectorPromotion(configFields.vectorPromotion);
|
|
30159
30573
|
validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
|
|
30160
30574
|
if (configFields.required !== void 0 && configFields.required.length > 0) {
|
|
@@ -30506,7 +30920,7 @@ function listBindingOf(filters) {
|
|
|
30506
30920
|
function requireBoostableFields(boost, flatSchema) {
|
|
30507
30921
|
for (const [field, weight] of Object.entries(boost)) {
|
|
30508
30922
|
const fieldType = flatSchema[field];
|
|
30509
|
-
if (fieldType === void 0 || !
|
|
30923
|
+
if (fieldType === void 0 || !isWordIndexedFieldType(fieldType)) {
|
|
30510
30924
|
throw new NarsilError(
|
|
30511
30925
|
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
30512
30926
|
fieldType === void 0 ? `The engine boosts a text field that the schema declares, and the schema declares no field "${field}"` : `The engine boosts a text field alone, and the schema declares field "${field}" as "${fieldType}"`,
|
|
@@ -31924,15 +32338,18 @@ function createNarsilFromCore(core, config) {
|
|
|
31924
32338
|
release();
|
|
31925
32339
|
}
|
|
31926
32340
|
}
|
|
32341
|
+
function withOpenIndexWrite(indexName, action) {
|
|
32342
|
+
return emitWriteEventAfter(core, indexName, withOpenIndex(indexName, action));
|
|
32343
|
+
}
|
|
31927
32344
|
const narsil = {
|
|
31928
32345
|
createIndex(name, indexConfig) {
|
|
31929
|
-
return createEngineIndex(core, config, name, indexConfig);
|
|
32346
|
+
return emitWriteEventAfter(core, name, createEngineIndex(core, config, name, indexConfig));
|
|
31930
32347
|
},
|
|
31931
32348
|
registerEmbeddingAdapter(name, adapter) {
|
|
31932
32349
|
registerEngineEmbeddingAdapter(core, name, adapter);
|
|
31933
32350
|
},
|
|
31934
32351
|
dropIndex(name) {
|
|
31935
|
-
return dropEngineIndex(core, name);
|
|
32352
|
+
return emitWriteEventAfter(core, name, dropEngineIndex(core, name));
|
|
31936
32353
|
},
|
|
31937
32354
|
open(indexName) {
|
|
31938
32355
|
guardShutdown();
|
|
@@ -31978,22 +32395,22 @@ function createNarsilFromCore(core, config) {
|
|
|
31978
32395
|
return executor.getManager(indexName)?.getPartitionStats() ?? [];
|
|
31979
32396
|
},
|
|
31980
32397
|
insert(indexName, document2, docId, options) {
|
|
31981
|
-
return
|
|
32398
|
+
return withOpenIndexWrite(indexName, () => insertDocument(mutationCtx, indexName, document2, docId, options));
|
|
31982
32399
|
},
|
|
31983
32400
|
insertBatch(indexName, documents, options) {
|
|
31984
|
-
return
|
|
32401
|
+
return withOpenIndexWrite(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents, options));
|
|
31985
32402
|
},
|
|
31986
32403
|
remove(indexName, docId, options) {
|
|
31987
|
-
return
|
|
32404
|
+
return withOpenIndexWrite(indexName, () => removeDocument(mutationCtx, indexName, docId, options));
|
|
31988
32405
|
},
|
|
31989
32406
|
removeBatch(indexName, docIds, options) {
|
|
31990
|
-
return
|
|
32407
|
+
return withOpenIndexWrite(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
|
|
31991
32408
|
},
|
|
31992
32409
|
update(indexName, docId, document2, options) {
|
|
31993
|
-
return
|
|
32410
|
+
return withOpenIndexWrite(indexName, () => updateDocument(mutationCtx, indexName, docId, document2, options));
|
|
31994
32411
|
},
|
|
31995
32412
|
updateBatch(indexName, updates, options) {
|
|
31996
|
-
return
|
|
32413
|
+
return withOpenIndexWrite(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
|
|
31997
32414
|
},
|
|
31998
32415
|
waitForWrites(indexName) {
|
|
31999
32416
|
guardShutdown();
|
|
@@ -32041,7 +32458,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32041
32458
|
async rebuildAnalysis(indexName) {
|
|
32042
32459
|
guardShutdown();
|
|
32043
32460
|
requireIndex(indexName);
|
|
32044
|
-
await
|
|
32461
|
+
await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
|
|
32045
32462
|
},
|
|
32046
32463
|
async snapshot(indexName) {
|
|
32047
32464
|
guardShutdown();
|
|
@@ -32049,11 +32466,11 @@ function createNarsilFromCore(core, config) {
|
|
|
32049
32466
|
},
|
|
32050
32467
|
async restore(indexName, data) {
|
|
32051
32468
|
guardShutdown();
|
|
32052
|
-
|
|
32469
|
+
const restored = restoreFromSnapshot(indexName, data, {
|
|
32053
32470
|
executor,
|
|
32054
32471
|
indexRegistry,
|
|
32055
32472
|
getVectorFieldPaths,
|
|
32056
|
-
dropIndex:
|
|
32473
|
+
dropIndex: (name) => dropEngineIndex(core, name),
|
|
32057
32474
|
announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
|
|
32058
32475
|
requireManager,
|
|
32059
32476
|
durability,
|
|
@@ -32064,6 +32481,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32064
32481
|
clearAnalysisStale: core.analysisRebuild.clearStale,
|
|
32065
32482
|
indexState: core.indexState
|
|
32066
32483
|
});
|
|
32484
|
+
await emitWriteEventAfter(core, indexName, restored);
|
|
32067
32485
|
core.heapPressureNotifier.check(indexName);
|
|
32068
32486
|
},
|
|
32069
32487
|
async checkpoint(indexName) {
|
|
@@ -32076,7 +32494,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32076
32494
|
async clear(indexName) {
|
|
32077
32495
|
guardShutdown();
|
|
32078
32496
|
requireIndex(indexName);
|
|
32079
|
-
await
|
|
32497
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
32080
32498
|
await executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
32081
32499
|
await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
32082
32500
|
core.watermarkNotifier.forget(indexName);
|
|
@@ -32085,7 +32503,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32085
32503
|
async rebalance(indexName, targetPartitionCount) {
|
|
32086
32504
|
guardShutdown();
|
|
32087
32505
|
requireIndex(indexName);
|
|
32088
|
-
return
|
|
32506
|
+
return withOpenIndexWrite(
|
|
32089
32507
|
indexName,
|
|
32090
32508
|
() => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
|
|
32091
32509
|
);
|
|
@@ -32151,14 +32569,14 @@ function createNarsilFromCore(core, config) {
|
|
|
32151
32569
|
async compactVectors(indexName, fieldName) {
|
|
32152
32570
|
guardShutdown();
|
|
32153
32571
|
requireIndex(indexName);
|
|
32154
|
-
await
|
|
32572
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
32155
32573
|
compactVectors(requireManager(indexName), indexName, fieldName);
|
|
32156
32574
|
});
|
|
32157
32575
|
},
|
|
32158
32576
|
async optimizeVectors(indexName, fieldName) {
|
|
32159
32577
|
guardShutdown();
|
|
32160
32578
|
requireIndex(indexName);
|
|
32161
|
-
await
|
|
32579
|
+
await withOpenIndexWrite(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
|
|
32162
32580
|
},
|
|
32163
32581
|
vectorMaintenanceStatus(indexName) {
|
|
32164
32582
|
guardShutdown();
|