@delali/narsil 0.2.4-next.55 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -28
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-DlYw0Ov0.d.ts → capabilities-DSsOGz6F.d.ts} +2 -2
- package/dist/{chunk-BH5SYB5W.mjs → chunk-24RXHMEL.mjs} +604 -150
- package/dist/{chunk-D2S642IV.mjs → chunk-3LHGNRYE.mjs} +1 -1
- package/dist/{chunk-TVT5V4E7.mjs → chunk-B2TFROFX.mjs} +299 -168
- package/dist/{chunk-L225SHNB.mjs → chunk-L6TNPLAR.mjs} +19 -5
- package/dist/{chunk-U2AL3SON.mjs → chunk-THYRHHO2.mjs} +3 -3
- package/dist/{chunk-NTZYA3SJ.mjs → chunk-U7KYW4BA.mjs} +3 -3
- package/dist/{chunk-FRQVMFY7.mjs → chunk-USKFU7J5.mjs} +1 -0
- package/dist/{chunk-4NVPQ6S5.mjs → chunk-UXNWQZST.mjs} +12 -3
- package/dist/{chunk-RGS7FMCB.mjs → chunk-VMRR3DQU.mjs} +74 -16
- package/dist/{chunk-2GWVQWGA.mjs → chunk-YMZCMH5M.mjs} +45 -1
- package/dist/{chunk-UTOFOJOR.mjs → chunk-YNVIUGF5.mjs} +6 -2
- package/dist/{chunk-5T6AOR37.mjs → chunk-ZNZBXMP5.mjs} +601 -60
- package/dist/{chunk-XNQAIO74.mjs → chunk-ZZT3EAUC.mjs} +177 -309
- package/dist/{client-CQUqt9Ir.d.ts → client-BQK6Tf6A.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +1 -0
- package/dist/{config-CNas6wJH.d.ts → config-BhldZgaK.d.ts} +2 -2
- package/dist/{config-bXxnJrQc.d.ts → config-DmXULPa-.d.ts} +15 -7
- package/dist/direct-executor-ALAVE7FC.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +382 -162
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-Dj4I91Jv.d.ts → engine-CIOyFozK.d.ts} +15 -12
- package/dist/index.browser.mjs +1565 -528
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +11 -11
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-xrXOZMbw.d.ts → memory-yFaVn2xZ.d.ts} +199 -107
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +11 -11
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/{schema-mLpXZBEh.d.ts → schema-d5Afu03A.d.ts} +7 -2
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +10 -10
- package/dist/{tasks-Df73Tjgq.d.ts → tasks-CH2Dod8w.d.ts} +1 -1
- package/dist/{thread-5UP6MGED.mjs → thread-DYHEQRVY.mjs} +14 -10
- package/dist/{types-BTIVO8DI.d.ts → types-DAK7mnUZ.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +4 -4
- package/package.json +9 -9
- package/dist/direct-executor-RBA5O7EN.mjs +0 -12
package/dist/index.browser.mjs
CHANGED
|
@@ -14,6 +14,7 @@ var ErrorCodes = {
|
|
|
14
14
|
INDEX_ALREADY_EXISTS: "INDEX_ALREADY_EXISTS",
|
|
15
15
|
INDEX_ORPHANED: "INDEX_ORPHANED",
|
|
16
16
|
INDEX_REOPEN_CAPACITY_EXHAUSTED: "INDEX_REOPEN_CAPACITY_EXHAUSTED",
|
|
17
|
+
INSTANCE_SHUT_DOWN: "INSTANCE_SHUT_DOWN",
|
|
17
18
|
PARTITION_CORRUPTED: "PARTITION_CORRUPTED",
|
|
18
19
|
PARTITION_REBALANCING_BACKPRESSURE: "PARTITION_REBALANCING_BACKPRESSURE",
|
|
19
20
|
WORKER_CRASHED: "WORKER_CRASHED",
|
|
@@ -1808,6 +1809,84 @@ function getLanguage(name) {
|
|
|
1808
1809
|
return lang;
|
|
1809
1810
|
}
|
|
1810
1811
|
|
|
1812
|
+
// src/engine/config-keys.ts
|
|
1813
|
+
var NARSIL_CONFIG_KEYS = {
|
|
1814
|
+
persistence: true,
|
|
1815
|
+
invalidation: true,
|
|
1816
|
+
plugins: true,
|
|
1817
|
+
idGenerator: true,
|
|
1818
|
+
workers: true,
|
|
1819
|
+
embedding: true,
|
|
1820
|
+
embeddingAdapters: true,
|
|
1821
|
+
durability: true,
|
|
1822
|
+
analysis: true,
|
|
1823
|
+
lifecycle: true
|
|
1824
|
+
};
|
|
1825
|
+
var DURABILITY_CONFIG_KEYS = {
|
|
1826
|
+
tier: true,
|
|
1827
|
+
directory: true,
|
|
1828
|
+
mode: true,
|
|
1829
|
+
flushIntervalMs: true,
|
|
1830
|
+
segmentMaxBytes: true,
|
|
1831
|
+
checkpointIntervalMs: true,
|
|
1832
|
+
checkpointMutationThreshold: true,
|
|
1833
|
+
compactionThreshold: true
|
|
1834
|
+
};
|
|
1835
|
+
var WORKER_CONFIG_KEYS = {
|
|
1836
|
+
enabled: true,
|
|
1837
|
+
count: true,
|
|
1838
|
+
promotionThreshold: true,
|
|
1839
|
+
idleTimeoutMs: true,
|
|
1840
|
+
mainCopyQueries: true,
|
|
1841
|
+
bootstrapModule: true
|
|
1842
|
+
};
|
|
1843
|
+
var LIFECYCLE_CONFIG_KEYS = {
|
|
1844
|
+
idleTimeoutMs: true,
|
|
1845
|
+
maxOpenIndexes: true,
|
|
1846
|
+
maxOpenBytes: true,
|
|
1847
|
+
maxReopenWaiters: true
|
|
1848
|
+
};
|
|
1849
|
+
var ANALYSIS_CONFIG_KEYS = {
|
|
1850
|
+
rebuild: true,
|
|
1851
|
+
onStaleAnalysis: true
|
|
1852
|
+
};
|
|
1853
|
+
function requireKnownKeys(section, value, known) {
|
|
1854
|
+
if (value === void 0 || value === null || typeof value !== "object") return;
|
|
1855
|
+
for (const key of Object.keys(value)) {
|
|
1856
|
+
if (Object.hasOwn(known, key)) continue;
|
|
1857
|
+
throw new NarsilError(
|
|
1858
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1859
|
+
`The configuration sets "${section}${key}", which is no setting that createNarsil accepts. The settings at that level are ${Object.keys(known).join(", ")}`,
|
|
1860
|
+
{ setting: `${section}${key}`, settings: Object.keys(known) }
|
|
1861
|
+
);
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1864
|
+
function requireNamedPlugins(plugins) {
|
|
1865
|
+
if (plugins === void 0) return;
|
|
1866
|
+
if (!Array.isArray(plugins)) {
|
|
1867
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The configuration sets "plugins" to something other than a list');
|
|
1868
|
+
}
|
|
1869
|
+
plugins.forEach((plugin, position) => {
|
|
1870
|
+
const name = plugin?.name;
|
|
1871
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
1872
|
+
throw new NarsilError(
|
|
1873
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1874
|
+
`The plugin at position ${position} carries no name, and the engine names each plugin in its errors and event payloads`,
|
|
1875
|
+
{ position }
|
|
1876
|
+
);
|
|
1877
|
+
}
|
|
1878
|
+
});
|
|
1879
|
+
}
|
|
1880
|
+
function requireKnownConfig(config) {
|
|
1881
|
+
if (config === void 0) return;
|
|
1882
|
+
requireKnownKeys("", config, NARSIL_CONFIG_KEYS);
|
|
1883
|
+
requireKnownKeys("durability.", config.durability, DURABILITY_CONFIG_KEYS);
|
|
1884
|
+
requireKnownKeys("workers.", config.workers, WORKER_CONFIG_KEYS);
|
|
1885
|
+
requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
|
|
1886
|
+
requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
|
|
1887
|
+
requireNamedPlugins(config.plugins);
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1811
1890
|
// src/core/id-generator.ts
|
|
1812
1891
|
function generateId() {
|
|
1813
1892
|
const bytes = new Uint8Array(16);
|
|
@@ -1843,6 +1922,9 @@ function isTextFieldType(fieldType) {
|
|
|
1843
1922
|
return fieldType === "string" || fieldType === SORTABLE_TEXT_FIELD_TYPE;
|
|
1844
1923
|
}
|
|
1845
1924
|
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
1925
|
+
function isGeopointOrVectorType(fieldType) {
|
|
1926
|
+
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
1927
|
+
}
|
|
1846
1928
|
var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
|
|
1847
1929
|
var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
|
|
1848
1930
|
var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
@@ -2129,7 +2211,7 @@ function collectExtraFields(doc, schema, prefix, extras) {
|
|
|
2129
2211
|
for (const key of Object.keys(doc)) {
|
|
2130
2212
|
if (key === "id") continue;
|
|
2131
2213
|
const path = prefix ? `${prefix}.${key}` : key;
|
|
2132
|
-
if (!(key
|
|
2214
|
+
if (!Object.hasOwn(schema, key)) {
|
|
2133
2215
|
extras.push(path);
|
|
2134
2216
|
continue;
|
|
2135
2217
|
}
|
|
@@ -2257,9 +2339,13 @@ function flattenRecursive(schema, prefix, result) {
|
|
|
2257
2339
|
}
|
|
2258
2340
|
}
|
|
2259
2341
|
}
|
|
2342
|
+
var flattenedSchemas = /* @__PURE__ */ new WeakMap();
|
|
2260
2343
|
function flattenSchema(schema) {
|
|
2261
|
-
const
|
|
2344
|
+
const cached = flattenedSchemas.get(schema);
|
|
2345
|
+
if (cached !== void 0) return cached;
|
|
2346
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
2262
2347
|
flattenRecursive(schema, "", result);
|
|
2348
|
+
flattenedSchemas.set(schema, Object.freeze(result));
|
|
2263
2349
|
return result;
|
|
2264
2350
|
}
|
|
2265
2351
|
function extractVectorFieldsFromSchema(schema) {
|
|
@@ -2911,6 +2997,16 @@ function compareSortStrings(a, b) {
|
|
|
2911
2997
|
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
2912
2998
|
|
|
2913
2999
|
// src/core/ordering/sort-values.ts
|
|
3000
|
+
var SORT_MODES = ["min", "max", "avg", "median"];
|
|
3001
|
+
function isSortMode(value) {
|
|
3002
|
+
return value === "min" || value === "max" || value === "avg" || value === "median";
|
|
3003
|
+
}
|
|
3004
|
+
function defaultSortMode(direction) {
|
|
3005
|
+
return direction === "desc" ? "max" : "min";
|
|
3006
|
+
}
|
|
3007
|
+
function sortModeOf(entry) {
|
|
3008
|
+
return entry.mode ?? defaultSortMode(entry.direction);
|
|
3009
|
+
}
|
|
2914
3010
|
function truncateSortString(value) {
|
|
2915
3011
|
if (value.length <= SORT_VALUE_MAX_CODE_POINTS) return value;
|
|
2916
3012
|
let index = 0;
|
|
@@ -2932,6 +3028,40 @@ function toComparableSortValue(value) {
|
|
|
2932
3028
|
if (typeof value === "boolean") return value;
|
|
2933
3029
|
return null;
|
|
2934
3030
|
}
|
|
3031
|
+
function numericReduction(values, mode) {
|
|
3032
|
+
const numbers = [];
|
|
3033
|
+
for (const element of values) {
|
|
3034
|
+
if (typeof element === "number" && Number.isFinite(element)) numbers.push(element);
|
|
3035
|
+
}
|
|
3036
|
+
if (numbers.length === 0) return null;
|
|
3037
|
+
if (mode === "avg") {
|
|
3038
|
+
let mean = 0;
|
|
3039
|
+
for (let index = 0; index < numbers.length; index++) {
|
|
3040
|
+
mean += (numbers[index] - mean) / (index + 1);
|
|
3041
|
+
}
|
|
3042
|
+
return mean;
|
|
3043
|
+
}
|
|
3044
|
+
numbers.sort((a, b) => a - b);
|
|
3045
|
+
const middle = numbers.length >> 1;
|
|
3046
|
+
if (numbers.length % 2 === 1) return numbers[middle];
|
|
3047
|
+
return numbers[middle - 1] / 2 + numbers[middle] / 2;
|
|
3048
|
+
}
|
|
3049
|
+
function toReducedSortValue(value, mode) {
|
|
3050
|
+
if (!Array.isArray(value)) return toComparableSortValue(value);
|
|
3051
|
+
if (mode === "avg" || mode === "median") return numericReduction(value, mode);
|
|
3052
|
+
let chosen = null;
|
|
3053
|
+
for (const element of value) {
|
|
3054
|
+
const comparable = toComparableSortValue(element);
|
|
3055
|
+
if (comparable === null) continue;
|
|
3056
|
+
if (chosen === null) {
|
|
3057
|
+
chosen = comparable;
|
|
3058
|
+
continue;
|
|
3059
|
+
}
|
|
3060
|
+
const comparison = comparePresentValues(comparable, chosen);
|
|
3061
|
+
if (mode === "min" ? comparison < 0 : comparison > 0) chosen = comparable;
|
|
3062
|
+
}
|
|
3063
|
+
return chosen;
|
|
3064
|
+
}
|
|
2935
3065
|
function readSortField(document2, path) {
|
|
2936
3066
|
const segments = path.split(".");
|
|
2937
3067
|
let current = document2;
|
|
@@ -3236,6 +3366,14 @@ var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
|
3236
3366
|
var MAX_TERM_FREQUENCY = 65535;
|
|
3237
3367
|
|
|
3238
3368
|
// src/core/fuzzy.ts
|
|
3369
|
+
function fuzzyTermMatches(queryTerm, indexTerm, tolerance, prefixLength) {
|
|
3370
|
+
if (queryTerm === indexTerm) return true;
|
|
3371
|
+
if (tolerance <= 0) return false;
|
|
3372
|
+
if (prefixLength > 0 && queryTerm.length >= prefixLength && !indexTerm.startsWith(queryTerm.slice(0, prefixLength))) {
|
|
3373
|
+
return false;
|
|
3374
|
+
}
|
|
3375
|
+
return boundedLevenshtein(queryTerm, indexTerm, tolerance).withinTolerance;
|
|
3376
|
+
}
|
|
3239
3377
|
function boundedLevenshtein(a, b, tolerance) {
|
|
3240
3378
|
if (tolerance < 0) return { distance: -1, withinTolerance: false };
|
|
3241
3379
|
if (a === b) return { distance: 0, withinTolerance: true };
|
|
@@ -4033,6 +4171,7 @@ function getNestedValue(obj, path) {
|
|
|
4033
4171
|
let current = obj;
|
|
4034
4172
|
for (let i = 0; i < parts.length; i++) {
|
|
4035
4173
|
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
4174
|
+
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
4036
4175
|
current = current[parts[i]];
|
|
4037
4176
|
}
|
|
4038
4177
|
return current;
|
|
@@ -4093,6 +4232,10 @@ function getAllInternalDocIds(docStore) {
|
|
|
4093
4232
|
}
|
|
4094
4233
|
|
|
4095
4234
|
// src/core/partition/facets.ts
|
|
4235
|
+
var UNDECLARED_FIELD_TYPE = "";
|
|
4236
|
+
function keptFacetValues(limit) {
|
|
4237
|
+
return limit !== void 0 && limit > 0 ? Math.floor(limit) : void 0;
|
|
4238
|
+
}
|
|
4096
4239
|
function ordinalBitsetOf(state, matched) {
|
|
4097
4240
|
if ("ordinalBitset" in matched) return matched.ordinalBitset;
|
|
4098
4241
|
const bitset = createBitSet(state.docStore.internalIdCapacity());
|
|
@@ -4140,7 +4283,7 @@ function countValuesFromDocuments(state, fieldPath, matched) {
|
|
|
4140
4283
|
function countRangeFromDocuments(state, fieldPath, fieldType, range, matched) {
|
|
4141
4284
|
let count = 0;
|
|
4142
4285
|
for (const value of matchedFieldValues(state, fieldPath, matched)) {
|
|
4143
|
-
if (fieldType === "number[]" && Array.isArray(value)) {
|
|
4286
|
+
if ((fieldType === "number[]" || fieldType === UNDECLARED_FIELD_TYPE) && Array.isArray(value)) {
|
|
4144
4287
|
for (const v of value) {
|
|
4145
4288
|
if (v >= range.from && v < range.to) {
|
|
4146
4289
|
count++;
|
|
@@ -4202,13 +4345,12 @@ function computeFacets(state, matched, config, schema) {
|
|
|
4202
4345
|
return matchedBitset;
|
|
4203
4346
|
};
|
|
4204
4347
|
for (const [fieldPath, facetOpts] of Object.entries(config)) {
|
|
4205
|
-
const fieldType = flatSchema[fieldPath];
|
|
4206
|
-
|
|
4207
|
-
const valueCounts = facetOpts.ranges && (fieldType === "number" || fieldType === "number[]") ? countNumericRanges(state, fieldPath, fieldType, facetOpts.ranges, bitset, matched) : countFieldValues(state, fieldPath, fieldType, bitset, matched);
|
|
4348
|
+
const fieldType = flatSchema[fieldPath] ?? UNDECLARED_FIELD_TYPE;
|
|
4349
|
+
const valueCounts = facetOpts.ranges && (fieldType === "number" || fieldType === "number[]" || fieldType === UNDECLARED_FIELD_TYPE) ? countNumericRanges(state, fieldPath, fieldType, facetOpts.ranges, bitset, matched) : countFieldValues(state, fieldPath, fieldType, bitset, matched);
|
|
4208
4350
|
const ordered = Array.from(valueCounts.entries());
|
|
4209
4351
|
const sortDir = facetOpts.sort ?? "desc";
|
|
4210
4352
|
ordered.sort((a, b) => (sortDir === "asc" ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
4211
|
-
const keep =
|
|
4353
|
+
const keep = Math.min(keptFacetValues(facetOpts.limit) ?? ordered.length, ordered.length);
|
|
4212
4354
|
const entries = ordered.slice(0, keep);
|
|
4213
4355
|
let errorBound = 0;
|
|
4214
4356
|
for (let index = keep; index < ordered.length; index++) {
|
|
@@ -4225,6 +4367,9 @@ function computeFacets(state, matched, config, schema) {
|
|
|
4225
4367
|
|
|
4226
4368
|
// src/geo/constants.ts
|
|
4227
4369
|
var DEG_TO_RAD = Math.PI / 180;
|
|
4370
|
+
var MIN_POLYGON_POINTS = 3;
|
|
4371
|
+
var ANTIMERIDIAN_SPAN_DEGREES = 180;
|
|
4372
|
+
var FULL_TURN_DEGREES = 360;
|
|
4228
4373
|
var VINCENTY_MAX_ITERATIONS = 200;
|
|
4229
4374
|
var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
|
|
4230
4375
|
|
|
@@ -4243,24 +4388,66 @@ function haversineDistance(lat1, lon1, lat2, lon2) {
|
|
|
4243
4388
|
}
|
|
4244
4389
|
|
|
4245
4390
|
// src/geo/polygon.ts
|
|
4246
|
-
function
|
|
4247
|
-
if (
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
}
|
|
4261
|
-
|
|
4391
|
+
function requirePolygonRing(points) {
|
|
4392
|
+
if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
|
|
4393
|
+
throw new NarsilError(
|
|
4394
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4395
|
+
`A polygon encloses an area only with at least ${MIN_POLYGON_POINTS} points`,
|
|
4396
|
+
{ points: Array.isArray(points) ? points.length : typeof points, minimum: MIN_POLYGON_POINTS }
|
|
4397
|
+
);
|
|
4398
|
+
}
|
|
4399
|
+
points.forEach((point, position) => {
|
|
4400
|
+
const { lat, lon } = point ?? {};
|
|
4401
|
+
if (typeof lat === "number" && Number.isFinite(lat) && typeof lon === "number" && Number.isFinite(lon)) return;
|
|
4402
|
+
throw new NarsilError(
|
|
4403
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4404
|
+
`A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
|
|
4405
|
+
{ position }
|
|
4406
|
+
);
|
|
4407
|
+
});
|
|
4408
|
+
}
|
|
4409
|
+
function signedArea(polygon) {
|
|
4410
|
+
let area = 0;
|
|
4411
|
+
for (let i = 0, j = polygon.length - 1; i < polygon.length; j = i++) {
|
|
4412
|
+
area += polygon[j].lon * polygon[i].lat - polygon[i].lon * polygon[j].lat;
|
|
4262
4413
|
}
|
|
4263
|
-
return
|
|
4414
|
+
return area;
|
|
4415
|
+
}
|
|
4416
|
+
function crossesAntimeridian(polygon) {
|
|
4417
|
+
let west = Number.POSITIVE_INFINITY;
|
|
4418
|
+
let east = Number.NEGATIVE_INFINITY;
|
|
4419
|
+
for (const point of polygon) {
|
|
4420
|
+
if (point.lon < west) west = point.lon;
|
|
4421
|
+
if (point.lon > east) east = point.lon;
|
|
4422
|
+
}
|
|
4423
|
+
return east - west >= ANTIMERIDIAN_SPAN_DEGREES && signedArea(polygon) < 0;
|
|
4424
|
+
}
|
|
4425
|
+
function eastward(lon) {
|
|
4426
|
+
return lon < 0 ? lon + FULL_TURN_DEGREES : lon;
|
|
4427
|
+
}
|
|
4428
|
+
function polygonContainment(polygon) {
|
|
4429
|
+
const n = polygon.length;
|
|
4430
|
+
if (n < MIN_POLYGON_POINTS) return () => false;
|
|
4431
|
+
const unwrap = crossesAntimeridian(polygon);
|
|
4432
|
+
const lats = new Float64Array(n);
|
|
4433
|
+
const lons = new Float64Array(n);
|
|
4434
|
+
for (let i = 0; i < n; i++) {
|
|
4435
|
+
lats[i] = polygon[i].lat;
|
|
4436
|
+
lons[i] = unwrap ? eastward(polygon[i].lon) : polygon[i].lon;
|
|
4437
|
+
}
|
|
4438
|
+
return (lat, rawLon) => {
|
|
4439
|
+
const lon = unwrap ? eastward(rawLon) : rawLon;
|
|
4440
|
+
let inside = false;
|
|
4441
|
+
for (let i = 0, j = n - 1; i < n; j = i++) {
|
|
4442
|
+
const lonI = lons[i];
|
|
4443
|
+
const lonJ = lons[j];
|
|
4444
|
+
if (lonI > lon !== lonJ > lon) {
|
|
4445
|
+
const intersectLat = lats[i] + (lats[j] - lats[i]) / (lonJ - lonI) * (lon - lonI);
|
|
4446
|
+
if (lat < intersectLat) inside = !inside;
|
|
4447
|
+
}
|
|
4448
|
+
}
|
|
4449
|
+
return inside;
|
|
4450
|
+
};
|
|
4264
4451
|
}
|
|
4265
4452
|
|
|
4266
4453
|
// src/geo/vincenty.ts
|
|
@@ -4345,9 +4532,10 @@ function createGeoIndex() {
|
|
|
4345
4532
|
},
|
|
4346
4533
|
polygonQuery(points, inside) {
|
|
4347
4534
|
const result = /* @__PURE__ */ new Set();
|
|
4535
|
+
const contains = polygonContainment(points);
|
|
4348
4536
|
for (let i = 0; i < entries.length; i++) {
|
|
4349
4537
|
const entry = entries[i];
|
|
4350
|
-
const withinPolygon =
|
|
4538
|
+
const withinPolygon = contains(entry.lat, entry.lon);
|
|
4351
4539
|
if (inside ? withinPolygon : !withinPolygon) {
|
|
4352
4540
|
result.add(entry.docId);
|
|
4353
4541
|
}
|
|
@@ -5108,13 +5296,175 @@ var FIELD_FILTER_OPERATORS = [
|
|
|
5108
5296
|
"polygon"
|
|
5109
5297
|
];
|
|
5110
5298
|
|
|
5299
|
+
// src/filters/constants.ts
|
|
5300
|
+
var MAX_FILTER_DEPTH = 30;
|
|
5301
|
+
|
|
5302
|
+
// src/filters/operands.ts
|
|
5303
|
+
var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
|
|
5304
|
+
var TEXT_OPERATORS = ["startsWith", "endsWith"];
|
|
5305
|
+
var EQUALITY_OPERATORS = ["eq", "ne"];
|
|
5306
|
+
var LIST_OPERATORS = ["in", "nin"];
|
|
5307
|
+
function operandKindOf(type) {
|
|
5308
|
+
if (type === void 0) return void 0;
|
|
5309
|
+
if (type === "number" || type === "number[]") return "number";
|
|
5310
|
+
if (type === "boolean" || type === "boolean[]") return "boolean";
|
|
5311
|
+
if (type === "geopoint") return "geopoint";
|
|
5312
|
+
if (type.startsWith("vector[")) return "vector";
|
|
5313
|
+
return "text";
|
|
5314
|
+
}
|
|
5315
|
+
function isFiniteNumber(value) {
|
|
5316
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
5317
|
+
}
|
|
5318
|
+
function kindOfValue(value) {
|
|
5319
|
+
if (isFiniteNumber(value)) return "number";
|
|
5320
|
+
if (typeof value === "boolean") return "boolean";
|
|
5321
|
+
if (typeof value === "string") return "text";
|
|
5322
|
+
return void 0;
|
|
5323
|
+
}
|
|
5324
|
+
function invalid(message, fieldPath, operator) {
|
|
5325
|
+
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message, { fieldPath, operator });
|
|
5326
|
+
}
|
|
5327
|
+
function requireFieldKind(fieldPath, operator, type, accepted) {
|
|
5328
|
+
const kind = operandKindOf(type);
|
|
5329
|
+
if (kind === void 0 || accepted.includes(kind)) return;
|
|
5330
|
+
invalid(
|
|
5331
|
+
`The engine applies "${operator}" to ${accepted.join(" or ")} fields alone, and the schema declares field "${fieldPath}" as "${type}"`,
|
|
5332
|
+
fieldPath,
|
|
5333
|
+
operator
|
|
5334
|
+
);
|
|
5335
|
+
}
|
|
5336
|
+
function requireRadius(fieldPath, radius) {
|
|
5337
|
+
const circle = radius;
|
|
5338
|
+
if (circle === null || typeof circle !== "object") {
|
|
5339
|
+
invalid(`"radius" on field "${fieldPath}" must be an object`, fieldPath, "radius");
|
|
5340
|
+
}
|
|
5341
|
+
for (const key of ["lat", "lon", "distance"]) {
|
|
5342
|
+
if (!isFiniteNumber(circle[key])) {
|
|
5343
|
+
invalid(`"radius.${key}" on field "${fieldPath}" must be a finite number`, fieldPath, "radius");
|
|
5344
|
+
}
|
|
5345
|
+
}
|
|
5346
|
+
if (circle.distance < 0) {
|
|
5347
|
+
invalid(
|
|
5348
|
+
`"radius.distance" on field "${fieldPath}" must be zero or more, and this filter sets ${String(circle.distance)}`,
|
|
5349
|
+
fieldPath,
|
|
5350
|
+
"radius"
|
|
5351
|
+
);
|
|
5352
|
+
}
|
|
5353
|
+
}
|
|
5354
|
+
function invalidShape(message) {
|
|
5355
|
+
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message);
|
|
5356
|
+
}
|
|
5357
|
+
function isObjectRecord(value) {
|
|
5358
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
5359
|
+
}
|
|
5360
|
+
function undeclaredFieldOnStrictIndex(field, option) {
|
|
5361
|
+
return new NarsilError(
|
|
5362
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
5363
|
+
`The index is strict and its schema declares no field "${field}", so no stored document has a value in that field for this ${option}`,
|
|
5364
|
+
{ field, option }
|
|
5365
|
+
);
|
|
5366
|
+
}
|
|
5367
|
+
function requireValidExpression(expression, rules, depth) {
|
|
5368
|
+
if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
|
|
5369
|
+
if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
|
|
5370
|
+
const { fields, and, or, not } = expression;
|
|
5371
|
+
if (fields !== void 0) {
|
|
5372
|
+
if (!isObjectRecord(fields)) invalidShape('"fields" in a filter expression must be an object keyed by field path');
|
|
5373
|
+
for (const [fieldPath, filter] of Object.entries(fields)) {
|
|
5374
|
+
if (rules.declaredOnly && !Object.hasOwn(rules.fieldTypes, fieldPath)) {
|
|
5375
|
+
throw undeclaredFieldOnStrictIndex(fieldPath, "filter");
|
|
5376
|
+
}
|
|
5377
|
+
if (isObjectRecord(filter)) requireValidOperands(fieldPath, filter, rules.fieldTypes[fieldPath]);
|
|
5378
|
+
}
|
|
5379
|
+
}
|
|
5380
|
+
for (const [clause, list] of [
|
|
5381
|
+
["and", and],
|
|
5382
|
+
["or", or]
|
|
5383
|
+
]) {
|
|
5384
|
+
if (list === void 0) continue;
|
|
5385
|
+
if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
|
|
5386
|
+
for (const nested of list) requireValidExpression(nested, rules, depth + 1);
|
|
5387
|
+
}
|
|
5388
|
+
if (not !== void 0) requireValidExpression(not, rules, depth + 1);
|
|
5389
|
+
}
|
|
5390
|
+
function requireValidFilter(expression, fieldTypes, declaredOnly = false) {
|
|
5391
|
+
if (expression === void 0 || expression === null) return;
|
|
5392
|
+
requireValidExpression(expression, { fieldTypes, declaredOnly }, 0);
|
|
5393
|
+
}
|
|
5394
|
+
function requireValidOperands(fieldPath, filter, type) {
|
|
5395
|
+
for (const operator of RANGE_OPERATORS) {
|
|
5396
|
+
if (filter[operator] === void 0) continue;
|
|
5397
|
+
if (!isFiniteNumber(filter[operator])) {
|
|
5398
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a finite number`, fieldPath, operator);
|
|
5399
|
+
}
|
|
5400
|
+
requireFieldKind(fieldPath, operator, type, ["number"]);
|
|
5401
|
+
}
|
|
5402
|
+
if (filter.between !== void 0) {
|
|
5403
|
+
const bounds = filter.between;
|
|
5404
|
+
if (!Array.isArray(bounds) || bounds.length !== 2 || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
|
|
5405
|
+
invalid(
|
|
5406
|
+
`"between" on field "${fieldPath}" must be two finite numbers, the lower bound first`,
|
|
5407
|
+
fieldPath,
|
|
5408
|
+
"between"
|
|
5409
|
+
);
|
|
5410
|
+
}
|
|
5411
|
+
requireFieldKind(fieldPath, "between", type, ["number"]);
|
|
5412
|
+
}
|
|
5413
|
+
for (const operator of TEXT_OPERATORS) {
|
|
5414
|
+
if (filter[operator] === void 0) continue;
|
|
5415
|
+
if (typeof filter[operator] !== "string") {
|
|
5416
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
|
|
5417
|
+
}
|
|
5418
|
+
requireFieldKind(fieldPath, operator, type, ["text"]);
|
|
5419
|
+
}
|
|
5420
|
+
for (const operator of EQUALITY_OPERATORS) {
|
|
5421
|
+
if (filter[operator] === void 0) continue;
|
|
5422
|
+
const kind = kindOfValue(filter[operator]);
|
|
5423
|
+
if (kind === void 0) {
|
|
5424
|
+
invalid(
|
|
5425
|
+
`"${operator}" on field "${fieldPath}" must be a string, a finite number, or a boolean`,
|
|
5426
|
+
fieldPath,
|
|
5427
|
+
operator
|
|
5428
|
+
);
|
|
5429
|
+
}
|
|
5430
|
+
requireFieldKind(fieldPath, operator, type, [kind]);
|
|
5431
|
+
}
|
|
5432
|
+
for (const operator of LIST_OPERATORS) {
|
|
5433
|
+
if (filter[operator] === void 0) continue;
|
|
5434
|
+
const values = filter[operator];
|
|
5435
|
+
if (!Array.isArray(values) || values.some((value) => typeof value !== "string")) {
|
|
5436
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a list of strings`, fieldPath, operator);
|
|
5437
|
+
}
|
|
5438
|
+
requireFieldKind(fieldPath, operator, type, ["text"]);
|
|
5439
|
+
}
|
|
5440
|
+
if (filter.size !== void 0) {
|
|
5441
|
+
const size = filter.size;
|
|
5442
|
+
if (size === null || typeof size !== "object") {
|
|
5443
|
+
invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
|
|
5444
|
+
}
|
|
5445
|
+
requireValidOperands(`${fieldPath}.size`, size, "number");
|
|
5446
|
+
}
|
|
5447
|
+
if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
|
|
5448
|
+
if (filter.polygon !== void 0) {
|
|
5449
|
+
if (!isObjectRecord(filter.polygon)) {
|
|
5450
|
+
invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
|
|
5451
|
+
}
|
|
5452
|
+
requirePolygonRing(filter.polygon.points);
|
|
5453
|
+
}
|
|
5454
|
+
}
|
|
5455
|
+
|
|
5111
5456
|
// src/filters/operators/shared.ts
|
|
5112
5457
|
var METERS_PER_KM = 1e3;
|
|
5113
5458
|
var METERS_PER_MI = 1609.344;
|
|
5114
5459
|
function convertToMeters(distance, unit) {
|
|
5115
5460
|
if (unit === "km") return distance * METERS_PER_KM;
|
|
5116
5461
|
if (unit === "mi") return distance * METERS_PER_MI;
|
|
5117
|
-
return distance;
|
|
5462
|
+
if (unit === "m") return distance;
|
|
5463
|
+
throw new NarsilError(
|
|
5464
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
5465
|
+
`The engine measures a radius in "km", "mi", or "m", and "${String(unit)}" is none of them`,
|
|
5466
|
+
{ unit: String(unit) }
|
|
5467
|
+
);
|
|
5118
5468
|
}
|
|
5119
5469
|
function matchesNumericComparison(value, filter) {
|
|
5120
5470
|
if (filter.eq !== void 0 && value !== filter.eq) return false;
|
|
@@ -5126,6 +5476,27 @@ function matchesNumericComparison(value, filter) {
|
|
|
5126
5476
|
if (filter.between !== void 0 && !(value >= filter.between[0] && value <= filter.between[1])) return false;
|
|
5127
5477
|
return true;
|
|
5128
5478
|
}
|
|
5479
|
+
function holdsValue(stored, value) {
|
|
5480
|
+
return Array.isArray(stored) ? stored.includes(value) : stored === value;
|
|
5481
|
+
}
|
|
5482
|
+
function lacksValue(stored, value) {
|
|
5483
|
+
if (stored === void 0 || stored === null) return false;
|
|
5484
|
+
return Array.isArray(stored) ? !stored.includes(value) : stored !== value;
|
|
5485
|
+
}
|
|
5486
|
+
function isStringIn(values) {
|
|
5487
|
+
return (element) => typeof element === "string" && values.has(element);
|
|
5488
|
+
}
|
|
5489
|
+
function holdsStringIn(stored, values) {
|
|
5490
|
+
return Array.isArray(stored) ? stored.some(isStringIn(values)) : isStringIn(values)(stored);
|
|
5491
|
+
}
|
|
5492
|
+
function holdsNoStringIn(stored, values) {
|
|
5493
|
+
if (Array.isArray(stored)) return !stored.some(isStringIn(values));
|
|
5494
|
+
return typeof stored === "string" && !values.has(stored);
|
|
5495
|
+
}
|
|
5496
|
+
function holdsStringWhere(stored, test) {
|
|
5497
|
+
const matches = (element) => typeof element === "string" && test(element);
|
|
5498
|
+
return Array.isArray(stored) ? stored.some(matches) : matches(stored);
|
|
5499
|
+
}
|
|
5129
5500
|
|
|
5130
5501
|
// src/filters/operators/bitset.ts
|
|
5131
5502
|
function resolveBitset(bs) {
|
|
@@ -5159,7 +5530,7 @@ function applyEqBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
|
5159
5530
|
if (fieldIndex?.type === "enum" && typeof value === "string") {
|
|
5160
5531
|
return fieldIndex.index.getDocIdsBitset(value, capacity);
|
|
5161
5532
|
}
|
|
5162
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => v
|
|
5533
|
+
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsValue(v, value));
|
|
5163
5534
|
}
|
|
5164
5535
|
function applyNeBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5165
5536
|
if (fieldIndex?.type === "numeric" && typeof value === "number") {
|
|
@@ -5175,7 +5546,7 @@ function applyNeBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
|
5175
5546
|
const eq = fieldIndex.index.getDocIdsBitset(value, capacity);
|
|
5176
5547
|
return bitsetAnd(all, bitsetNot(eq, capacity));
|
|
5177
5548
|
}
|
|
5178
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => v
|
|
5549
|
+
return scanToBitset(allDocsBitset, capacity, getValue, (v) => lacksValue(v, value));
|
|
5179
5550
|
}
|
|
5180
5551
|
function applyGtBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5181
5552
|
if (fieldIndex?.type === "numeric") {
|
|
@@ -5212,7 +5583,7 @@ function applyInBitset(values, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
|
5212
5583
|
return fieldIndex.index.getDocIdsInBitset(values, capacity);
|
|
5213
5584
|
}
|
|
5214
5585
|
const valSet = new Set(values);
|
|
5215
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) =>
|
|
5586
|
+
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringIn(v, valSet));
|
|
5216
5587
|
}
|
|
5217
5588
|
function applyNinBitset(values, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5218
5589
|
if (fieldIndex?.type === "enum") {
|
|
@@ -5221,18 +5592,13 @@ function applyNinBitset(values, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
|
5221
5592
|
return bitsetAnd(all, bitsetNot(matched, capacity));
|
|
5222
5593
|
}
|
|
5223
5594
|
const valSet = new Set(values);
|
|
5224
|
-
return scanToBitset(
|
|
5225
|
-
allDocsBitset,
|
|
5226
|
-
capacity,
|
|
5227
|
-
getValue,
|
|
5228
|
-
(v) => v !== void 0 && v !== null && typeof v === "string" && !valSet.has(v)
|
|
5229
|
-
);
|
|
5595
|
+
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsNoStringIn(v, valSet));
|
|
5230
5596
|
}
|
|
5231
5597
|
function applyStartsWithBitset(prefix, allDocsBitset, capacity, getValue) {
|
|
5232
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) =>
|
|
5598
|
+
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringWhere(v, (text) => text.startsWith(prefix)));
|
|
5233
5599
|
}
|
|
5234
5600
|
function applyEndsWithBitset(suffix, allDocsBitset, capacity, getValue) {
|
|
5235
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) =>
|
|
5601
|
+
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringWhere(v, (text) => text.endsWith(suffix)));
|
|
5236
5602
|
}
|
|
5237
5603
|
function applyContainsAllBitset(values, allDocsBitset, capacity, getValue) {
|
|
5238
5604
|
return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
|
|
@@ -5296,6 +5662,7 @@ function applyGeoRadiusBitset(filter, geoIndex, capacity) {
|
|
|
5296
5662
|
return bitsetFromSet(setResult, capacity);
|
|
5297
5663
|
}
|
|
5298
5664
|
function applyGeoPolygonBitset(filter, geoIndex, capacity) {
|
|
5665
|
+
requirePolygonRing(filter.points);
|
|
5299
5666
|
const setResult = geoIndex.polygonQuery(filter.points, filter.inside ?? true);
|
|
5300
5667
|
return bitsetFromSet(setResult, capacity);
|
|
5301
5668
|
}
|
|
@@ -5303,7 +5670,7 @@ function applyGeoPolygonBitset(filter, geoIndex, capacity) {
|
|
|
5303
5670
|
// src/filters/evaluator.ts
|
|
5304
5671
|
var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
|
|
5305
5672
|
var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
|
|
5306
|
-
function
|
|
5673
|
+
function requireKnownKeys2(value, known, describe) {
|
|
5307
5674
|
for (const key of Object.keys(value)) {
|
|
5308
5675
|
if (!known.has(key)) {
|
|
5309
5676
|
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, describe(key), { key });
|
|
@@ -5311,7 +5678,7 @@ function requireKnownKeys(value, known, describe) {
|
|
|
5311
5678
|
}
|
|
5312
5679
|
}
|
|
5313
5680
|
function evaluateFilters(expression, context) {
|
|
5314
|
-
|
|
5681
|
+
requireKnownKeys2(
|
|
5315
5682
|
expression,
|
|
5316
5683
|
EXPRESSION_KEYS,
|
|
5317
5684
|
(key) => `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`
|
|
@@ -5340,14 +5707,15 @@ function evaluateFilters(expression, context) {
|
|
|
5340
5707
|
return applyAndBitset(bitsets);
|
|
5341
5708
|
}
|
|
5342
5709
|
function evaluateFieldFilter(fieldPath, filter, context) {
|
|
5343
|
-
|
|
5710
|
+
requireKnownKeys2(
|
|
5344
5711
|
filter,
|
|
5345
5712
|
OPERATOR_KEYS,
|
|
5346
5713
|
(key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
|
|
5347
5714
|
);
|
|
5715
|
+
const f = filter;
|
|
5716
|
+
requireValidOperands(fieldPath, f, context.fieldTypes[fieldPath]);
|
|
5348
5717
|
const fieldIndex = context.fieldIndexes[fieldPath];
|
|
5349
5718
|
const getValue = (internalId) => context.getFieldValue(internalId, fieldPath);
|
|
5350
|
-
const f = filter;
|
|
5351
5719
|
const bitsets = [];
|
|
5352
5720
|
const { capacity } = context;
|
|
5353
5721
|
let allDocsBitset = null;
|
|
@@ -5519,6 +5887,7 @@ function buildFilterContext(state, schema) {
|
|
|
5519
5887
|
let cachedAllDocIds = null;
|
|
5520
5888
|
let cachedAllDocIdsBitset = null;
|
|
5521
5889
|
return {
|
|
5890
|
+
fieldTypes: flat,
|
|
5522
5891
|
fieldIndexes,
|
|
5523
5892
|
getFieldValue: (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath),
|
|
5524
5893
|
get allDocIds() {
|
|
@@ -5840,8 +6209,7 @@ function createValueStore(initialKind) {
|
|
|
5840
6209
|
get kind() {
|
|
5841
6210
|
return kind;
|
|
5842
6211
|
},
|
|
5843
|
-
set(internalId,
|
|
5844
|
-
const comparable = toComparableSortValue(value);
|
|
6212
|
+
set(internalId, comparable) {
|
|
5845
6213
|
if (kind === "number") {
|
|
5846
6214
|
if (comparable === null) {
|
|
5847
6215
|
setNumber(internalId, Number.NaN);
|
|
@@ -5911,6 +6279,12 @@ function createValueStore(initialKind) {
|
|
|
5911
6279
|
}
|
|
5912
6280
|
|
|
5913
6281
|
// src/core/partition/sort-columns/index.ts
|
|
6282
|
+
function mayHoldList(fieldType) {
|
|
6283
|
+
return fieldType === void 0 || fieldType.endsWith("[]");
|
|
6284
|
+
}
|
|
6285
|
+
function columnKeyOf(field, fieldType, mode) {
|
|
6286
|
+
return mayHoldList(fieldType) ? `${mode}:${field}` : `value:${field}`;
|
|
6287
|
+
}
|
|
5914
6288
|
function liveInternalIds(docStore) {
|
|
5915
6289
|
const ids = [];
|
|
5916
6290
|
for (const internalId of docStore.allInternalIds()) ids.push(internalId);
|
|
@@ -5928,21 +6302,22 @@ function createSortColumnSet(docStore) {
|
|
|
5928
6302
|
entry.dirty.clear();
|
|
5929
6303
|
entry.dirtyStream = null;
|
|
5930
6304
|
}
|
|
5931
|
-
function backfill(field, fieldType) {
|
|
6305
|
+
function backfill(key, field, fieldType, mode) {
|
|
5932
6306
|
const store = createValueStore(kindForFieldType(fieldType));
|
|
5933
6307
|
for (const [docId, stored] of docStore.all()) {
|
|
5934
6308
|
const internalId = docStore.getInternalId(docId);
|
|
5935
6309
|
if (internalId === void 0) continue;
|
|
5936
|
-
store.set(internalId, readSortField(stored.fields, field));
|
|
6310
|
+
store.set(internalId, toReducedSortValue(readSortField(stored.fields, field), mode));
|
|
5937
6311
|
}
|
|
5938
6312
|
const entry = {
|
|
5939
6313
|
field,
|
|
6314
|
+
mode,
|
|
5940
6315
|
store,
|
|
5941
6316
|
order: buildOrder(store, liveInternalIds(docStore), docStore.internalIdCapacity()),
|
|
5942
6317
|
dirty: /* @__PURE__ */ new Set(),
|
|
5943
6318
|
dirtyStream: null
|
|
5944
6319
|
};
|
|
5945
|
-
columns.set(
|
|
6320
|
+
columns.set(key, entry);
|
|
5946
6321
|
return entry;
|
|
5947
6322
|
}
|
|
5948
6323
|
function buildDirtyStream(entry) {
|
|
@@ -5995,13 +6370,14 @@ function createSortColumnSet(docStore) {
|
|
|
5995
6370
|
};
|
|
5996
6371
|
}
|
|
5997
6372
|
return {
|
|
5998
|
-
holds(field) {
|
|
5999
|
-
return columns.has(field);
|
|
6373
|
+
holds(field, fieldType, mode) {
|
|
6374
|
+
return columns.has(columnKeyOf(field, fieldType, mode));
|
|
6000
6375
|
},
|
|
6001
|
-
column(field, fieldType) {
|
|
6002
|
-
|
|
6376
|
+
column(field, fieldType, mode) {
|
|
6377
|
+
const key = columnKeyOf(field, fieldType, mode);
|
|
6378
|
+
let entry = columns.get(key);
|
|
6003
6379
|
if (entry === void 0) {
|
|
6004
|
-
entry = backfill(field, fieldType);
|
|
6380
|
+
entry = backfill(key, field, fieldType, mode);
|
|
6005
6381
|
} else if (entry.dirty.size > rebuildThreshold()) {
|
|
6006
6382
|
rebuild2(entry);
|
|
6007
6383
|
}
|
|
@@ -6009,7 +6385,7 @@ function createSortColumnSet(docStore) {
|
|
|
6009
6385
|
},
|
|
6010
6386
|
record(internalId, document2) {
|
|
6011
6387
|
for (const entry of columns.values()) {
|
|
6012
|
-
entry.store.set(internalId, readSortField(document2, entry.field));
|
|
6388
|
+
entry.store.set(internalId, toReducedSortValue(readSortField(document2, entry.field), entry.mode));
|
|
6013
6389
|
entry.dirty.add(internalId);
|
|
6014
6390
|
entry.dirtyStream = null;
|
|
6015
6391
|
}
|
|
@@ -6164,11 +6540,13 @@ function createStream(column, direction, anchorRank) {
|
|
|
6164
6540
|
};
|
|
6165
6541
|
}
|
|
6166
6542
|
function selectSortedPage(state, request) {
|
|
6167
|
-
const { fields, directions, fieldTypes, limit, anchorKey, anchorId, matches } = request;
|
|
6543
|
+
const { fields, directions, fieldTypes, modes, limit, anchorKey, anchorId, matches } = request;
|
|
6168
6544
|
if (limit <= 0 || fields.length === 0) return [];
|
|
6169
6545
|
const set = state.sortColumns;
|
|
6170
6546
|
if (set === null) return [];
|
|
6171
|
-
const columns = fields.map(
|
|
6547
|
+
const columns = fields.map(
|
|
6548
|
+
(field, index) => set.column(field, fieldTypes[index], modes[index] ?? defaultSortMode(directions[index] ?? "asc"))
|
|
6549
|
+
);
|
|
6172
6550
|
const resolver = state.docStore.resolver();
|
|
6173
6551
|
const leading = columns[0];
|
|
6174
6552
|
const leadingDirection = directions[0];
|
|
@@ -6210,17 +6588,18 @@ function sortedPageOf(state, request) {
|
|
|
6210
6588
|
ensureColumns(state);
|
|
6211
6589
|
return selectSortedPage(state, request);
|
|
6212
6590
|
}
|
|
6213
|
-
function sortValuesOf(state, docId, fields, fieldTypes) {
|
|
6591
|
+
function sortValuesOf(state, docId, fields, fieldTypes, modes) {
|
|
6214
6592
|
const set = state.sortColumns;
|
|
6215
6593
|
const internalId = state.docStore.getInternalId(docId);
|
|
6216
6594
|
const stored = state.docStore.get(docId);
|
|
6217
6595
|
const key = new Array(fields.length);
|
|
6218
6596
|
for (let i = 0; i < fields.length; i++) {
|
|
6219
|
-
|
|
6220
|
-
|
|
6597
|
+
const mode = modes[i] ?? "min";
|
|
6598
|
+
if (set !== null && internalId !== void 0 && set.holds(fields[i], fieldTypes[i], mode)) {
|
|
6599
|
+
key[i] = set.column(fields[i], fieldTypes[i], mode).valueOf(internalId);
|
|
6221
6600
|
continue;
|
|
6222
6601
|
}
|
|
6223
|
-
key[i] = stored === void 0 ? null :
|
|
6602
|
+
key[i] = stored === void 0 ? null : toReducedSortValue(readSortField(stored.fields, fields[i]), mode);
|
|
6224
6603
|
}
|
|
6225
6604
|
return key;
|
|
6226
6605
|
}
|
|
@@ -8251,8 +8630,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8251
8630
|
sortedPage(request) {
|
|
8252
8631
|
return sortedPageOf(state, request);
|
|
8253
8632
|
},
|
|
8254
|
-
sortValues(docId, fields, fieldTypes) {
|
|
8255
|
-
return sortValuesOf(state, docId, fields, fieldTypes);
|
|
8633
|
+
sortValues(docId, fields, fieldTypes, modes) {
|
|
8634
|
+
return sortValuesOf(state, docId, fields, fieldTypes, modes);
|
|
8256
8635
|
},
|
|
8257
8636
|
applyFilters(filters, schema) {
|
|
8258
8637
|
return applyPartitionFilters(state, filters, schema);
|
|
@@ -8287,6 +8666,18 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8287
8666
|
return partition;
|
|
8288
8667
|
}
|
|
8289
8668
|
|
|
8669
|
+
// src/engine/orchestration/turn.ts
|
|
8670
|
+
function afterCurrentTurn(callback) {
|
|
8671
|
+
if (typeof setImmediate === "function") {
|
|
8672
|
+
setImmediate(callback);
|
|
8673
|
+
return;
|
|
8674
|
+
}
|
|
8675
|
+
setTimeout(callback, 0);
|
|
8676
|
+
}
|
|
8677
|
+
function yieldToEventLoop() {
|
|
8678
|
+
return new Promise((resolve) => afterCurrentTurn(resolve));
|
|
8679
|
+
}
|
|
8680
|
+
|
|
8290
8681
|
// src/partitioning/constants.ts
|
|
8291
8682
|
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
8292
8683
|
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
@@ -8361,7 +8752,7 @@ function createRebalancer() {
|
|
|
8361
8752
|
documentsProcessed,
|
|
8362
8753
|
documentsTotal
|
|
8363
8754
|
});
|
|
8364
|
-
await
|
|
8755
|
+
await yieldToEventLoop();
|
|
8365
8756
|
}
|
|
8366
8757
|
}
|
|
8367
8758
|
}
|
|
@@ -8371,7 +8762,7 @@ function createRebalancer() {
|
|
|
8371
8762
|
documentsProcessed,
|
|
8372
8763
|
documentsTotal
|
|
8373
8764
|
});
|
|
8374
|
-
await
|
|
8765
|
+
await yieldToEventLoop();
|
|
8375
8766
|
}
|
|
8376
8767
|
onProgress?.({
|
|
8377
8768
|
phase: "swapping",
|
|
@@ -8442,13 +8833,43 @@ function createPartitionRouter() {
|
|
|
8442
8833
|
}
|
|
8443
8834
|
|
|
8444
8835
|
// src/plugins/registry.ts
|
|
8836
|
+
var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
|
|
8445
8837
|
function isThenable(value) {
|
|
8446
8838
|
return value !== null && typeof value === "object" && typeof value.then === "function";
|
|
8447
8839
|
}
|
|
8448
|
-
|
|
8840
|
+
function thrownText(err, plugin, hookName) {
|
|
8841
|
+
if (err instanceof Error) return err.message;
|
|
8842
|
+
try {
|
|
8843
|
+
return String(err);
|
|
8844
|
+
} catch {
|
|
8845
|
+
return `The ${hookName} hook of plugin "${plugin}" rejects the write`;
|
|
8846
|
+
}
|
|
8847
|
+
}
|
|
8848
|
+
function asWriteRejection(err, plugin, hookName) {
|
|
8849
|
+
if (err instanceof NarsilError) return err;
|
|
8850
|
+
return new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, thrownText(err, plugin, hookName), {
|
|
8851
|
+
plugin,
|
|
8852
|
+
hook: hookName
|
|
8853
|
+
});
|
|
8854
|
+
}
|
|
8855
|
+
function invoke(hook, hookName, context) {
|
|
8856
|
+
const rejectsWrites = WRITE_REJECTING_HOOKS.has(hookName);
|
|
8857
|
+
let result;
|
|
8858
|
+
try {
|
|
8859
|
+
result = hook.run(context);
|
|
8860
|
+
} catch (err) {
|
|
8861
|
+
throw rejectsWrites ? asWriteRejection(err, hook.plugin, hookName) : err;
|
|
8862
|
+
}
|
|
8863
|
+
if (!isThenable(result)) return null;
|
|
8864
|
+
if (!rejectsWrites) return result;
|
|
8865
|
+
return Promise.resolve(result).catch((err) => {
|
|
8866
|
+
throw asWriteRejection(err, hook.plugin, hookName);
|
|
8867
|
+
});
|
|
8868
|
+
}
|
|
8869
|
+
async function continueAsync(pending, hooks, startIndex, hookName, context) {
|
|
8449
8870
|
await pending;
|
|
8450
8871
|
for (let i = startIndex; i < hooks.length; i++) {
|
|
8451
|
-
await hooks[i]
|
|
8872
|
+
await invoke(hooks[i], hookName, context);
|
|
8452
8873
|
}
|
|
8453
8874
|
}
|
|
8454
8875
|
function createPluginRegistry() {
|
|
@@ -8465,14 +8886,14 @@ function createPluginRegistry() {
|
|
|
8465
8886
|
for (const plugin of plugins) {
|
|
8466
8887
|
const hook = plugin[hookName];
|
|
8467
8888
|
if (typeof hook === "function") {
|
|
8468
|
-
hooks.push(hook.bind(plugin));
|
|
8889
|
+
hooks.push({ plugin: plugin.name, run: hook.bind(plugin) });
|
|
8469
8890
|
}
|
|
8470
8891
|
}
|
|
8471
8892
|
if (hooks.length === 0) return;
|
|
8472
8893
|
for (let i = 0; i < hooks.length; i++) {
|
|
8473
|
-
const
|
|
8474
|
-
if (
|
|
8475
|
-
return continueAsync(
|
|
8894
|
+
const pending = invoke(hooks[i], hookName, context);
|
|
8895
|
+
if (pending !== null) {
|
|
8896
|
+
return continueAsync(pending, hooks, i + 1, hookName, context);
|
|
8476
8897
|
}
|
|
8477
8898
|
}
|
|
8478
8899
|
}
|
|
@@ -8816,8 +9237,35 @@ function heapPop(heap) {
|
|
|
8816
9237
|
return top;
|
|
8817
9238
|
}
|
|
8818
9239
|
|
|
9240
|
+
// src/search/oversample.ts
|
|
9241
|
+
function oversampledShardSize(size) {
|
|
9242
|
+
return Math.ceil(size * 1.5) + 10;
|
|
9243
|
+
}
|
|
9244
|
+
|
|
8819
9245
|
// src/search/facets.ts
|
|
8820
|
-
function
|
|
9246
|
+
function keptValueCount(fieldConfig) {
|
|
9247
|
+
return keptFacetValues(fieldConfig?.limit);
|
|
9248
|
+
}
|
|
9249
|
+
function everyValueFacetConfig(config) {
|
|
9250
|
+
const widened = {};
|
|
9251
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
9252
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
9253
|
+
}
|
|
9254
|
+
return widened;
|
|
9255
|
+
}
|
|
9256
|
+
function oversampledFacetConfig(config) {
|
|
9257
|
+
const widened = {};
|
|
9258
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
9259
|
+
if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
|
|
9260
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
9261
|
+
continue;
|
|
9262
|
+
}
|
|
9263
|
+
const kept = keptValueCount(fieldConfig);
|
|
9264
|
+
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
9265
|
+
}
|
|
9266
|
+
return widened;
|
|
9267
|
+
}
|
|
9268
|
+
function mergeFacets(partitionFacets, config) {
|
|
8821
9269
|
const merged = /* @__PURE__ */ new Map();
|
|
8822
9270
|
const bounds = /* @__PURE__ */ new Map();
|
|
8823
9271
|
for (const partition of partitionFacets) {
|
|
@@ -8835,11 +9283,21 @@ function mergeFacets(partitionFacets) {
|
|
|
8835
9283
|
}
|
|
8836
9284
|
const result = {};
|
|
8837
9285
|
for (const [field, valueMap] of merged) {
|
|
9286
|
+
const fieldConfig = config[field];
|
|
9287
|
+
const ascending = fieldConfig?.sort === "asc";
|
|
9288
|
+
const ordered = Array.from(valueMap.entries());
|
|
9289
|
+
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
9290
|
+
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
9291
|
+
let largestLeftOut = 0;
|
|
9292
|
+
for (let index = kept; index < ordered.length; index++) {
|
|
9293
|
+
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
9294
|
+
}
|
|
9295
|
+
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
8838
9296
|
const values = {};
|
|
8839
|
-
for (
|
|
8840
|
-
values[
|
|
9297
|
+
for (let index = 0; index < kept; index++) {
|
|
9298
|
+
values[ordered[index][0]] = ordered[index][1];
|
|
8841
9299
|
}
|
|
8842
|
-
result[field] = { values, count:
|
|
9300
|
+
result[field] = { values, count: kept, errorBound };
|
|
8843
9301
|
}
|
|
8844
9302
|
return result;
|
|
8845
9303
|
}
|
|
@@ -8852,6 +9310,13 @@ var MAX_SORT_FIELDS = 8;
|
|
|
8852
9310
|
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
8853
9311
|
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
8854
9312
|
var PREFIX_MAX_EXPANSIONS = 50;
|
|
9313
|
+
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
9314
|
+
var DEFAULT_MAX_PER_GROUP = 1;
|
|
9315
|
+
var DEFAULT_RANK_CONSTANT = 60;
|
|
9316
|
+
var MIN_RANK_CONSTANT = 1;
|
|
9317
|
+
var DEFAULT_LINEAR_ALPHA = 0.5;
|
|
9318
|
+
var MIN_LINEAR_ALPHA = 0;
|
|
9319
|
+
var MAX_LINEAR_ALPHA = 1;
|
|
8855
9320
|
|
|
8856
9321
|
// src/search/cursor-codec.ts
|
|
8857
9322
|
function encodeCursorText(text) {
|
|
@@ -8876,21 +9341,66 @@ function isFieldList(sort) {
|
|
|
8876
9341
|
function normalizeSort(sort) {
|
|
8877
9342
|
if (sort === void 0) return [];
|
|
8878
9343
|
if (isFieldList(sort)) {
|
|
8879
|
-
return sort.map(
|
|
9344
|
+
return sort.map(
|
|
9345
|
+
(entry) => entry.mode === void 0 ? { field: entry.field, direction: entry.direction } : { field: entry.field, direction: entry.direction, mode: entry.mode }
|
|
9346
|
+
);
|
|
8880
9347
|
}
|
|
8881
9348
|
return Object.entries(sort).map(([field, direction]) => ({ field, direction }));
|
|
8882
9349
|
}
|
|
8883
|
-
function
|
|
9350
|
+
function sortModesOf(fields) {
|
|
9351
|
+
return fields.map(sortModeOf);
|
|
9352
|
+
}
|
|
9353
|
+
function sortSignatureEntry(entry) {
|
|
9354
|
+
const mode = sortModeOf(entry);
|
|
9355
|
+
return mode === defaultSortMode(entry.direction) ? [entry.field, entry.direction] : [entry.field, entry.direction, mode];
|
|
9356
|
+
}
|
|
9357
|
+
function requireSortMode(entry, fieldType) {
|
|
9358
|
+
if (entry.mode === void 0) return;
|
|
9359
|
+
if (!isSortMode(entry.mode)) {
|
|
9360
|
+
throw new NarsilError(
|
|
9361
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
9362
|
+
`A sort mode is one of ${SORT_MODES.map((mode) => `"${mode}"`).join(", ")}, and the mode on "${entry.field}" is "${String(entry.mode)}"`,
|
|
9363
|
+
{ field: entry.field, mode: String(entry.mode) }
|
|
9364
|
+
);
|
|
9365
|
+
}
|
|
9366
|
+
const averages = entry.mode === "avg" || entry.mode === "median";
|
|
9367
|
+
if (averages && fieldType !== void 0 && fieldType !== "number" && fieldType !== "number[]") {
|
|
9368
|
+
throw new NarsilError(
|
|
9369
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
9370
|
+
`The engine applies the "${entry.mode}" mode to a number field alone, and "${entry.field}" is a ${fieldType} field`,
|
|
9371
|
+
{ field: entry.field, fieldType, mode: entry.mode }
|
|
9372
|
+
);
|
|
9373
|
+
}
|
|
9374
|
+
}
|
|
9375
|
+
function requireSortableFields(sort, schema, strict = false) {
|
|
8884
9376
|
const fields = normalizeSort(sort);
|
|
8885
9377
|
if (fields.length === 0) return;
|
|
8886
9378
|
const flatSchema = flattenSchema(schema);
|
|
8887
9379
|
for (const entry of fields) {
|
|
8888
|
-
|
|
8889
|
-
throw
|
|
8890
|
-
|
|
8891
|
-
|
|
8892
|
-
|
|
8893
|
-
|
|
9380
|
+
const fieldType = flatSchema[entry.field];
|
|
9381
|
+
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(entry.field, "sort");
|
|
9382
|
+
if (entry.direction !== "asc" && entry.direction !== "desc") {
|
|
9383
|
+
throw new NarsilError(
|
|
9384
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
9385
|
+
`A sort direction is "asc" or "desc", and the direction on "${entry.field}" is "${String(entry.direction)}"`,
|
|
9386
|
+
{ field: entry.field, direction: String(entry.direction) }
|
|
9387
|
+
);
|
|
9388
|
+
}
|
|
9389
|
+
requireSortMode(entry, fieldType);
|
|
9390
|
+
if (fieldType === "string") {
|
|
9391
|
+
throw new NarsilError(
|
|
9392
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
9393
|
+
`The engine sorts by text field "${entry.field}" only where the schema declares it "${SORTABLE_TEXT_FIELD_TYPE}", because ordering text takes far more memory per document than ordering a number`,
|
|
9394
|
+
{ field: entry.field, fieldType }
|
|
9395
|
+
);
|
|
9396
|
+
}
|
|
9397
|
+
if (isGeopointOrVectorType(fieldType)) {
|
|
9398
|
+
throw new NarsilError(
|
|
9399
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
9400
|
+
`The engine sorts by a number, boolean, enum, or sortable text field, and "${entry.field}" is a ${fieldType} field`,
|
|
9401
|
+
{ field: entry.field, fieldType }
|
|
9402
|
+
);
|
|
9403
|
+
}
|
|
8894
9404
|
}
|
|
8895
9405
|
}
|
|
8896
9406
|
function applySorting(hits, sort, sortKeyOf) {
|
|
@@ -8909,7 +9419,7 @@ function applySorting(hits, sort, sortKeyOf) {
|
|
|
8909
9419
|
}
|
|
8910
9420
|
|
|
8911
9421
|
// src/search/cursor.ts
|
|
8912
|
-
var CURSOR_VERSION =
|
|
9422
|
+
var CURSOR_VERSION = 4;
|
|
8913
9423
|
function invalidCursor(cursor, reason) {
|
|
8914
9424
|
const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
|
|
8915
9425
|
return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
|
|
@@ -8949,7 +9459,7 @@ function sortSignatureOf(sort) {
|
|
|
8949
9459
|
);
|
|
8950
9460
|
}
|
|
8951
9461
|
}
|
|
8952
|
-
return JSON.stringify(fields.map(
|
|
9462
|
+
return JSON.stringify(fields.map(sortSignatureEntry));
|
|
8953
9463
|
}
|
|
8954
9464
|
function encodePageCursor(cursor) {
|
|
8955
9465
|
const payload = { v: CURSOR_VERSION, a: cursor.anchor };
|
|
@@ -8959,6 +9469,7 @@ function encodePageCursor(cursor) {
|
|
|
8959
9469
|
} else if (cursor.score !== null) {
|
|
8960
9470
|
payload.s = cursor.score;
|
|
8961
9471
|
}
|
|
9472
|
+
if (cursor.depth !== void 0 && cursor.depth > 0) payload.d = cursor.depth;
|
|
8962
9473
|
payload.q = cursor.binding;
|
|
8963
9474
|
return encodeCursorText(JSON.stringify(payload));
|
|
8964
9475
|
}
|
|
@@ -9004,7 +9515,7 @@ function decodePageCursor(cursor) {
|
|
|
9004
9515
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
9005
9516
|
throw invalidCursor(cursor, "expected an object");
|
|
9006
9517
|
}
|
|
9007
|
-
const { v, a, s, k, o, q } = parsed;
|
|
9518
|
+
const { v, a, s, k, o, d, q } = parsed;
|
|
9008
9519
|
if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
|
|
9009
9520
|
if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
|
|
9010
9521
|
throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
|
|
@@ -9017,6 +9528,9 @@ function decodePageCursor(cursor) {
|
|
|
9017
9528
|
throw invalidCursor(cursor, '"s" must be a finite number');
|
|
9018
9529
|
}
|
|
9019
9530
|
if (s !== void 0 && k !== void 0) throw invalidCursor(cursor, 'a cursor carries "s" or "k", never both');
|
|
9531
|
+
if (d !== void 0 && (typeof d !== "number" || !Number.isSafeInteger(d) || d < 1)) {
|
|
9532
|
+
throw invalidCursor(cursor, '"d" must be a positive integer');
|
|
9533
|
+
}
|
|
9020
9534
|
const sortKey = k === void 0 ? null : decodeSortKey(k, cursor);
|
|
9021
9535
|
if (o !== void 0 && typeof o !== "string") throw invalidCursor(cursor, '"o" must be a string');
|
|
9022
9536
|
if (sortKey === null !== (o === void 0)) {
|
|
@@ -9027,6 +9541,7 @@ function decodePageCursor(cursor) {
|
|
|
9027
9541
|
score: typeof s === "number" ? s : null,
|
|
9028
9542
|
sortKey,
|
|
9029
9543
|
sortSignature: typeof o === "string" ? o : null,
|
|
9544
|
+
depth: typeof d === "number" ? d : 0,
|
|
9030
9545
|
binding: q
|
|
9031
9546
|
};
|
|
9032
9547
|
}
|
|
@@ -9104,18 +9619,21 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
|
|
|
9104
9619
|
let nextCursor;
|
|
9105
9620
|
const hasMore = afterOffset + limit < results.length;
|
|
9106
9621
|
const lastResult = lastOrganicResult(sliced, pinnedIds);
|
|
9622
|
+
const depth = afterOffset + sliced.length;
|
|
9107
9623
|
if (hasMore && lastResult !== void 0) {
|
|
9108
9624
|
nextCursor = sort !== void 0 ? encodePageCursor({
|
|
9109
9625
|
anchor: lastResult.id,
|
|
9110
9626
|
score: null,
|
|
9111
9627
|
sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
|
|
9112
9628
|
sortSignature: sort.signature,
|
|
9629
|
+
depth,
|
|
9113
9630
|
binding
|
|
9114
9631
|
}) : encodePageCursor({
|
|
9115
9632
|
anchor: lastResult.id,
|
|
9116
9633
|
score: lastResult.score ?? null,
|
|
9117
9634
|
sortKey: null,
|
|
9118
9635
|
sortSignature: null,
|
|
9636
|
+
depth,
|
|
9119
9637
|
binding
|
|
9120
9638
|
});
|
|
9121
9639
|
}
|
|
@@ -9187,7 +9705,7 @@ function fulltextMatches(partition, params, language, schema, options) {
|
|
|
9187
9705
|
prefixExpansion: prepared.prefixExpansion,
|
|
9188
9706
|
fields: params.fields,
|
|
9189
9707
|
tolerance: params.tolerance ?? 0,
|
|
9190
|
-
prefixLength: params.prefixLength ??
|
|
9708
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
9191
9709
|
exact: params.exact ?? false,
|
|
9192
9710
|
filterBitset: prepared.filterBitset
|
|
9193
9711
|
});
|
|
@@ -9208,7 +9726,7 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
9208
9726
|
fields: params.fields,
|
|
9209
9727
|
boost: params.boost,
|
|
9210
9728
|
tolerance: params.tolerance ?? 0,
|
|
9211
|
-
prefixLength: params.prefixLength ??
|
|
9729
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
9212
9730
|
exact: params.exact ?? false,
|
|
9213
9731
|
bm25Params: options?.bm25Params,
|
|
9214
9732
|
globalStats: options?.globalStats,
|
|
@@ -9368,20 +9886,21 @@ async function fanOutQuery(manager, params, language, schema, config, searchOpti
|
|
|
9368
9886
|
if (config.dispatcher) {
|
|
9369
9887
|
outcomes = await dispatchWithDispatcher(partitions, params, language, schema, options, config.dispatcher);
|
|
9370
9888
|
if (params.facets) {
|
|
9371
|
-
facets = collectAndMergeFacets(outcomes, params, schema);
|
|
9889
|
+
facets = collectAndMergeFacets(outcomes, params.facets, schema);
|
|
9372
9890
|
}
|
|
9373
9891
|
} else {
|
|
9374
9892
|
outcomes = [];
|
|
9375
9893
|
const partitionFacets = [];
|
|
9894
|
+
const everyValue = params.facets ? everyValueFacetConfig(params.facets) : void 0;
|
|
9376
9895
|
for (const partition of partitions) {
|
|
9377
9896
|
const result = dispatchSinglePartition(partition, params, language, schema, options);
|
|
9378
|
-
if (
|
|
9379
|
-
partitionFacets.push(partition.computeFacets(facetMatchSetOf(result),
|
|
9897
|
+
if (everyValue) {
|
|
9898
|
+
partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), everyValue, schema));
|
|
9380
9899
|
}
|
|
9381
9900
|
outcomes.push({ result, partition });
|
|
9382
9901
|
}
|
|
9383
9902
|
if (params.facets) {
|
|
9384
|
-
facets = mergeFacets(partitionFacets);
|
|
9903
|
+
facets = mergeFacets(partitionFacets, params.facets);
|
|
9385
9904
|
}
|
|
9386
9905
|
}
|
|
9387
9906
|
const allScoredArrays = outcomes.map((o) => o.result.scored);
|
|
@@ -9419,14 +9938,13 @@ function dispatchWithDispatcher(partitions, params, language, schema, options, d
|
|
|
9419
9938
|
function dispatchSinglePartition(partition, params, language, schema, options) {
|
|
9420
9939
|
return fulltextSearch(partition, params, language, schema, options);
|
|
9421
9940
|
}
|
|
9422
|
-
function collectAndMergeFacets(outcomes,
|
|
9941
|
+
function collectAndMergeFacets(outcomes, facetConfig, schema) {
|
|
9942
|
+
const everyValue = everyValueFacetConfig(facetConfig);
|
|
9423
9943
|
const partitionFacets = [];
|
|
9424
9944
|
for (const outcome of outcomes) {
|
|
9425
|
-
|
|
9426
|
-
const facetResult = outcome.partition.computeFacets(facetMatchSetOf(outcome.result), params.facets, schema);
|
|
9427
|
-
partitionFacets.push(facetResult);
|
|
9945
|
+
partitionFacets.push(outcome.partition.computeFacets(facetMatchSetOf(outcome.result), everyValue, schema));
|
|
9428
9946
|
}
|
|
9429
|
-
return mergeFacets(partitionFacets);
|
|
9947
|
+
return mergeFacets(partitionFacets, facetConfig);
|
|
9430
9948
|
}
|
|
9431
9949
|
|
|
9432
9950
|
// src/core/partition/frozen/memory.ts
|
|
@@ -9786,10 +10304,10 @@ function compositeSortedPage(subs, layout, request) {
|
|
|
9786
10304
|
if (merged.length > request.limit) merged.length = request.limit;
|
|
9787
10305
|
return merged;
|
|
9788
10306
|
}
|
|
9789
|
-
function compositeSortValues(subs, docId, fields, fieldTypes) {
|
|
10307
|
+
function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
|
|
9790
10308
|
for (const sub of subs) {
|
|
9791
10309
|
if (sub.docStore.has(docId)) {
|
|
9792
|
-
return sortValuesOf(sub, docId, fields, fieldTypes);
|
|
10310
|
+
return sortValuesOf(sub, docId, fields, fieldTypes, modes);
|
|
9793
10311
|
}
|
|
9794
10312
|
}
|
|
9795
10313
|
return fields.map(() => null);
|
|
@@ -10014,6 +10532,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10014
10532
|
return compositeFilterMatches(subs(), layout(), filters, schema);
|
|
10015
10533
|
},
|
|
10016
10534
|
computeFacets(matched, config, schema) {
|
|
10535
|
+
const everyValue = everyValueFacetConfig(config);
|
|
10017
10536
|
if ("ordinalBitset" in matched) {
|
|
10018
10537
|
const ordinalLayout = layout();
|
|
10019
10538
|
return mergeFacets(
|
|
@@ -10021,19 +10540,23 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10021
10540
|
(sub, index) => computeFacets(
|
|
10022
10541
|
sub,
|
|
10023
10542
|
{ ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
|
|
10024
|
-
|
|
10543
|
+
everyValue,
|
|
10025
10544
|
schema
|
|
10026
10545
|
)
|
|
10027
|
-
)
|
|
10546
|
+
),
|
|
10547
|
+
config
|
|
10028
10548
|
);
|
|
10029
10549
|
}
|
|
10030
|
-
return mergeFacets(
|
|
10550
|
+
return mergeFacets(
|
|
10551
|
+
subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
|
|
10552
|
+
config
|
|
10553
|
+
);
|
|
10031
10554
|
},
|
|
10032
10555
|
sortedPage(request) {
|
|
10033
10556
|
return compositeSortedPage(subs(), layout(), request);
|
|
10034
10557
|
},
|
|
10035
|
-
sortValues(docId, fields, fieldTypes) {
|
|
10036
|
-
return compositeSortValues(subs(), docId, fields, fieldTypes);
|
|
10558
|
+
sortValues(docId, fields, fieldTypes, modes) {
|
|
10559
|
+
return compositeSortValues(subs(), docId, fields, fieldTypes, modes);
|
|
10037
10560
|
},
|
|
10038
10561
|
suggestTerms(surfacePrefix, stemmedPrefix, limit) {
|
|
10039
10562
|
return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
|
|
@@ -10337,10 +10860,10 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
10337
10860
|
if (pid === void 0) return void 0;
|
|
10338
10861
|
return partitions[pid].getRef(docId);
|
|
10339
10862
|
},
|
|
10340
|
-
sortValues(docId, fields, fieldTypes) {
|
|
10863
|
+
sortValues(docId, fields, fieldTypes, modes) {
|
|
10341
10864
|
const pid = locateDocument(docId);
|
|
10342
10865
|
if (pid === void 0) return fields.map(() => null);
|
|
10343
|
-
return partitions[pid].sortValues(docId, fields, fieldTypes);
|
|
10866
|
+
return partitions[pid].sortValues(docId, fields, fieldTypes, modes);
|
|
10344
10867
|
},
|
|
10345
10868
|
has(docId) {
|
|
10346
10869
|
return locateDocument(docId) !== void 0;
|
|
@@ -14487,7 +15010,7 @@ function graphNeedsRebuild(state) {
|
|
|
14487
15010
|
if (held === 0) return false;
|
|
14488
15011
|
return removed / held > REBUILD_REMOVED_RATIO;
|
|
14489
15012
|
}
|
|
14490
|
-
function
|
|
15013
|
+
function yieldToEventLoop2() {
|
|
14491
15014
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
14492
15015
|
}
|
|
14493
15016
|
function* allLiveDocIds(state) {
|
|
@@ -14599,6 +15122,9 @@ function splitWorkerBudget(total) {
|
|
|
14599
15122
|
async function spawnNodeWorker(_entryPoint, _resourceLimits) {
|
|
14600
15123
|
return null;
|
|
14601
15124
|
}
|
|
15125
|
+
async function nodeThreadId() {
|
|
15126
|
+
return 0;
|
|
15127
|
+
}
|
|
14602
15128
|
async function isNodeMainThread() {
|
|
14603
15129
|
return true;
|
|
14604
15130
|
}
|
|
@@ -14637,6 +15163,21 @@ function detectRuntime() {
|
|
|
14637
15163
|
};
|
|
14638
15164
|
}
|
|
14639
15165
|
|
|
15166
|
+
// src/workers/entry-point.ts
|
|
15167
|
+
var DIST_SEGMENT = "/dist/";
|
|
15168
|
+
function resolveWorkerEntry(moduleUrl, sourceDirectory, distEntry) {
|
|
15169
|
+
const distIndex = moduleUrl.lastIndexOf(DIST_SEGMENT);
|
|
15170
|
+
const entry = distIndex !== -1 ? new URL(distEntry, moduleUrl.slice(0, distIndex + DIST_SEGMENT.length)).href : sourceDirectory.test(moduleUrl) ? moduleUrl.replace(sourceDirectory, `${DIST_SEGMENT}${distEntry}`) : null;
|
|
15171
|
+
return entry === moduleUrl ? null : entry;
|
|
15172
|
+
}
|
|
15173
|
+
function missingWorkerEntry(moduleUrl) {
|
|
15174
|
+
return new NarsilError(
|
|
15175
|
+
ErrorCodes.CONFIG_INVALID,
|
|
15176
|
+
`The engine finds no worker entry beside its own module at "${moduleUrl}", which happens where a bundler folds @delali/narsil into an application bundle. Keep @delali/narsil outside the bundle, or set workers.enabled to false`,
|
|
15177
|
+
{ moduleUrl }
|
|
15178
|
+
);
|
|
15179
|
+
}
|
|
15180
|
+
|
|
14640
15181
|
// src/workers/resource-limits.ts
|
|
14641
15182
|
function workerResourceLimits() {
|
|
14642
15183
|
return { maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_MB };
|
|
@@ -14644,14 +15185,10 @@ function workerResourceLimits() {
|
|
|
14644
15185
|
|
|
14645
15186
|
// src/vector/search-pool/spawn.ts
|
|
14646
15187
|
function resolveWorkerEntryPoint() {
|
|
14647
|
-
|
|
14648
|
-
const distIndex = base.lastIndexOf("/dist/");
|
|
14649
|
-
if (distIndex !== -1) {
|
|
14650
|
-
return new URL("vector/search-worker.mjs", base.slice(0, distIndex + 6)).href;
|
|
14651
|
-
}
|
|
14652
|
-
return base.replace(/\/src\/vector\/search-pool\/[^/]+$/, "/dist/vector/search-worker.mjs");
|
|
15188
|
+
return resolveWorkerEntry(import.meta.url, /\/src\/vector\/search-pool\/[^/]+$/, "vector/search-worker.mjs");
|
|
14653
15189
|
}
|
|
14654
15190
|
async function spawnWorker(entryPoint) {
|
|
15191
|
+
if (entryPoint === null) return null;
|
|
14655
15192
|
const runtime = detectRuntime();
|
|
14656
15193
|
if (runtime.supportsWorkerThreads) {
|
|
14657
15194
|
try {
|
|
@@ -15243,7 +15780,7 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
15243
15780
|
state.buffer.delete(docId);
|
|
15244
15781
|
count += 1;
|
|
15245
15782
|
if (count % BUILD_CHUNK_SIZE === 0) {
|
|
15246
|
-
await
|
|
15783
|
+
await yieldToEventLoop2();
|
|
15247
15784
|
}
|
|
15248
15785
|
}
|
|
15249
15786
|
return !state.disposed;
|
|
@@ -15274,7 +15811,7 @@ async function releaseLocations(state, paths, locations) {
|
|
|
15274
15811
|
let count = 0;
|
|
15275
15812
|
for (const [docId, location] of locations) {
|
|
15276
15813
|
if (count > 0 && count % DISK_ADOPTION_YIELD_INTERVAL === 0) {
|
|
15277
|
-
await
|
|
15814
|
+
await yieldToEventLoop2();
|
|
15278
15815
|
if (state.disposed) return;
|
|
15279
15816
|
}
|
|
15280
15817
|
count += 1;
|
|
@@ -15409,7 +15946,7 @@ function bytesToVectors(bytes) {
|
|
|
15409
15946
|
for (let i = 0; i < vectors.length; i++) vectors[i] = view.getFloat32(i * 4, true);
|
|
15410
15947
|
return vectors;
|
|
15411
15948
|
}
|
|
15412
|
-
function
|
|
15949
|
+
function invalid2(message, details) {
|
|
15413
15950
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Invalid vector index payload: ${message}`, details);
|
|
15414
15951
|
}
|
|
15415
15952
|
function isRecord(value) {
|
|
@@ -15423,20 +15960,20 @@ function isSerializedGraph(value) {
|
|
|
15423
15960
|
}
|
|
15424
15961
|
function codesOf(raw, dimension, count) {
|
|
15425
15962
|
if (raw === null || raw === void 0) return null;
|
|
15426
|
-
if (!isRecord(raw))
|
|
15963
|
+
if (!isRecord(raw)) invalid2("codes must be a map or nil");
|
|
15427
15964
|
const bits = raw.bits;
|
|
15428
|
-
if (bits !== 1 && bits !== 2 && bits !== 4 && bits !== 8)
|
|
15965
|
+
if (bits !== 1 && bits !== 2 && bits !== 4 && bits !== 8) invalid2("codes.bits must be 8, 4, 2, or 1", { bits });
|
|
15429
15966
|
if (!Array.isArray(raw.centroid) || raw.centroid.length !== dimension) {
|
|
15430
|
-
|
|
15967
|
+
invalid2("codes.centroid must hold one number per dimension");
|
|
15431
15968
|
}
|
|
15432
|
-
if (!(raw.records instanceof Uint8Array))
|
|
15969
|
+
if (!(raw.records instanceof Uint8Array)) invalid2("codes.records must be bytes");
|
|
15433
15970
|
if (raw.records.byteLength !== count * osqRecordBytes(dimension, bits)) {
|
|
15434
|
-
|
|
15971
|
+
invalid2("codes.records must hold one record per vector", { count, bytes: raw.records.byteLength });
|
|
15435
15972
|
}
|
|
15436
15973
|
return { bits, centroid: raw.centroid.map(Number), records: raw.records };
|
|
15437
15974
|
}
|
|
15438
15975
|
function decodeVectorIndexPart(raw) {
|
|
15439
|
-
if (!isRecord(raw))
|
|
15976
|
+
if (!isRecord(raw)) invalid2("payload must be a map");
|
|
15440
15977
|
if (raw.v !== VECTOR_INDEX_PAYLOAD_VERSION) {
|
|
15441
15978
|
throw new NarsilError(
|
|
15442
15979
|
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
@@ -15444,24 +15981,24 @@ function decodeVectorIndexPart(raw) {
|
|
|
15444
15981
|
{ version: raw.v ?? 1 }
|
|
15445
15982
|
);
|
|
15446
15983
|
}
|
|
15447
|
-
if (typeof raw.fieldName !== "string")
|
|
15448
|
-
if (!isNonNegativeInteger2(raw.dimension) || raw.dimension === 0)
|
|
15449
|
-
if (!isNonNegativeInteger2(raw.part))
|
|
15984
|
+
if (typeof raw.fieldName !== "string") invalid2("fieldName must be a string");
|
|
15985
|
+
if (!isNonNegativeInteger2(raw.dimension) || raw.dimension === 0) invalid2("dimension must be a positive integer");
|
|
15986
|
+
if (!isNonNegativeInteger2(raw.part)) invalid2("part must be a non-negative integer");
|
|
15450
15987
|
if (!isNonNegativeInteger2(raw.parts) || raw.parts === 0 || raw.part >= raw.parts) {
|
|
15451
|
-
|
|
15988
|
+
invalid2("parts must exceed part", { part: raw.part, parts: raw.parts });
|
|
15452
15989
|
}
|
|
15453
15990
|
if (!Array.isArray(raw.docIds) || !raw.docIds.every((docId) => typeof docId === "string")) {
|
|
15454
|
-
|
|
15991
|
+
invalid2("docIds must be a list of strings");
|
|
15455
15992
|
}
|
|
15456
15993
|
if (raw.docIds.length > VECTOR_INDEX_PART_VECTORS) {
|
|
15457
|
-
|
|
15994
|
+
invalid2(`a part holds at most ${VECTOR_INDEX_PART_VECTORS} vectors`, { count: raw.docIds.length });
|
|
15458
15995
|
}
|
|
15459
15996
|
if (!Array.isArray(raw.graphs) || !raw.graphs.every(isSerializedGraph)) {
|
|
15460
|
-
|
|
15997
|
+
invalid2("graphs must be a list of graphs, each holding a list of nodes");
|
|
15461
15998
|
}
|
|
15462
|
-
if (!(raw.vectors instanceof Uint8Array))
|
|
15999
|
+
if (!(raw.vectors instanceof Uint8Array)) invalid2("vectors must be bytes");
|
|
15463
16000
|
if (raw.vectors.byteLength !== raw.docIds.length * raw.dimension * 4) {
|
|
15464
|
-
|
|
16001
|
+
invalid2("vectors must hold dimension float32 components per document", {
|
|
15465
16002
|
count: raw.docIds.length,
|
|
15466
16003
|
bytes: raw.vectors.byteLength
|
|
15467
16004
|
});
|
|
@@ -15479,7 +16016,7 @@ function decodeVectorIndexPart(raw) {
|
|
|
15479
16016
|
};
|
|
15480
16017
|
}
|
|
15481
16018
|
function decodeVectorIndexParts(raw) {
|
|
15482
|
-
if (!Array.isArray(raw))
|
|
16019
|
+
if (!Array.isArray(raw)) invalid2("a field must carry a list of parts");
|
|
15483
16020
|
return raw.map(decodeVectorIndexPart);
|
|
15484
16021
|
}
|
|
15485
16022
|
|
|
@@ -15488,7 +16025,7 @@ var VECTOR_FILE_PAYLOAD_VERSION = 1;
|
|
|
15488
16025
|
var VECTOR_GRAPH_PAYLOAD_VERSION = 1;
|
|
15489
16026
|
var VECTOR_FILE_MAX_VECTORS = VECTOR_INDEX_PART_VECTORS;
|
|
15490
16027
|
var METRICS = ["cosine", "dotProduct", "euclidean"];
|
|
15491
|
-
function
|
|
16028
|
+
function invalid3(payload, message, details) {
|
|
15492
16029
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Invalid ${payload} payload: ${message}`, details);
|
|
15493
16030
|
}
|
|
15494
16031
|
function isRecord2(value) {
|
|
@@ -15507,19 +16044,19 @@ function requireVersion(payload, found, expected) {
|
|
|
15507
16044
|
}
|
|
15508
16045
|
function decodeVectorFilePayload(raw) {
|
|
15509
16046
|
const name = "vector file";
|
|
15510
|
-
if (!isRecord2(raw))
|
|
16047
|
+
if (!isRecord2(raw)) invalid3(name, "payload must be a map");
|
|
15511
16048
|
requireVersion(name, raw.v, VECTOR_FILE_PAYLOAD_VERSION);
|
|
15512
|
-
if (typeof raw.fieldName !== "string")
|
|
15513
|
-
if (!isIntegerUpTo(raw.dimension, 65535) || raw.dimension === 0)
|
|
16049
|
+
if (typeof raw.fieldName !== "string") invalid3(name, "fieldName must be a string");
|
|
16050
|
+
if (!isIntegerUpTo(raw.dimension, 65535) || raw.dimension === 0) invalid3(name, "dimension must be a positive uint16");
|
|
15514
16051
|
if (!Array.isArray(raw.docIds) || !raw.docIds.every((docId) => typeof docId === "string")) {
|
|
15515
|
-
|
|
16052
|
+
invalid3(name, "docIds must be a list of strings");
|
|
15516
16053
|
}
|
|
15517
16054
|
if (raw.docIds.length > VECTOR_FILE_MAX_VECTORS) {
|
|
15518
|
-
|
|
16055
|
+
invalid3(name, `a file holds at most ${VECTOR_FILE_MAX_VECTORS} vectors`, { count: raw.docIds.length });
|
|
15519
16056
|
}
|
|
15520
|
-
if (!(raw.vectors instanceof Uint8Array))
|
|
16057
|
+
if (!(raw.vectors instanceof Uint8Array)) invalid3(name, "vectors must be bytes");
|
|
15521
16058
|
if (raw.vectors.byteLength !== raw.docIds.length * raw.dimension * 4) {
|
|
15522
|
-
|
|
16059
|
+
invalid3(name, "vectors must hold dimension float32 components per document", {
|
|
15523
16060
|
count: raw.docIds.length,
|
|
15524
16061
|
bytes: raw.vectors.byteLength
|
|
15525
16062
|
});
|
|
@@ -15538,16 +16075,16 @@ function isMetric(value) {
|
|
|
15538
16075
|
}
|
|
15539
16076
|
function decodeNumberedGraph(raw) {
|
|
15540
16077
|
const name = "vector graph";
|
|
15541
|
-
if (!isRecord2(raw))
|
|
16078
|
+
if (!isRecord2(raw)) invalid3(name, "each graph must be a map");
|
|
15542
16079
|
const entryPoint = raw.entryPoint ?? null;
|
|
15543
16080
|
if (entryPoint !== null && !isIntegerUpTo(entryPoint, 4294967295))
|
|
15544
|
-
|
|
15545
|
-
if (!isIntegerUpTo(raw.maxLayer, 255))
|
|
15546
|
-
if (!isIntegerUpTo(raw.m, 65535) || raw.m === 0)
|
|
15547
|
-
if (!isIntegerUpTo(raw.efConstruction, 65535))
|
|
15548
|
-
if (!isMetric(raw.metric))
|
|
15549
|
-
if (!(raw.levels instanceof Uint8Array))
|
|
15550
|
-
if (!(raw.neighbours instanceof Uint8Array))
|
|
16081
|
+
invalid3(name, "entryPoint must be a uint32 or nil");
|
|
16082
|
+
if (!isIntegerUpTo(raw.maxLayer, 255)) invalid3(name, "maxLayer must be a uint8");
|
|
16083
|
+
if (!isIntegerUpTo(raw.m, 65535) || raw.m === 0) invalid3(name, "m must be a positive uint16");
|
|
16084
|
+
if (!isIntegerUpTo(raw.efConstruction, 65535)) invalid3(name, "efConstruction must be a uint16");
|
|
16085
|
+
if (!isMetric(raw.metric)) invalid3(name, "metric must name a supported metric", { metric: raw.metric });
|
|
16086
|
+
if (!(raw.levels instanceof Uint8Array)) invalid3(name, "levels must be bytes");
|
|
16087
|
+
if (!(raw.neighbours instanceof Uint8Array)) invalid3(name, "neighbours must be bytes");
|
|
15551
16088
|
return {
|
|
15552
16089
|
entryPoint,
|
|
15553
16090
|
maxLayer: raw.maxLayer,
|
|
@@ -15560,10 +16097,10 @@ function decodeNumberedGraph(raw) {
|
|
|
15560
16097
|
}
|
|
15561
16098
|
function decodeVectorGraphPayload(raw) {
|
|
15562
16099
|
const name = "vector graph";
|
|
15563
|
-
if (!isRecord2(raw))
|
|
16100
|
+
if (!isRecord2(raw)) invalid3(name, "payload must be a map");
|
|
15564
16101
|
requireVersion(name, raw.v, VECTOR_GRAPH_PAYLOAD_VERSION);
|
|
15565
|
-
if (typeof raw.fieldName !== "string")
|
|
15566
|
-
if (!Array.isArray(raw.graphs))
|
|
16102
|
+
if (typeof raw.fieldName !== "string") invalid3(name, "fieldName must be a string");
|
|
16103
|
+
if (!Array.isArray(raw.graphs)) invalid3(name, "graphs must be a list");
|
|
15567
16104
|
return { v: VECTOR_GRAPH_PAYLOAD_VERSION, fieldName: raw.fieldName, graphs: raw.graphs.map(decodeNumberedGraph) };
|
|
15568
16105
|
}
|
|
15569
16106
|
|
|
@@ -16313,6 +16850,18 @@ function* filteredDocIds(state, filter) {
|
|
|
16313
16850
|
if (docId !== void 0) yield docId;
|
|
16314
16851
|
}
|
|
16315
16852
|
}
|
|
16853
|
+
function admittedTotal(state, filter) {
|
|
16854
|
+
return filter === void 0 ? liveSize(state) : filter.count;
|
|
16855
|
+
}
|
|
16856
|
+
function fetchedOutcome(state, results, minSimilarity, filter) {
|
|
16857
|
+
if (minSimilarity === Number.NEGATIVE_INFINITY) {
|
|
16858
|
+
return { results, matched: admittedTotal(state, filter), matchedExact: true };
|
|
16859
|
+
}
|
|
16860
|
+
return { results, matched: results.length, matchedExact: false };
|
|
16861
|
+
}
|
|
16862
|
+
function scannedOutcome(scan2) {
|
|
16863
|
+
return { results: scan2.results, matched: scan2.matched, matchedExact: true };
|
|
16864
|
+
}
|
|
16316
16865
|
function highScoreFirst(a, b) {
|
|
16317
16866
|
return b.score - a.score || compareCodePoints(a.docId, b.docId);
|
|
16318
16867
|
}
|
|
@@ -16332,12 +16881,15 @@ function scanThroughTheCore(state, query, k, metric, minSimilarity, scanned) {
|
|
|
16332
16881
|
const distances = nativeScoresOf(state.store.handles, query, metric, scanned.ordinals);
|
|
16333
16882
|
if (distances === null) return null;
|
|
16334
16883
|
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
16884
|
+
let matched = 0;
|
|
16335
16885
|
for (let i = 0; i < scanned.ordinals.length; i++) {
|
|
16336
16886
|
if (distances[i] === Number.POSITIVE_INFINITY) continue;
|
|
16337
16887
|
const score = toScore(distances[i], metric);
|
|
16338
|
-
if (score
|
|
16888
|
+
if (score < minSimilarity) continue;
|
|
16889
|
+
matched++;
|
|
16890
|
+
heap.push({ docId: scanned.docIds[i], score });
|
|
16339
16891
|
}
|
|
16340
|
-
return heap.toSortedArray().reverse();
|
|
16892
|
+
return { results: heap.toSortedArray().reverse(), matched };
|
|
16341
16893
|
}
|
|
16342
16894
|
function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
|
|
16343
16895
|
const scanned = liveCandidates(state, candidates);
|
|
@@ -16346,6 +16898,7 @@ function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
|
|
|
16346
16898
|
const arenaQuery = state.store.prepareQueryArena(query);
|
|
16347
16899
|
const queryMag = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
16348
16900
|
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
16901
|
+
let matched = 0;
|
|
16349
16902
|
for (let i = 0; i < scanned.docIds.length; i++) {
|
|
16350
16903
|
const docId = scanned.docIds[i];
|
|
16351
16904
|
let score;
|
|
@@ -16370,11 +16923,11 @@ function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
|
|
|
16370
16923
|
}
|
|
16371
16924
|
}
|
|
16372
16925
|
}
|
|
16373
|
-
if (score
|
|
16374
|
-
|
|
16375
|
-
}
|
|
16926
|
+
if (score < minSimilarity) continue;
|
|
16927
|
+
matched++;
|
|
16928
|
+
heap.push({ docId, score });
|
|
16376
16929
|
}
|
|
16377
|
-
return heap.toSortedArray().reverse();
|
|
16930
|
+
return { results: heap.toSortedArray().reverse(), matched };
|
|
16378
16931
|
}
|
|
16379
16932
|
function mergeResults(hnswResults, bufferResults, k) {
|
|
16380
16933
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -16420,32 +16973,38 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
16420
16973
|
);
|
|
16421
16974
|
}
|
|
16422
16975
|
const currentLiveSize = liveSize(state);
|
|
16423
|
-
|
|
16424
|
-
if (
|
|
16976
|
+
const { metric, minSimilarity, efSearch, oversample } = options;
|
|
16977
|
+
if (currentLiveSize === 0) return { results: [], matched: 0, matchedExact: true };
|
|
16978
|
+
if (filter && filter.count === 0) return { results: [], matched: 0, matchedExact: true };
|
|
16979
|
+
if (k <= 0) return fetchedOutcome(state, [], minSimilarity, filter);
|
|
16425
16980
|
if (state.buffer.size > 0 && !state.building && !state.buildScheduled) {
|
|
16426
16981
|
scheduleBuild(state);
|
|
16427
16982
|
}
|
|
16428
|
-
const { metric, minSimilarity, efSearch, oversample } = options;
|
|
16429
|
-
if (filter && filter.count === 0) return [];
|
|
16430
16983
|
if (!state.hnsw) {
|
|
16431
16984
|
const candidates = filter ? filteredDocIds(state, filter) : allLiveDocIds(state);
|
|
16432
|
-
return bruteForceSearch(state, query, k, metric, minSimilarity, candidates);
|
|
16985
|
+
return scannedOutcome(bruteForceSearch(state, query, k, metric, minSimilarity, candidates));
|
|
16433
16986
|
}
|
|
16434
16987
|
if (filter) {
|
|
16435
16988
|
const hnswLiveSize = state.hnsw.size;
|
|
16436
16989
|
const selectivity = hnswLiveSize > 0 ? filter.count / hnswLiveSize : 1;
|
|
16437
16990
|
if (selectivity < state.filterThreshold) {
|
|
16438
|
-
|
|
16991
|
+
const scan2 = bruteForceSearch(state, query, k, metric, minSimilarity, filteredDocIds(state, filter));
|
|
16992
|
+
return scannedOutcome(scan2);
|
|
16439
16993
|
}
|
|
16440
16994
|
}
|
|
16441
16995
|
const graphOptions = { filter, efSearch, oversample };
|
|
16442
16996
|
if (state.buffer.size === 0) {
|
|
16443
16997
|
const hnswResults2 = state.hnsw.search(query, k, metric, minSimilarity, graphOptions);
|
|
16444
|
-
return
|
|
16998
|
+
return fetchedOutcome(
|
|
16999
|
+
state,
|
|
17000
|
+
hnswResults2.map((r) => ({ docId: r.docId, score: r.score })),
|
|
17001
|
+
minSimilarity,
|
|
17002
|
+
filter
|
|
17003
|
+
);
|
|
16445
17004
|
}
|
|
16446
17005
|
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, graphOptions).map((r) => ({ docId: r.docId, score: r.score }));
|
|
16447
|
-
const
|
|
16448
|
-
return mergeResults(hnswResults,
|
|
17006
|
+
const buffered = bruteForceSearch(state, query, k, metric, minSimilarity, bufferCandidates(state, filter));
|
|
17007
|
+
return fetchedOutcome(state, mergeResults(hnswResults, buffered.results, k), minSimilarity, filter);
|
|
16449
17008
|
}
|
|
16450
17009
|
|
|
16451
17010
|
// src/vector/vector-index/index.ts
|
|
@@ -16620,7 +17179,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
16620
17179
|
efSearch: options.efSearch,
|
|
16621
17180
|
oversample: options.oversample
|
|
16622
17181
|
});
|
|
16623
|
-
if (viaWorkerCopy !== null) return viaWorkerCopy;
|
|
17182
|
+
if (viaWorkerCopy !== null) return fetchedOutcome(state, viaWorkerCopy, options.minSimilarity, filter);
|
|
16624
17183
|
if (confined && state.revision !== filterRevision) {
|
|
16625
17184
|
filter = filterForOptions(state, options);
|
|
16626
17185
|
}
|
|
@@ -18611,20 +19170,26 @@ function bruteForceOrdinals(state, docIdOf, query, k, metric, minSimilarity, fil
|
|
|
18611
19170
|
}
|
|
18612
19171
|
const distances = nativeScoresOf(state.store.handles, query, metric, ordinals) ?? distancesInTypeScript(state, query, metric, ordinals);
|
|
18613
19172
|
const heap = createBoundedMaxHeap(bestFirst2, k);
|
|
19173
|
+
let matched = 0;
|
|
18614
19174
|
for (let i = 0; i < ordinals.length; i++) {
|
|
18615
19175
|
if (distances[i] === Number.POSITIVE_INFINITY) continue;
|
|
18616
19176
|
const score = toScore(distances[i], metric);
|
|
18617
|
-
if (score
|
|
19177
|
+
if (score < minSimilarity) continue;
|
|
19178
|
+
matched++;
|
|
19179
|
+
heap.push({ ord: ordinals[i], score, docId: docIds[i] });
|
|
18618
19180
|
}
|
|
18619
|
-
return heap.toSortedArray().reverse();
|
|
19181
|
+
return { hits: heap.toSortedArray().reverse(), matched, matchedExact: true };
|
|
18620
19182
|
}
|
|
18621
19183
|
function createSharedVectorSearcher(options) {
|
|
18622
19184
|
const { fieldName, view, holdsDocument } = options;
|
|
18623
19185
|
function filterForDocIds(allowed) {
|
|
18624
19186
|
const filter = createOrdinalFilter(view.store.slots);
|
|
19187
|
+
const graph = view.graph;
|
|
18625
19188
|
for (const docId of allowed) {
|
|
18626
19189
|
const ordinal = view.ordinalOf(docId);
|
|
18627
|
-
if (ordinal
|
|
19190
|
+
if (ordinal === void 0) continue;
|
|
19191
|
+
if (graph !== null && isTombstoned(graph, ordinal)) continue;
|
|
19192
|
+
addToOrdinalFilter(filter, ordinal);
|
|
18628
19193
|
}
|
|
18629
19194
|
return filter;
|
|
18630
19195
|
}
|
|
@@ -18651,7 +19216,11 @@ function createSharedVectorSearcher(options) {
|
|
|
18651
19216
|
if (graph !== null && filter !== void 0 && liveSize2 > 0 && filter.count / liveSize2 < view.handles.filterThreshold) {
|
|
18652
19217
|
return bruteForceOrdinals(graph, view.docIdOf, query, k, metric, minSimilarity, filter);
|
|
18653
19218
|
}
|
|
18654
|
-
|
|
19219
|
+
const hits = view.searchOrdinals(query, k, metric, minSimilarity, { filter, efSearch, oversample });
|
|
19220
|
+
if (minSimilarity === Number.NEGATIVE_INFINITY) {
|
|
19221
|
+
return { hits, matched: filter === void 0 ? liveSize2 : filter.count, matchedExact: true };
|
|
19222
|
+
}
|
|
19223
|
+
return { hits, matched: hits.length, matchedExact: false };
|
|
18655
19224
|
}
|
|
18656
19225
|
return {
|
|
18657
19226
|
fieldName,
|
|
@@ -18667,15 +19236,22 @@ function createSharedVectorSearcher(options) {
|
|
|
18667
19236
|
);
|
|
18668
19237
|
}
|
|
18669
19238
|
const filter = filterFor(searchOptions);
|
|
18670
|
-
if (filter !== void 0 && filter.count === 0)
|
|
18671
|
-
|
|
19239
|
+
if (filter !== void 0 && filter.count === 0) {
|
|
19240
|
+
return Promise.resolve({ results: [], matched: 0, matchedExact: true });
|
|
19241
|
+
}
|
|
19242
|
+
const scanned = hitsFor(query, k, searchOptions, filter);
|
|
18672
19243
|
const results = [];
|
|
18673
|
-
for (const hit of hits) {
|
|
19244
|
+
for (const hit of scanned.hits) {
|
|
18674
19245
|
const docId = view.docIdOf(hit.ord);
|
|
18675
19246
|
if (docId === void 0 || !holdsDocument(docId)) continue;
|
|
18676
19247
|
results.push({ docId, score: hit.score });
|
|
18677
19248
|
}
|
|
18678
|
-
|
|
19249
|
+
const holdsEveryHit = results.length === scanned.hits.length;
|
|
19250
|
+
return Promise.resolve({
|
|
19251
|
+
results,
|
|
19252
|
+
matched: scanned.matched,
|
|
19253
|
+
matchedExact: scanned.matchedExact && holdsEveryHit
|
|
19254
|
+
});
|
|
18679
19255
|
}
|
|
18680
19256
|
};
|
|
18681
19257
|
}
|
|
@@ -18802,7 +19378,7 @@ function closeHeldVectorCopies(held) {
|
|
|
18802
19378
|
held.fields.clear();
|
|
18803
19379
|
held.searchers.clear();
|
|
18804
19380
|
}
|
|
18805
|
-
function
|
|
19381
|
+
function yieldToEventLoop3() {
|
|
18806
19382
|
return new Promise((resolve) => {
|
|
18807
19383
|
if (typeof setImmediate === "function") setImmediate(resolve);
|
|
18808
19384
|
else setTimeout(resolve, 0);
|
|
@@ -18813,7 +19389,7 @@ async function insertIntoHeldGraph(held, fieldName, handle, ordinals) {
|
|
|
18813
19389
|
if (field === void 0) return null;
|
|
18814
19390
|
for (const ordinal of ordinals) {
|
|
18815
19391
|
field.view.insertOrdinal(ordinal);
|
|
18816
|
-
await
|
|
19392
|
+
await yieldToEventLoop3();
|
|
18817
19393
|
}
|
|
18818
19394
|
return field.view.takeOutcome();
|
|
18819
19395
|
}
|
|
@@ -19101,7 +19677,7 @@ function emitEngineEvent(eventHandlers, event, payload) {
|
|
|
19101
19677
|
}
|
|
19102
19678
|
|
|
19103
19679
|
// src/engine/analysis-rebuild.ts
|
|
19104
|
-
function
|
|
19680
|
+
function yieldToEventLoop4() {
|
|
19105
19681
|
return new Promise((resolve) => {
|
|
19106
19682
|
setTimeout(resolve, 0);
|
|
19107
19683
|
});
|
|
@@ -19142,7 +19718,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
19142
19718
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
19143
19719
|
manager.rebuildTextIndex(partitionId);
|
|
19144
19720
|
entry.partitionsRebuilt = partitionId + 1;
|
|
19145
|
-
await
|
|
19721
|
+
await yieldToEventLoop4();
|
|
19146
19722
|
}
|
|
19147
19723
|
stale.delete(indexName);
|
|
19148
19724
|
await deps.persistAnalysisRevision(indexName);
|
|
@@ -19321,6 +19897,10 @@ function resolveDurabilityTier(config) {
|
|
|
19321
19897
|
var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
|
|
19322
19898
|
var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
|
|
19323
19899
|
var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
|
|
19900
|
+
var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
|
|
19901
|
+
var DIRECTORY_LOCK_ATTEMPTS = 3;
|
|
19902
|
+
var DIRECTORY_LOCK_SETTLE_MS = 50;
|
|
19903
|
+
var DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS = 1e3;
|
|
19324
19904
|
var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
|
|
19325
19905
|
var DEFAULT_COMPACTION_THRESHOLD = 12;
|
|
19326
19906
|
var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
|
|
@@ -19346,35 +19926,192 @@ async function getPath() {
|
|
|
19346
19926
|
return filesystemUnavailable2();
|
|
19347
19927
|
}
|
|
19348
19928
|
|
|
19349
|
-
// src/persistence/durability/
|
|
19350
|
-
|
|
19351
|
-
|
|
19352
|
-
|
|
19929
|
+
// src/persistence/durability/directory-lock.ts
|
|
19930
|
+
var LOCK_FILE_NAME = ".narsil.lock";
|
|
19931
|
+
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
19932
|
+
var lockedDirectories = /* @__PURE__ */ new Set();
|
|
19933
|
+
function errnoOf(err) {
|
|
19934
|
+
return err?.code;
|
|
19935
|
+
}
|
|
19936
|
+
function processIsRunning(pid) {
|
|
19937
|
+
try {
|
|
19938
|
+
process.kill(pid, 0);
|
|
19939
|
+
return true;
|
|
19940
|
+
} catch (err) {
|
|
19941
|
+
return errnoOf(err) === "EPERM";
|
|
19353
19942
|
}
|
|
19354
|
-
|
|
19355
|
-
|
|
19943
|
+
}
|
|
19944
|
+
function integerOrNull(text) {
|
|
19945
|
+
const value = Number.parseInt(text ?? "", 10);
|
|
19946
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
19947
|
+
}
|
|
19948
|
+
function parseHolder(text) {
|
|
19949
|
+
const [pidText, threadText, startText, nonceText] = text.split("\n");
|
|
19950
|
+
const pid = integerOrNull(pidText);
|
|
19951
|
+
if (pid === null || pid <= 0) return null;
|
|
19952
|
+
const nonce = nonceText?.trim() ?? "";
|
|
19953
|
+
return {
|
|
19954
|
+
pid,
|
|
19955
|
+
threadId: integerOrNull(threadText),
|
|
19956
|
+
processStartMs: integerOrNull(startText),
|
|
19957
|
+
nonce: nonce.length > 0 ? nonce : null
|
|
19958
|
+
};
|
|
19959
|
+
}
|
|
19960
|
+
function formatHolder(holder) {
|
|
19961
|
+
return `${holder.pid}
|
|
19962
|
+
${holder.threadId ?? ""}
|
|
19963
|
+
${holder.processStartMs ?? ""}
|
|
19964
|
+
${holder.nonce ?? ""}
|
|
19965
|
+
`;
|
|
19966
|
+
}
|
|
19967
|
+
async function readHolder(fs, lockPath) {
|
|
19968
|
+
try {
|
|
19969
|
+
return parseHolder(await fs.readFile(lockPath, "utf8")) ?? "unreadable";
|
|
19970
|
+
} catch (err) {
|
|
19971
|
+
if (errnoOf(err) === "ENOENT") return "absent";
|
|
19972
|
+
throw err;
|
|
19356
19973
|
}
|
|
19357
19974
|
}
|
|
19358
|
-
function
|
|
19359
|
-
|
|
19360
|
-
|
|
19361
|
-
|
|
19362
|
-
|
|
19975
|
+
async function settledHolder(fs, lockPath) {
|
|
19976
|
+
const first = await readHolder(fs, lockPath);
|
|
19977
|
+
if (first !== "unreadable") return first;
|
|
19978
|
+
await new Promise((resolve) => setTimeout(resolve, DIRECTORY_LOCK_SETTLE_MS));
|
|
19979
|
+
return readHolder(fs, lockPath);
|
|
19980
|
+
}
|
|
19981
|
+
function holderIsLive(holder, self) {
|
|
19982
|
+
if (holder.pid !== self.pid) return processIsRunning(holder.pid);
|
|
19983
|
+
if (holder.processStartMs === null || self.processStartMs === null) return false;
|
|
19984
|
+
return Math.abs(holder.processStartMs - self.processStartMs) <= DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS;
|
|
19985
|
+
}
|
|
19986
|
+
function heldElsewhere(root, holder, lockPath) {
|
|
19987
|
+
return new NarsilError(
|
|
19988
|
+
ErrorCodes.CONFIG_INVALID,
|
|
19989
|
+
`The durability directory "${root}" is in use by ${holder}, and only one engine may write a write-ahead log directory. Point this engine at another directory, or shut the other engine down first. Where no engine uses the directory, delete "${lockPath}"`,
|
|
19990
|
+
{ directory: root }
|
|
19363
19991
|
);
|
|
19364
19992
|
}
|
|
19365
|
-
|
|
19993
|
+
function holderLabel(holder, self) {
|
|
19994
|
+
if (holder.pid !== self.pid) return `process ${holder.pid}`;
|
|
19995
|
+
return holder.threadId === self.threadId ? "another copy of Narsil in this thread" : "another thread of this process";
|
|
19996
|
+
}
|
|
19997
|
+
async function createLockFileExclusively(fs, lockPath, content) {
|
|
19366
19998
|
try {
|
|
19367
|
-
|
|
19999
|
+
await fs.writeFile(lockPath, content, { flag: "wx" });
|
|
20000
|
+
return true;
|
|
19368
20001
|
} catch (err) {
|
|
19369
|
-
if (err
|
|
19370
|
-
|
|
19371
|
-
}
|
|
20002
|
+
if (errnoOf(err) === "EEXIST") return false;
|
|
20003
|
+
throw err;
|
|
19372
20004
|
}
|
|
20005
|
+
}
|
|
20006
|
+
async function createLockFile(fs, lockPath, draftPath, content) {
|
|
20007
|
+
await fs.writeFile(draftPath, content, { flag: "w" });
|
|
19373
20008
|
try {
|
|
19374
|
-
|
|
20009
|
+
await fs.link(draftPath, lockPath);
|
|
20010
|
+
return true;
|
|
19375
20011
|
} catch (err) {
|
|
19376
|
-
|
|
19377
|
-
|
|
20012
|
+
const code = errnoOf(err);
|
|
20013
|
+
if (code === "EEXIST") return false;
|
|
20014
|
+
if (code !== void 0 && HARD_LINK_UNSUPPORTED_CODES.has(code)) {
|
|
20015
|
+
return createLockFileExclusively(fs, lockPath, content);
|
|
20016
|
+
}
|
|
20017
|
+
throw err;
|
|
20018
|
+
} finally {
|
|
20019
|
+
await fs.unlink(draftPath).catch(() => void 0);
|
|
20020
|
+
}
|
|
20021
|
+
}
|
|
20022
|
+
async function acquire(fs, root, lockPath, self) {
|
|
20023
|
+
const draftPath = `${lockPath}.${self.pid}.${self.threadId ?? 0}`;
|
|
20024
|
+
const content = formatHolder(self);
|
|
20025
|
+
for (let attempt = 0; attempt < DIRECTORY_LOCK_ATTEMPTS; attempt++) {
|
|
20026
|
+
if (await createLockFile(fs, lockPath, draftPath, content)) return;
|
|
20027
|
+
const holder = await settledHolder(fs, lockPath);
|
|
20028
|
+
if (typeof holder === "object" && holderIsLive(holder, self)) {
|
|
20029
|
+
throw heldElsewhere(root, holderLabel(holder, self), lockPath);
|
|
20030
|
+
}
|
|
20031
|
+
await fs.unlink(lockPath).catch((err) => {
|
|
20032
|
+
if (errnoOf(err) !== "ENOENT") throw err;
|
|
20033
|
+
});
|
|
20034
|
+
}
|
|
20035
|
+
throw heldElsewhere(root, "another process", lockPath);
|
|
20036
|
+
}
|
|
20037
|
+
async function describeSelf() {
|
|
20038
|
+
return {
|
|
20039
|
+
pid: process.pid,
|
|
20040
|
+
threadId: await nodeThreadId(),
|
|
20041
|
+
processStartMs: Math.round(Date.now() - process.uptime() * 1e3),
|
|
20042
|
+
nonce: globalThis.crypto.randomUUID()
|
|
20043
|
+
};
|
|
20044
|
+
}
|
|
20045
|
+
async function lockDirectory(root) {
|
|
20046
|
+
const fs = await getFs();
|
|
20047
|
+
const pathMod = await getPath();
|
|
20048
|
+
const resolvedRoot = pathMod.resolve(root);
|
|
20049
|
+
const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
|
|
20050
|
+
const self = await describeSelf();
|
|
20051
|
+
let directoryKey = null;
|
|
20052
|
+
try {
|
|
20053
|
+
await fs.mkdir(resolvedRoot, { recursive: true });
|
|
20054
|
+
const stat = await fs.stat(resolvedRoot, { bigint: true });
|
|
20055
|
+
const key = `${stat.dev}:${stat.ino}`;
|
|
20056
|
+
if (lockedDirectories.has(key)) throw heldElsewhere(root, "another engine in this process", lockPath);
|
|
20057
|
+
lockedDirectories.add(key);
|
|
20058
|
+
directoryKey = key;
|
|
20059
|
+
await acquire(fs, root, lockPath, self);
|
|
20060
|
+
} catch (err) {
|
|
20061
|
+
if (directoryKey !== null) lockedDirectories.delete(directoryKey);
|
|
20062
|
+
if (err instanceof NarsilError) throw err;
|
|
20063
|
+
throw new NarsilError(
|
|
20064
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
20065
|
+
`The engine cannot lock the durability directory "${root}"`,
|
|
20066
|
+
{ directory: root, cause: err instanceof Error ? err.message : String(err) }
|
|
20067
|
+
);
|
|
20068
|
+
}
|
|
20069
|
+
const heldKey = directoryKey;
|
|
20070
|
+
let released = false;
|
|
20071
|
+
async function holds() {
|
|
20072
|
+
const holder = await readHolder(fs, lockPath);
|
|
20073
|
+
return typeof holder === "object" && holder.nonce === self.nonce;
|
|
20074
|
+
}
|
|
20075
|
+
return {
|
|
20076
|
+
holds,
|
|
20077
|
+
async release() {
|
|
20078
|
+
if (released) return;
|
|
20079
|
+
released = true;
|
|
20080
|
+
lockedDirectories.delete(heldKey);
|
|
20081
|
+
if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
|
|
20082
|
+
}
|
|
20083
|
+
};
|
|
20084
|
+
}
|
|
20085
|
+
|
|
20086
|
+
// src/persistence/durability/durable-filesystem.ts
|
|
20087
|
+
function validateRelativeKey(key) {
|
|
20088
|
+
if (key.length === 0) {
|
|
20089
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
|
|
20090
|
+
}
|
|
20091
|
+
if (key.includes("\0")) {
|
|
20092
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: null byte detected", { key });
|
|
20093
|
+
}
|
|
20094
|
+
}
|
|
20095
|
+
function wrapFsyncError(err, key) {
|
|
20096
|
+
throw new NarsilError(
|
|
20097
|
+
ErrorCodes.PERSISTENCE_FSYNC_FAILED,
|
|
20098
|
+
`fsync failed for "${key}"; the write is not acknowledged and the node must recover from the durable log`,
|
|
20099
|
+
{ key, cause: err instanceof Error ? err.message : String(err) }
|
|
20100
|
+
);
|
|
20101
|
+
}
|
|
20102
|
+
async function openReadWriteCreating(filePath, fs) {
|
|
20103
|
+
try {
|
|
20104
|
+
return await fs.open(filePath, "r+");
|
|
20105
|
+
} catch (err) {
|
|
20106
|
+
if (err.code !== "ENOENT") {
|
|
20107
|
+
throw err;
|
|
20108
|
+
}
|
|
20109
|
+
}
|
|
20110
|
+
try {
|
|
20111
|
+
return await fs.open(filePath, "wx+");
|
|
20112
|
+
} catch (err) {
|
|
20113
|
+
if (err.code !== "EEXIST") {
|
|
20114
|
+
throw err;
|
|
19378
20115
|
}
|
|
19379
20116
|
return fs.open(filePath, "r+");
|
|
19380
20117
|
}
|
|
@@ -19431,6 +20168,20 @@ function createDurableDirectory(root) {
|
|
|
19431
20168
|
get root() {
|
|
19432
20169
|
return root;
|
|
19433
20170
|
},
|
|
20171
|
+
lock() {
|
|
20172
|
+
return lockDirectory(root);
|
|
20173
|
+
},
|
|
20174
|
+
async identity() {
|
|
20175
|
+
const fs = await getFs();
|
|
20176
|
+
const pathMod = await getPath();
|
|
20177
|
+
try {
|
|
20178
|
+
const stat = await fs.stat(pathMod.resolve(root), { bigint: true });
|
|
20179
|
+
return `${stat.dev}:${stat.ino}`;
|
|
20180
|
+
} catch (err) {
|
|
20181
|
+
if (err.code === "ENOENT") return null;
|
|
20182
|
+
throw err;
|
|
20183
|
+
}
|
|
20184
|
+
},
|
|
19434
20185
|
async appendHandle(key) {
|
|
19435
20186
|
const { path: filePath, pathMod } = await resolve(key);
|
|
19436
20187
|
const fs = await getFs();
|
|
@@ -19662,22 +20413,20 @@ async function isOnMainThread() {
|
|
|
19662
20413
|
}
|
|
19663
20414
|
return onMainThread;
|
|
19664
20415
|
}
|
|
19665
|
-
function resolveWorkerEntryPoint2() {
|
|
19666
|
-
const base = import.meta.url;
|
|
19667
|
-
const distIndex = base.lastIndexOf("/dist/");
|
|
19668
|
-
if (distIndex !== -1) {
|
|
19669
|
-
return new URL("serialization/crc32-worker.mjs", base.slice(0, distIndex + 6)).href;
|
|
19670
|
-
}
|
|
19671
|
-
return base.replace(/\/src\/serialization\/[^/]+$/, "/dist/serialization/crc32-worker.mjs");
|
|
19672
|
-
}
|
|
19673
20416
|
async function spawnWorker2() {
|
|
20417
|
+
const entryPoint = resolveWorkerEntry(
|
|
20418
|
+
import.meta.url,
|
|
20419
|
+
/\/src\/serialization\/[^/]+$/,
|
|
20420
|
+
"serialization/crc32-worker.mjs"
|
|
20421
|
+
);
|
|
20422
|
+
if (entryPoint === null) return null;
|
|
19674
20423
|
try {
|
|
19675
|
-
return await spawnNodeWorker(new URL(
|
|
20424
|
+
return await spawnNodeWorker(new URL(entryPoint));
|
|
19676
20425
|
} catch {
|
|
19677
20426
|
return null;
|
|
19678
20427
|
}
|
|
19679
20428
|
}
|
|
19680
|
-
function
|
|
20429
|
+
function yieldToEventLoop5() {
|
|
19681
20430
|
if (typeof setImmediate === "function") {
|
|
19682
20431
|
return new Promise((resolve) => setImmediate(resolve));
|
|
19683
20432
|
}
|
|
@@ -19689,7 +20438,7 @@ async function advanceChecksum(start, payload) {
|
|
|
19689
20438
|
const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
|
|
19690
20439
|
state = crc32Update(state, payload.subarray(offset, end));
|
|
19691
20440
|
if (end < payload.length) {
|
|
19692
|
-
await
|
|
20441
|
+
await yieldToEventLoop5();
|
|
19693
20442
|
}
|
|
19694
20443
|
}
|
|
19695
20444
|
return state;
|
|
@@ -20967,7 +21716,7 @@ function attachDocuments(manager, partitionId, segment) {
|
|
|
20967
21716
|
}
|
|
20968
21717
|
manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
|
|
20969
21718
|
}
|
|
20970
|
-
function
|
|
21719
|
+
function yieldToEventLoop6() {
|
|
20971
21720
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
20972
21721
|
}
|
|
20973
21722
|
async function loadPartitionSegmentBySegment(directory, partition, manager) {
|
|
@@ -20977,12 +21726,12 @@ async function loadPartitionSegmentBySegment(directory, partition, manager) {
|
|
|
20977
21726
|
addSegmentSchema(agreedSchema, segment.schema);
|
|
20978
21727
|
removeWhatTheSegmentReplaces(manager, segment);
|
|
20979
21728
|
attachDocuments(manager, partition.partitionId, segment);
|
|
20980
|
-
await
|
|
21729
|
+
await yieldToEventLoop6();
|
|
20981
21730
|
}
|
|
20982
21731
|
}
|
|
20983
21732
|
|
|
20984
21733
|
// src/persistence/durability/segment/manifest-vectors-decode.ts
|
|
20985
|
-
function
|
|
21734
|
+
function invalid4(message, details) {
|
|
20986
21735
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Segment manifest ${message}`, details);
|
|
20987
21736
|
}
|
|
20988
21737
|
function isRecord3(value) {
|
|
@@ -20995,19 +21744,19 @@ function isKey(value) {
|
|
|
20995
21744
|
return typeof value === "string" && value.length > 0 && !value.includes("\0");
|
|
20996
21745
|
}
|
|
20997
21746
|
function normalizeFile(raw, fieldPath, nextFileId, seenIds) {
|
|
20998
|
-
if (!isRecord3(raw) || !isId(raw.id) || !isKey(raw.key))
|
|
21747
|
+
if (!isRecord3(raw) || !isId(raw.id) || !isKey(raw.key)) invalid4("has an invalid vector file reference", { fieldPath });
|
|
20999
21748
|
if (raw.id >= nextFileId || seenIds.has(raw.id)) {
|
|
21000
|
-
|
|
21749
|
+
invalid4(`lists vector file ${raw.id} of "${fieldPath}" twice or at its next file id`, { fieldPath, id: raw.id });
|
|
21001
21750
|
}
|
|
21002
21751
|
seenIds.add(raw.id);
|
|
21003
21752
|
const count = raw.count;
|
|
21004
21753
|
if (typeof count !== "number" || !Number.isInteger(count) || count < 0 || count > VECTOR_FILE_MAX_VECTORS) {
|
|
21005
|
-
|
|
21754
|
+
invalid4(`gives vector file ${raw.id} of "${fieldPath}" an invalid count`, { fieldPath, id: raw.id, count });
|
|
21006
21755
|
}
|
|
21007
21756
|
const dead = raw.dead ?? null;
|
|
21008
|
-
if (dead !== null && !(dead instanceof Uint8Array))
|
|
21757
|
+
if (dead !== null && !(dead instanceof Uint8Array)) invalid4("has dead vectors that are not bytes", { fieldPath });
|
|
21009
21758
|
if (dead !== null && dead.byteLength !== deadBytesFor(count)) {
|
|
21010
|
-
|
|
21759
|
+
invalid4(`marks the dead vectors of file ${raw.id} of "${fieldPath}" with the wrong number of bytes`, {
|
|
21011
21760
|
fieldPath,
|
|
21012
21761
|
id: raw.id,
|
|
21013
21762
|
count,
|
|
@@ -21017,17 +21766,17 @@ function normalizeFile(raw, fieldPath, nextFileId, seenIds) {
|
|
|
21017
21766
|
return { id: raw.id, key: raw.key, count, dead };
|
|
21018
21767
|
}
|
|
21019
21768
|
function normalizeField(raw, seenFields) {
|
|
21020
|
-
if (!isRecord3(raw) || typeof raw.fieldPath !== "string")
|
|
21769
|
+
if (!isRecord3(raw) || typeof raw.fieldPath !== "string") invalid4("has an invalid vector reference", { vector: raw });
|
|
21021
21770
|
const fieldPath = raw.fieldPath;
|
|
21022
|
-
if (seenFields.has(fieldPath))
|
|
21771
|
+
if (seenFields.has(fieldPath)) invalid4(`lists the vector field "${fieldPath}" twice`, { fieldPath });
|
|
21023
21772
|
seenFields.add(fieldPath);
|
|
21024
|
-
if (!isId(raw.nextFileId))
|
|
21025
|
-
if (!Array.isArray(raw.files))
|
|
21026
|
-
if (!isId(raw.graphGeneration))
|
|
21773
|
+
if (!isId(raw.nextFileId)) invalid4(`gives "${fieldPath}" an invalid next file id`, { fieldPath });
|
|
21774
|
+
if (!Array.isArray(raw.files)) invalid4(`gives "${fieldPath}" no list of vector files`, { fieldPath });
|
|
21775
|
+
if (!isId(raw.graphGeneration)) invalid4(`gives "${fieldPath}" an invalid graph generation`, { fieldPath });
|
|
21027
21776
|
const graphKey = raw.graphKey ?? null;
|
|
21028
|
-
if (graphKey !== null && !isKey(graphKey))
|
|
21777
|
+
if (graphKey !== null && !isKey(graphKey)) invalid4(`gives "${fieldPath}" an invalid graph key`, { fieldPath });
|
|
21029
21778
|
if (graphKey !== null && raw.graphGeneration === 0) {
|
|
21030
|
-
|
|
21779
|
+
invalid4(`names a graph file for "${fieldPath}" at generation 0`, { fieldPath });
|
|
21031
21780
|
}
|
|
21032
21781
|
const nextFileId = raw.nextFileId;
|
|
21033
21782
|
const seenIds = /* @__PURE__ */ new Set();
|
|
@@ -21041,7 +21790,7 @@ function normalizeField(raw, seenFields) {
|
|
|
21041
21790
|
}
|
|
21042
21791
|
function normalizeVectorFields(raw) {
|
|
21043
21792
|
if (raw === void 0 || raw === null) return [];
|
|
21044
|
-
if (!Array.isArray(raw))
|
|
21793
|
+
if (!Array.isArray(raw)) invalid4("has malformed vector references");
|
|
21045
21794
|
const seenFields = /* @__PURE__ */ new Set();
|
|
21046
21795
|
return raw.map((field) => normalizeField(field, seenFields));
|
|
21047
21796
|
}
|
|
@@ -22443,17 +23192,15 @@ var workerBusy = false;
|
|
|
22443
23192
|
var spawnedWorkerCount = 0;
|
|
22444
23193
|
var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
|
|
22445
23194
|
var idleTimer = null;
|
|
22446
|
-
function resolveWorkerEntryPoint3() {
|
|
22447
|
-
const base = import.meta.url;
|
|
22448
|
-
const distIndex = base.lastIndexOf("/dist/");
|
|
22449
|
-
if (distIndex !== -1) {
|
|
22450
|
-
return new URL("persistence/durability/checkpoint-worker.mjs", base.slice(0, distIndex + 6)).href;
|
|
22451
|
-
}
|
|
22452
|
-
return base.replace(/\/src\/persistence\/durability\/[^/]+$/, "/dist/persistence/durability/checkpoint-worker.mjs");
|
|
22453
|
-
}
|
|
22454
23195
|
async function spawnWorker3() {
|
|
23196
|
+
const entryPoint = resolveWorkerEntry(
|
|
23197
|
+
import.meta.url,
|
|
23198
|
+
/\/src\/persistence\/durability\/[^/]+$/,
|
|
23199
|
+
"persistence/durability/checkpoint-worker.mjs"
|
|
23200
|
+
);
|
|
23201
|
+
if (entryPoint === null) return null;
|
|
22455
23202
|
try {
|
|
22456
|
-
const worker = await spawnNodeWorker(new URL(
|
|
23203
|
+
const worker = await spawnNodeWorker(new URL(entryPoint));
|
|
22457
23204
|
if (worker === null) {
|
|
22458
23205
|
return null;
|
|
22459
23206
|
}
|
|
@@ -22684,11 +23431,29 @@ async function garbageThatNoFieldStillReads(directory, vectorIndexes, keys) {
|
|
|
22684
23431
|
return unread;
|
|
22685
23432
|
}
|
|
22686
23433
|
async function runDurableCheckpoint(input) {
|
|
22687
|
-
|
|
22688
|
-
|
|
22689
|
-
recordsLeftInLog
|
|
23434
|
+
try {
|
|
23435
|
+
let recordsLeftInLog = await writeOneBoundedCheckpoint(input);
|
|
23436
|
+
while (recordsLeftInLog > 0 && input.mayContinue()) {
|
|
23437
|
+
recordsLeftInLog = await writeOneBoundedCheckpoint(input);
|
|
23438
|
+
}
|
|
23439
|
+
} catch (err) {
|
|
23440
|
+
if (!isSystemError(err)) throw err;
|
|
23441
|
+
const error = new NarsilError(
|
|
23442
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
23443
|
+
`The engine cannot write the checkpoint of index "${input.indexName}" to the durability directory "${input.directory.root}"`,
|
|
23444
|
+
{
|
|
23445
|
+
indexName: input.indexName,
|
|
23446
|
+
directory: input.directory.root,
|
|
23447
|
+
cause: err.message
|
|
23448
|
+
}
|
|
23449
|
+
);
|
|
23450
|
+
input.markFatal(error);
|
|
23451
|
+
throw error;
|
|
22690
23452
|
}
|
|
22691
23453
|
}
|
|
23454
|
+
function isSystemError(err) {
|
|
23455
|
+
return err instanceof Error && !(err instanceof NarsilError) && typeof err.code === "string";
|
|
23456
|
+
}
|
|
22692
23457
|
async function writeOneBoundedCheckpoint(input) {
|
|
22693
23458
|
const manager = input.hooks.getManager(input.indexName);
|
|
22694
23459
|
if (manager === void 0) {
|
|
@@ -22768,6 +23533,94 @@ function recordsBeyond(indexState, targets) {
|
|
|
22768
23533
|
return records;
|
|
22769
23534
|
}
|
|
22770
23535
|
|
|
23536
|
+
// src/persistence/durability/directory-watch.ts
|
|
23537
|
+
function unrefInterval(tick, intervalMs) {
|
|
23538
|
+
const timer = setInterval(tick, intervalMs);
|
|
23539
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
23540
|
+
return timer;
|
|
23541
|
+
}
|
|
23542
|
+
function createDirectoryWatch(directory, isStopped, onLost) {
|
|
23543
|
+
let expected = null;
|
|
23544
|
+
let timer = null;
|
|
23545
|
+
let lock = null;
|
|
23546
|
+
function cannotCheck(err) {
|
|
23547
|
+
return new NarsilError(
|
|
23548
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
23549
|
+
`The engine cannot check the durability directory "${directory.root}", so it cannot confirm that a write is durable`,
|
|
23550
|
+
{ directory: directory.root, cause: err instanceof Error ? err.message : String(err) }
|
|
23551
|
+
);
|
|
23552
|
+
}
|
|
23553
|
+
async function currentIdentity() {
|
|
23554
|
+
try {
|
|
23555
|
+
return await directory.identity?.() ?? null;
|
|
23556
|
+
} catch (err) {
|
|
23557
|
+
return cannotCheck(err);
|
|
23558
|
+
}
|
|
23559
|
+
}
|
|
23560
|
+
async function lockTakenOver() {
|
|
23561
|
+
if (lock === null) return null;
|
|
23562
|
+
try {
|
|
23563
|
+
if (await lock.holds()) return null;
|
|
23564
|
+
} catch (err) {
|
|
23565
|
+
return cannotCheck(err);
|
|
23566
|
+
}
|
|
23567
|
+
return new NarsilError(
|
|
23568
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
23569
|
+
`The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
|
|
23570
|
+
{ directory: directory.root }
|
|
23571
|
+
);
|
|
23572
|
+
}
|
|
23573
|
+
async function verify() {
|
|
23574
|
+
if (directory.identity === void 0 || isStopped()) return true;
|
|
23575
|
+
const current = await currentIdentity();
|
|
23576
|
+
if (current instanceof Error) {
|
|
23577
|
+
onLost(current);
|
|
23578
|
+
return false;
|
|
23579
|
+
}
|
|
23580
|
+
if (expected === null) {
|
|
23581
|
+
expected = current;
|
|
23582
|
+
} else if (current !== expected) {
|
|
23583
|
+
onLost(
|
|
23584
|
+
new NarsilError(
|
|
23585
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
23586
|
+
`The durability directory "${directory.root}" is missing or is a different directory from the one that this instance opens at start-up, so a write to it is not durable`,
|
|
23587
|
+
{ directory: directory.root }
|
|
23588
|
+
)
|
|
23589
|
+
);
|
|
23590
|
+
return false;
|
|
23591
|
+
}
|
|
23592
|
+
const takenOver = await lockTakenOver();
|
|
23593
|
+
if (takenOver === null) return true;
|
|
23594
|
+
onLost(takenOver);
|
|
23595
|
+
return false;
|
|
23596
|
+
}
|
|
23597
|
+
function stop() {
|
|
23598
|
+
if (timer !== null) {
|
|
23599
|
+
clearInterval(timer);
|
|
23600
|
+
timer = null;
|
|
23601
|
+
}
|
|
23602
|
+
}
|
|
23603
|
+
return {
|
|
23604
|
+
async claim() {
|
|
23605
|
+
if (lock === null && directory.lock !== void 0) lock = await directory.lock();
|
|
23606
|
+
},
|
|
23607
|
+
async release() {
|
|
23608
|
+
const held = lock;
|
|
23609
|
+
lock = null;
|
|
23610
|
+
await held?.release();
|
|
23611
|
+
},
|
|
23612
|
+
verify,
|
|
23613
|
+
async verifyOnce() {
|
|
23614
|
+
if (expected === null && directory.identity !== void 0) await verify();
|
|
23615
|
+
},
|
|
23616
|
+
start() {
|
|
23617
|
+
if (directory.identity === void 0 || timer !== null || isStopped()) return;
|
|
23618
|
+
timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
|
|
23619
|
+
},
|
|
23620
|
+
stop
|
|
23621
|
+
};
|
|
23622
|
+
}
|
|
23623
|
+
|
|
22771
23624
|
// src/persistence/durability/manager-state.ts
|
|
22772
23625
|
async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
|
|
22773
23626
|
indexState.unloading = true;
|
|
@@ -23240,11 +24093,13 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
23240
24093
|
let asyncFlushTimer = null;
|
|
23241
24094
|
let shuttingDown = false;
|
|
23242
24095
|
let fatalError = null;
|
|
24096
|
+
const directoryWatch = createDirectoryWatch(directory, () => shuttingDown || fatalError !== null, markFatal);
|
|
23243
24097
|
function markFatal(error) {
|
|
23244
24098
|
if (fatalError !== null) {
|
|
23245
24099
|
return;
|
|
23246
24100
|
}
|
|
23247
24101
|
fatalError = error;
|
|
24102
|
+
directoryWatch.stop();
|
|
23248
24103
|
if (checkpointTimer !== null) {
|
|
23249
24104
|
clearInterval(checkpointTimer);
|
|
23250
24105
|
checkpointTimer = null;
|
|
@@ -23308,23 +24163,13 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
23308
24163
|
if (checkpointTimer !== null || shuttingDown || fatalError !== null || checkpointIntervalMs <= 0) {
|
|
23309
24164
|
return;
|
|
23310
24165
|
}
|
|
23311
|
-
checkpointTimer =
|
|
23312
|
-
void runScheduledCheckpoints();
|
|
23313
|
-
}, checkpointIntervalMs);
|
|
23314
|
-
if (typeof checkpointTimer.unref === "function") {
|
|
23315
|
-
checkpointTimer.unref();
|
|
23316
|
-
}
|
|
24166
|
+
checkpointTimer = unrefInterval(() => void runScheduledCheckpoints(), checkpointIntervalMs);
|
|
23317
24167
|
}
|
|
23318
24168
|
function startAsyncFlushTimer() {
|
|
23319
24169
|
if (mode !== "async" || asyncFlushTimer !== null || shuttingDown || fatalError !== null || flushIntervalMs <= 0) {
|
|
23320
24170
|
return;
|
|
23321
24171
|
}
|
|
23322
|
-
asyncFlushTimer =
|
|
23323
|
-
void flushAllPartitions();
|
|
23324
|
-
}, flushIntervalMs);
|
|
23325
|
-
if (typeof asyncFlushTimer.unref === "function") {
|
|
23326
|
-
asyncFlushTimer.unref();
|
|
23327
|
-
}
|
|
24172
|
+
asyncFlushTimer = unrefInterval(() => void flushAllPartitions(), flushIntervalMs);
|
|
23328
24173
|
}
|
|
23329
24174
|
async function flushAllPartitions() {
|
|
23330
24175
|
if (fatalError !== null) {
|
|
@@ -23386,6 +24231,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
23386
24231
|
return run;
|
|
23387
24232
|
}
|
|
23388
24233
|
async function performCheckpoint(indexName, indexState, fromMemory = false) {
|
|
24234
|
+
await directoryWatch.verify();
|
|
24235
|
+
if (fatalError !== null) throw fatalError;
|
|
23389
24236
|
await runDurableCheckpoint({
|
|
23390
24237
|
directory,
|
|
23391
24238
|
hooks,
|
|
@@ -23435,10 +24282,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
23435
24282
|
}
|
|
23436
24283
|
await stallWhileCheckpointsFallBehind(indexState);
|
|
23437
24284
|
}
|
|
23438
|
-
if (recordedByIndex.size
|
|
23439
|
-
|
|
23440
|
-
|
|
23441
|
-
}
|
|
24285
|
+
if (recordedByIndex.size === 0) return outcomes;
|
|
24286
|
+
if (mode === "sync") await directoryWatch.verify();
|
|
24287
|
+
const lost = mode === "sync" ? fatalError : null;
|
|
24288
|
+
if (lost !== null) return outcomes.map((outcome) => outcome.ok ? { ok: false, error: lost } : outcome);
|
|
24289
|
+
await directoryWatch.verifyOnce();
|
|
24290
|
+
directoryWatch.start();
|
|
24291
|
+
startCheckpointTimer();
|
|
24292
|
+
startAsyncFlushTimer();
|
|
23442
24293
|
return outcomes;
|
|
23443
24294
|
}
|
|
23444
24295
|
function recoverIndex(indexName, metadataOnly = false) {
|
|
@@ -23460,10 +24311,13 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
23460
24311
|
return true;
|
|
23461
24312
|
},
|
|
23462
24313
|
async recover(metadataOnly = false) {
|
|
24314
|
+
await directoryWatch.claim();
|
|
23463
24315
|
const names = await listPersistedIndexes(directory);
|
|
23464
24316
|
for (const indexName of names) {
|
|
23465
24317
|
await recoverIndex(indexName, metadataOnly);
|
|
23466
24318
|
}
|
|
24319
|
+
await directoryWatch.verify();
|
|
24320
|
+
directoryWatch.start();
|
|
23467
24321
|
startCheckpointTimer();
|
|
23468
24322
|
},
|
|
23469
24323
|
recoverIndex(indexName) {
|
|
@@ -23525,6 +24379,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
23525
24379
|
},
|
|
23526
24380
|
async shutdown() {
|
|
23527
24381
|
shuttingDown = true;
|
|
24382
|
+
directoryWatch.stop();
|
|
23528
24383
|
if (checkpointTimer !== null) {
|
|
23529
24384
|
clearInterval(checkpointTimer);
|
|
23530
24385
|
checkpointTimer = null;
|
|
@@ -23543,6 +24398,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
23543
24398
|
}
|
|
23544
24399
|
indexes.clear();
|
|
23545
24400
|
terminateCheckpointWorker();
|
|
24401
|
+
await directoryWatch.release();
|
|
23546
24402
|
}
|
|
23547
24403
|
};
|
|
23548
24404
|
}
|
|
@@ -24973,18 +25829,6 @@ function eligibleIndexNames(state) {
|
|
|
24973
25829
|
return names;
|
|
24974
25830
|
}
|
|
24975
25831
|
|
|
24976
|
-
// src/engine/orchestration/turn.ts
|
|
24977
|
-
function afterCurrentTurn(callback) {
|
|
24978
|
-
if (typeof setImmediate === "function") {
|
|
24979
|
-
setImmediate(callback);
|
|
24980
|
-
return;
|
|
24981
|
-
}
|
|
24982
|
-
setTimeout(callback, 0);
|
|
24983
|
-
}
|
|
24984
|
-
function yieldToEventLoop6() {
|
|
24985
|
-
return new Promise((resolve) => afterCurrentTurn(resolve));
|
|
24986
|
-
}
|
|
24987
|
-
|
|
24988
25832
|
// src/engine/orchestration/replication.ts
|
|
24989
25833
|
async function dispatchToWorkers(state, action) {
|
|
24990
25834
|
const pool = state.workerPool;
|
|
@@ -25142,7 +25986,7 @@ async function replicateToWorkers(state, action) {
|
|
|
25142
25986
|
const loading = state.copyLoadBuffers.get(action.indexName);
|
|
25143
25987
|
if (loading !== void 0) {
|
|
25144
25988
|
queueForCopies(state, detached);
|
|
25145
|
-
if (loading.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await
|
|
25989
|
+
if (loading.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop();
|
|
25146
25990
|
return;
|
|
25147
25991
|
}
|
|
25148
25992
|
const queue = state.replicationQueues.get(action.indexName);
|
|
@@ -25584,17 +26428,14 @@ function createWorkerExecutor(worker, config) {
|
|
|
25584
26428
|
}
|
|
25585
26429
|
|
|
25586
26430
|
// src/workers/factory.browser.ts
|
|
25587
|
-
function
|
|
25588
|
-
const
|
|
25589
|
-
|
|
25590
|
-
|
|
25591
|
-
return new URL("workers/entry.mjs", base.slice(0, distIndex + 6)).href;
|
|
25592
|
-
}
|
|
25593
|
-
return base.replace(/\/src\/workers\/[^/]+$/, "/dist/workers/entry.mjs");
|
|
26431
|
+
async function requireWorkerEntry() {
|
|
26432
|
+
const entry = resolveWorkerEntry(import.meta.url, /\/src\/workers\/[^/]+$/, "workers/entry.mjs");
|
|
26433
|
+
if (entry !== null) return entry;
|
|
26434
|
+
throw missingWorkerEntry(import.meta.url);
|
|
25594
26435
|
}
|
|
25595
26436
|
async function createWorkerFactory(entryPoint) {
|
|
25596
26437
|
const runtime = detectRuntime();
|
|
25597
|
-
const resolvedEntry =
|
|
26438
|
+
const resolvedEntry = await requireWorkerEntry();
|
|
25598
26439
|
if (runtime.supportsWebWorkers) {
|
|
25599
26440
|
return function webFactory(_workerId, onDeath, onGone) {
|
|
25600
26441
|
const instance = new Worker(resolvedEntry, { type: "module" });
|
|
@@ -25784,10 +26625,13 @@ function createWorkerPool(config) {
|
|
|
25784
26625
|
}
|
|
25785
26626
|
isShutdown = true;
|
|
25786
26627
|
const shutdownPromises = [...threadsStillRunning].map((executor) => {
|
|
26628
|
+
let timeoutId;
|
|
25787
26629
|
const timeoutPromise = new Promise((resolve) => {
|
|
25788
|
-
setTimeout(resolve,
|
|
26630
|
+
timeoutId = setTimeout(resolve, SHUTDOWN_TIMEOUT_MS);
|
|
25789
26631
|
});
|
|
25790
|
-
return Promise.race([shutDownUntilTheThreadIsGone(executor), timeoutPromise])
|
|
26632
|
+
return Promise.race([shutDownUntilTheThreadIsGone(executor), timeoutPromise]).finally(
|
|
26633
|
+
() => clearTimeout(timeoutId)
|
|
26634
|
+
);
|
|
25791
26635
|
});
|
|
25792
26636
|
await Promise.allSettled(shutdownPromises);
|
|
25793
26637
|
indexAssignment.clear();
|
|
@@ -26223,8 +27067,12 @@ async function ensurePool(state) {
|
|
|
26223
27067
|
function documentCountOf(state, indexName) {
|
|
26224
27068
|
return state.executor.getManager(indexName)?.countDocuments() ?? 0;
|
|
26225
27069
|
}
|
|
26226
|
-
function copyThresholdReason(state, indexName) {
|
|
26227
|
-
|
|
27070
|
+
function copyThresholdReason(state, indexName, incomingCount = 0) {
|
|
27071
|
+
const held = documentCountOf(state, indexName);
|
|
27072
|
+
if (incomingCount <= 0) {
|
|
27073
|
+
return `Index "${indexName}" holds ${held} documents, reaching the copy threshold of ${state.copyThreshold}`;
|
|
27074
|
+
}
|
|
27075
|
+
return `Index "${indexName}" holds ${held} documents, and the batch that the engine is writing adds ${incomingCount}, for ${held + incomingCount} against the copy threshold of ${state.copyThreshold}`;
|
|
26228
27076
|
}
|
|
26229
27077
|
function indexReadyForCopies(state, indexName, incomingCount = 0) {
|
|
26230
27078
|
if (!copiesAllowed(state)) return false;
|
|
@@ -26313,7 +27161,7 @@ async function scaleOutBeforeBatch(state, indexName, incomingCount) {
|
|
|
26313
27161
|
return;
|
|
26314
27162
|
}
|
|
26315
27163
|
if (!indexReadyForCopies(state, indexName, incomingCount)) return;
|
|
26316
|
-
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
27164
|
+
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName, incomingCount));
|
|
26317
27165
|
}
|
|
26318
27166
|
|
|
26319
27167
|
// src/engine/orchestration/idle.ts
|
|
@@ -26393,7 +27241,7 @@ function partitionsPerLease(scope, leaseCount) {
|
|
|
26393
27241
|
}
|
|
26394
27242
|
return assignments;
|
|
26395
27243
|
}
|
|
26396
|
-
function mergeWorkerResults(results) {
|
|
27244
|
+
function mergeWorkerResults(results, facetConfig) {
|
|
26397
27245
|
const merged = kWayMerge(results.map((result) => result.scored));
|
|
26398
27246
|
let totalMatched = 0;
|
|
26399
27247
|
const workerFacets = [];
|
|
@@ -26404,7 +27252,7 @@ function mergeWorkerResults(results) {
|
|
|
26404
27252
|
return {
|
|
26405
27253
|
scored: merged,
|
|
26406
27254
|
totalMatched,
|
|
26407
|
-
facets: workerFacets.length > 0 ? mergeFacets(workerFacets) : void 0
|
|
27255
|
+
facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0
|
|
26408
27256
|
};
|
|
26409
27257
|
}
|
|
26410
27258
|
function scoresPerPartition(state, indexName, params, globalStats) {
|
|
@@ -26424,12 +27272,13 @@ function queryAction(indexName, params, globalStats, partitionIds) {
|
|
|
26424
27272
|
}
|
|
26425
27273
|
async function runSplit(leases, scope, indexName, params, globalStats) {
|
|
26426
27274
|
const assignments = partitionsPerLease(scope, leases.length);
|
|
27275
|
+
const workerParams = params.facets !== void 0 ? { ...params, facets: oversampledFacetConfig(params.facets) } : params;
|
|
26427
27276
|
const results = await Promise.all(
|
|
26428
27277
|
assignments.map(
|
|
26429
|
-
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName,
|
|
27278
|
+
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds)).finally(() => leases[at].release())
|
|
26430
27279
|
)
|
|
26431
27280
|
);
|
|
26432
|
-
return mergeWorkerResults(results);
|
|
27281
|
+
return mergeWorkerResults(results, params.facets);
|
|
26433
27282
|
}
|
|
26434
27283
|
function takeMainCopyTurn(state) {
|
|
26435
27284
|
if (state.mainCopyQueries === "none" || state.mainCopyTurnTaken) return false;
|
|
@@ -26915,7 +27764,7 @@ function createEngineCore(config, hooks) {
|
|
|
26915
27764
|
const rebalanceTargets = /* @__PURE__ */ new Map();
|
|
26916
27765
|
function guardShutdown() {
|
|
26917
27766
|
if (shutdownState.isShutdown) {
|
|
26918
|
-
throw new NarsilError(ErrorCodes.
|
|
27767
|
+
throw new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
26919
27768
|
}
|
|
26920
27769
|
}
|
|
26921
27770
|
function requireIndex(indexName) {
|
|
@@ -27082,6 +27931,81 @@ function createEngineCore(config, hooks) {
|
|
|
27082
27931
|
};
|
|
27083
27932
|
}
|
|
27084
27933
|
|
|
27934
|
+
// src/engine/lifecycle.ts
|
|
27935
|
+
async function startEngineCore(core, config) {
|
|
27936
|
+
try {
|
|
27937
|
+
if (core.durability) {
|
|
27938
|
+
await core.durability.manager.recover(config?.lifecycle !== void 0);
|
|
27939
|
+
checkHeapAfterRecovery(core);
|
|
27940
|
+
}
|
|
27941
|
+
if (core.invalidation) await core.invalidation.start();
|
|
27942
|
+
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
27943
|
+
} catch (err) {
|
|
27944
|
+
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
27945
|
+
await shutdownEngine(core).catch(() => void 0);
|
|
27946
|
+
throw err;
|
|
27947
|
+
}
|
|
27948
|
+
}
|
|
27949
|
+
async function shutdownEngine(core) {
|
|
27950
|
+
const { executor, durability, indexRegistry, eventHandlers, orchestrator } = core;
|
|
27951
|
+
core.indexState.dispose();
|
|
27952
|
+
for (const [name] of indexRegistry) {
|
|
27953
|
+
const manager = executor.getManager(name);
|
|
27954
|
+
if (manager) {
|
|
27955
|
+
for (const [, vectorIndex] of manager.getVectorIndexes()) {
|
|
27956
|
+
vectorIndex.dispose();
|
|
27957
|
+
}
|
|
27958
|
+
}
|
|
27959
|
+
}
|
|
27960
|
+
if (durability) {
|
|
27961
|
+
await durability.manager.shutdown();
|
|
27962
|
+
}
|
|
27963
|
+
if (core.invalidation) {
|
|
27964
|
+
await core.invalidation.shutdown();
|
|
27965
|
+
}
|
|
27966
|
+
const adaptersToShutdown = /* @__PURE__ */ new Set();
|
|
27967
|
+
for (const [, entry] of indexRegistry) {
|
|
27968
|
+
if (entry.embeddingAdapter?.shutdown) {
|
|
27969
|
+
adaptersToShutdown.add(entry.embeddingAdapter);
|
|
27970
|
+
}
|
|
27971
|
+
}
|
|
27972
|
+
for (const adapter of adaptersToShutdown) {
|
|
27973
|
+
try {
|
|
27974
|
+
await adapter.shutdown?.();
|
|
27975
|
+
} catch {
|
|
27976
|
+
}
|
|
27977
|
+
}
|
|
27978
|
+
await orchestrator.shutdown();
|
|
27979
|
+
await executor.shutdown();
|
|
27980
|
+
eventHandlers.clear();
|
|
27981
|
+
indexRegistry.clear();
|
|
27982
|
+
}
|
|
27983
|
+
|
|
27984
|
+
// src/engine/startup-checks.ts
|
|
27985
|
+
var announcedMissingWorkerEntry = false;
|
|
27986
|
+
function withoutWorkers(config, reason) {
|
|
27987
|
+
if (!announcedMissingWorkerEntry) {
|
|
27988
|
+
announcedMissingWorkerEntry = true;
|
|
27989
|
+
console.warn(
|
|
27990
|
+
`Narsil answers every query on this thread, because it finds no worker entry beside its own module at "${String(reason.details.moduleUrl)}", which happens where a bundler folds @delali/narsil into an application bundle. Keep @delali/narsil outside the bundle to use worker threads, or set workers.enabled to false to silence this notice.`
|
|
27991
|
+
);
|
|
27992
|
+
}
|
|
27993
|
+
return { ...config, workers: { ...config?.workers, enabled: false } };
|
|
27994
|
+
}
|
|
27995
|
+
async function resolveRunnableConfig(config) {
|
|
27996
|
+
if (!(config?.workers?.enabled ?? workersEnabledByDefault())) return config;
|
|
27997
|
+
const runtime = detectRuntime();
|
|
27998
|
+
if (!runtime.supportsWorkerThreads && !runtime.supportsWebWorkers) return config;
|
|
27999
|
+
try {
|
|
28000
|
+
await requireWorkerEntry();
|
|
28001
|
+
return config;
|
|
28002
|
+
} catch (err) {
|
|
28003
|
+
const missingEntry = err instanceof NarsilError && err.code === ErrorCodes.CONFIG_INVALID;
|
|
28004
|
+
if (!missingEntry || config?.workers?.enabled === true) throw err;
|
|
28005
|
+
return withoutWorkers(config, err);
|
|
28006
|
+
}
|
|
28007
|
+
}
|
|
28008
|
+
|
|
27085
28009
|
// src/engine/index-lifecycle.ts
|
|
27086
28010
|
async function runLifecycleHook(core, hookName, context) {
|
|
27087
28011
|
try {
|
|
@@ -27215,42 +28139,6 @@ function registerEngineEmbeddingAdapter(core, name, adapter) {
|
|
|
27215
28139
|
}
|
|
27216
28140
|
}
|
|
27217
28141
|
|
|
27218
|
-
// src/engine/lifecycle.ts
|
|
27219
|
-
async function shutdownEngine(core) {
|
|
27220
|
-
const { executor, durability, indexRegistry, eventHandlers, orchestrator } = core;
|
|
27221
|
-
core.indexState.dispose();
|
|
27222
|
-
for (const [name] of indexRegistry) {
|
|
27223
|
-
const manager = executor.getManager(name);
|
|
27224
|
-
if (manager) {
|
|
27225
|
-
for (const [, vectorIndex] of manager.getVectorIndexes()) {
|
|
27226
|
-
vectorIndex.dispose();
|
|
27227
|
-
}
|
|
27228
|
-
}
|
|
27229
|
-
}
|
|
27230
|
-
if (durability) {
|
|
27231
|
-
await durability.manager.shutdown();
|
|
27232
|
-
}
|
|
27233
|
-
if (core.invalidation) {
|
|
27234
|
-
await core.invalidation.shutdown();
|
|
27235
|
-
}
|
|
27236
|
-
const adaptersToShutdown = /* @__PURE__ */ new Set();
|
|
27237
|
-
for (const [, entry] of indexRegistry) {
|
|
27238
|
-
if (entry.embeddingAdapter?.shutdown) {
|
|
27239
|
-
adaptersToShutdown.add(entry.embeddingAdapter);
|
|
27240
|
-
}
|
|
27241
|
-
}
|
|
27242
|
-
for (const adapter of adaptersToShutdown) {
|
|
27243
|
-
try {
|
|
27244
|
-
await adapter.shutdown?.();
|
|
27245
|
-
} catch {
|
|
27246
|
-
}
|
|
27247
|
-
}
|
|
27248
|
-
await orchestrator.shutdown();
|
|
27249
|
-
await executor.shutdown();
|
|
27250
|
-
eventHandlers.clear();
|
|
27251
|
-
indexRegistry.clear();
|
|
27252
|
-
}
|
|
27253
|
-
|
|
27254
28142
|
// src/engine/embed.ts
|
|
27255
28143
|
function assertDocumentCarriesMappedVectors(document2, embeddingConfig, adapterName) {
|
|
27256
28144
|
for (const targetField of Object.keys(embeddingConfig.fields)) {
|
|
@@ -27919,6 +28807,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
|
|
|
27919
28807
|
}
|
|
27920
28808
|
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
27921
28809
|
if (applied < admitted.length) {
|
|
28810
|
+
ctx.checkHeapPressure(indexName);
|
|
27922
28811
|
await new Promise((r) => setTimeout(r, 0));
|
|
27923
28812
|
}
|
|
27924
28813
|
}
|
|
@@ -28158,6 +29047,7 @@ async function recordMergedDocuments(ctx, indexName, built, frozen, memberIndexe
|
|
|
28158
29047
|
succeededIds.add(doc.docId);
|
|
28159
29048
|
}
|
|
28160
29049
|
if (index + 1 < chunks.length) {
|
|
29050
|
+
ctx.checkHeapPressure(indexName);
|
|
28161
29051
|
await new Promise((r) => setTimeout(r, 0));
|
|
28162
29052
|
}
|
|
28163
29053
|
}
|
|
@@ -28369,6 +29259,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
28369
29259
|
}
|
|
28370
29260
|
if (prepared.length > 0) await applyPrepared(prepared);
|
|
28371
29261
|
if (chunkEnd < documents.length) {
|
|
29262
|
+
ctx.checkHeapPressure(indexName);
|
|
28372
29263
|
await new Promise((r) => setTimeout(r, 0));
|
|
28373
29264
|
}
|
|
28374
29265
|
}
|
|
@@ -28566,6 +29457,18 @@ async function settleUpdate(ctx, indexName, prepared) {
|
|
|
28566
29457
|
async function prepareUpdate(ctx, indexName, entry, docId, document2) {
|
|
28567
29458
|
ctx.guardShutdown();
|
|
28568
29459
|
validateDocId(docId);
|
|
29460
|
+
const manager = ctx.requireManager(indexName);
|
|
29461
|
+
if (ctx.pluginRegistry.hasHooks("beforeUpdate")) {
|
|
29462
|
+
await ctx.pluginRegistry.runHook("beforeUpdate", {
|
|
29463
|
+
indexName,
|
|
29464
|
+
docId,
|
|
29465
|
+
oldDocument: manager.get(docId) ?? {},
|
|
29466
|
+
newDocument: document2
|
|
29467
|
+
});
|
|
29468
|
+
}
|
|
29469
|
+
if (entry.config.required && entry.config.required.length > 0) {
|
|
29470
|
+
validateRequiredFields(document2, entry.config.required);
|
|
29471
|
+
}
|
|
28569
29472
|
if (entry.config.embedding) {
|
|
28570
29473
|
if (entry.embeddingAdapter) {
|
|
28571
29474
|
await embedDocumentFields(
|
|
@@ -28582,16 +29485,9 @@ async function prepareUpdate(ctx, indexName, entry, docId, document2) {
|
|
|
28582
29485
|
);
|
|
28583
29486
|
}
|
|
28584
29487
|
}
|
|
28585
|
-
const manager = ctx.requireManager(indexName);
|
|
28586
29488
|
const oldDocument = manager.get(docId);
|
|
28587
29489
|
const oldPartitionDoc = manager.getRef(docId);
|
|
28588
29490
|
const rollbackDoc = oldPartitionDoc ? structuredClone(oldPartitionDoc) : void 0;
|
|
28589
|
-
await ctx.pluginRegistry.runHook("beforeUpdate", {
|
|
28590
|
-
indexName,
|
|
28591
|
-
docId,
|
|
28592
|
-
oldDocument: oldDocument ?? {},
|
|
28593
|
-
newDocument: document2
|
|
28594
|
-
});
|
|
28595
29491
|
const vecIndexes = manager.getVectorIndexes();
|
|
28596
29492
|
const extractedVectors = /* @__PURE__ */ new Map();
|
|
28597
29493
|
if (vecIndexes.size > 0) {
|
|
@@ -28710,6 +29606,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
28710
29606
|
succeeded.push(update.docId);
|
|
28711
29607
|
}
|
|
28712
29608
|
if (chunkEnd < updates.length) {
|
|
29609
|
+
ctx.checkHeapPressure(indexName);
|
|
28713
29610
|
await new Promise((r) => setTimeout(r, 0));
|
|
28714
29611
|
}
|
|
28715
29612
|
}
|
|
@@ -28954,7 +29851,7 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
28954
29851
|
await replayQueued(manager, indexName, waq, ctx);
|
|
28955
29852
|
if (ctx.durabilityManager) {
|
|
28956
29853
|
await ctx.durabilityManager.persistMetadata(indexName);
|
|
28957
|
-
await ctx.durabilityManager.checkpoint(indexName);
|
|
29854
|
+
await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
|
|
28958
29855
|
await replayQueued(manager, indexName, waq, ctx);
|
|
28959
29856
|
}
|
|
28960
29857
|
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
@@ -28985,7 +29882,7 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
28985
29882
|
}
|
|
28986
29883
|
|
|
28987
29884
|
// src/engine/snapshot/restore-config.ts
|
|
28988
|
-
function
|
|
29885
|
+
function invalid5(message) {
|
|
28989
29886
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
|
|
28990
29887
|
}
|
|
28991
29888
|
function isPlainObject3(value) {
|
|
@@ -28996,14 +29893,14 @@ function restoredBm25(raw) {
|
|
|
28996
29893
|
return void 0;
|
|
28997
29894
|
}
|
|
28998
29895
|
if (!isPlainObject3(raw)) {
|
|
28999
|
-
|
|
29896
|
+
invalid5("bm25 must be an object");
|
|
29000
29897
|
}
|
|
29001
29898
|
const { k1, b } = raw;
|
|
29002
29899
|
if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
|
|
29003
|
-
|
|
29900
|
+
invalid5("bm25 k1 must be a finite number");
|
|
29004
29901
|
}
|
|
29005
29902
|
if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
|
|
29006
|
-
|
|
29903
|
+
invalid5("bm25 b must be a finite number");
|
|
29007
29904
|
}
|
|
29008
29905
|
return {
|
|
29009
29906
|
...k1 !== void 0 ? { k1 } : {},
|
|
@@ -29012,13 +29909,13 @@ function restoredBm25(raw) {
|
|
|
29012
29909
|
}
|
|
29013
29910
|
function restoredStopWords(name, list) {
|
|
29014
29911
|
if (name !== void 0 && typeof name !== "string") {
|
|
29015
|
-
|
|
29912
|
+
invalid5("stopWords must be a name");
|
|
29016
29913
|
}
|
|
29017
29914
|
if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
|
|
29018
|
-
|
|
29915
|
+
invalid5("stopWordList must be a list of words");
|
|
29019
29916
|
}
|
|
29020
29917
|
if (typeof name === "string" && Array.isArray(list)) {
|
|
29021
|
-
|
|
29918
|
+
invalid5("stopWords and stopWordList cannot both be present");
|
|
29022
29919
|
}
|
|
29023
29920
|
if (typeof name === "string") {
|
|
29024
29921
|
return name;
|
|
@@ -29033,17 +29930,17 @@ function restoredPartitionConfig(raw) {
|
|
|
29033
29930
|
return void 0;
|
|
29034
29931
|
}
|
|
29035
29932
|
if (!isPlainObject3(raw)) {
|
|
29036
|
-
|
|
29933
|
+
invalid5("partitionConfig must be an object");
|
|
29037
29934
|
}
|
|
29038
29935
|
const { maxDocsPerPartition, maxPartitions, watermark } = raw;
|
|
29039
29936
|
if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
|
|
29040
|
-
|
|
29937
|
+
invalid5("partitionConfig maxDocsPerPartition must be a positive integer");
|
|
29041
29938
|
}
|
|
29042
29939
|
if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
|
|
29043
|
-
|
|
29940
|
+
invalid5("partitionConfig maxPartitions must be a positive integer");
|
|
29044
29941
|
}
|
|
29045
29942
|
if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
|
|
29046
|
-
|
|
29943
|
+
invalid5("partitionConfig watermark must be above 0 and at most 1");
|
|
29047
29944
|
}
|
|
29048
29945
|
return {
|
|
29049
29946
|
...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
|
|
@@ -29056,7 +29953,7 @@ function restoredScoring(raw) {
|
|
|
29056
29953
|
return void 0;
|
|
29057
29954
|
}
|
|
29058
29955
|
if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
|
|
29059
|
-
|
|
29956
|
+
invalid5("defaultScoring must be 'local', 'dfs', or 'broadcast'");
|
|
29060
29957
|
}
|
|
29061
29958
|
return raw;
|
|
29062
29959
|
}
|
|
@@ -29065,7 +29962,7 @@ function restoredBoolean(raw, field) {
|
|
|
29065
29962
|
return void 0;
|
|
29066
29963
|
}
|
|
29067
29964
|
if (typeof raw !== "boolean") {
|
|
29068
|
-
|
|
29965
|
+
invalid5(`${field} must be a boolean`);
|
|
29069
29966
|
}
|
|
29070
29967
|
return raw;
|
|
29071
29968
|
}
|
|
@@ -29074,7 +29971,7 @@ function restoredRequired(raw) {
|
|
|
29074
29971
|
return void 0;
|
|
29075
29972
|
}
|
|
29076
29973
|
if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
|
|
29077
|
-
|
|
29974
|
+
invalid5("required must be a list of field paths");
|
|
29078
29975
|
}
|
|
29079
29976
|
return raw;
|
|
29080
29977
|
}
|
|
@@ -29083,7 +29980,7 @@ function restoredVectorPromotion(raw) {
|
|
|
29083
29980
|
return void 0;
|
|
29084
29981
|
}
|
|
29085
29982
|
if (!isPlainObject3(raw) || raw.hnswConfig !== void 0 && !isPlainObject3(raw.hnswConfig)) {
|
|
29086
|
-
|
|
29983
|
+
invalid5("vectorPromotion must be an object");
|
|
29087
29984
|
}
|
|
29088
29985
|
return raw;
|
|
29089
29986
|
}
|
|
@@ -29092,20 +29989,20 @@ function restoredEmbedding(raw) {
|
|
|
29092
29989
|
return void 0;
|
|
29093
29990
|
}
|
|
29094
29991
|
if (!isPlainObject3(raw)) {
|
|
29095
|
-
|
|
29992
|
+
invalid5("embedding must be an object");
|
|
29096
29993
|
}
|
|
29097
29994
|
const { fields, adapter } = raw;
|
|
29098
29995
|
if (adapter !== void 0 && typeof adapter !== "string") {
|
|
29099
|
-
|
|
29996
|
+
invalid5("embedding adapter must be a name");
|
|
29100
29997
|
}
|
|
29101
29998
|
if (!isPlainObject3(fields)) {
|
|
29102
|
-
|
|
29999
|
+
invalid5("embedding fields must be an object");
|
|
29103
30000
|
}
|
|
29104
30001
|
for (const value of Object.values(fields)) {
|
|
29105
30002
|
const isPath = typeof value === "string";
|
|
29106
30003
|
const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
|
|
29107
30004
|
if (!isPath && !isPathList) {
|
|
29108
|
-
|
|
30005
|
+
invalid5("embedding fields must map vector fields to source paths");
|
|
29109
30006
|
}
|
|
29110
30007
|
}
|
|
29111
30008
|
return {
|
|
@@ -29115,7 +30012,7 @@ function restoredEmbedding(raw) {
|
|
|
29115
30012
|
}
|
|
29116
30013
|
function restoredConfigFields(envelope) {
|
|
29117
30014
|
if (envelope.tokenizer !== void 0 && typeof envelope.tokenizer !== "string") {
|
|
29118
|
-
|
|
30015
|
+
invalid5("tokenizer must be a name");
|
|
29119
30016
|
}
|
|
29120
30017
|
const stopWords2 = restoredStopWords(envelope.stopWords, envelope.stopWordList);
|
|
29121
30018
|
const bm25 = restoredBm25(envelope.bm25);
|
|
@@ -29596,13 +30493,176 @@ function listBindingOf(filters) {
|
|
|
29596
30493
|
return bindingOf(values);
|
|
29597
30494
|
}
|
|
29598
30495
|
|
|
29599
|
-
// src/
|
|
29600
|
-
function
|
|
29601
|
-
const
|
|
29602
|
-
|
|
29603
|
-
|
|
30496
|
+
// src/search/query-fields.ts
|
|
30497
|
+
function requireBoostableFields(boost, flatSchema) {
|
|
30498
|
+
for (const [field, weight] of Object.entries(boost)) {
|
|
30499
|
+
const fieldType = flatSchema[field];
|
|
30500
|
+
if (fieldType === void 0 || !isTextFieldType(fieldType) && fieldType !== "string[]") {
|
|
30501
|
+
throw new NarsilError(
|
|
30502
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
30503
|
+
fieldType === void 0 ? `The engine boosts a text field that the schema declares, and the schema declares no field "${field}"` : `The engine boosts a text field alone, and the schema declares field "${field}" as "${fieldType}"`,
|
|
30504
|
+
{ field, ...fieldType === void 0 ? {} : { fieldType } }
|
|
30505
|
+
);
|
|
30506
|
+
}
|
|
30507
|
+
if (typeof weight !== "number" || !Number.isFinite(weight)) {
|
|
30508
|
+
throw new NarsilError(
|
|
30509
|
+
ErrorCodes.CONFIG_INVALID,
|
|
30510
|
+
`The boost on field "${field}" must be a finite number, and this query sets ${String(weight)}`,
|
|
30511
|
+
{ field, boost: String(weight) }
|
|
30512
|
+
);
|
|
30513
|
+
}
|
|
30514
|
+
}
|
|
29604
30515
|
}
|
|
29605
|
-
function
|
|
30516
|
+
function requireFacetableFields(facets, flatSchema, strict) {
|
|
30517
|
+
for (const field of Object.keys(facets)) {
|
|
30518
|
+
const fieldType = flatSchema[field];
|
|
30519
|
+
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(field, "facet");
|
|
30520
|
+
if (isGeopointOrVectorType(fieldType)) {
|
|
30521
|
+
throw new NarsilError(
|
|
30522
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
30523
|
+
`The engine counts the values of a text, number, boolean, or enum field, and the schema declares field "${field}" as "${fieldType}"`,
|
|
30524
|
+
{ field, fieldType }
|
|
30525
|
+
);
|
|
30526
|
+
}
|
|
30527
|
+
}
|
|
30528
|
+
}
|
|
30529
|
+
function requireDeclaredGroupFields(fields, flatSchema) {
|
|
30530
|
+
for (const field of fields) {
|
|
30531
|
+
if (flatSchema[field] === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
|
|
30532
|
+
}
|
|
30533
|
+
}
|
|
30534
|
+
function requireUsableQueryFields(params, schema, strict = false) {
|
|
30535
|
+
const checksGroup = strict && Array.isArray(params.group?.fields);
|
|
30536
|
+
if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
|
|
30537
|
+
const flatSchema = flattenSchema(schema);
|
|
30538
|
+
if (params.boost !== void 0) requireBoostableFields(params.boost, flatSchema);
|
|
30539
|
+
if (params.facets !== void 0) requireFacetableFields(params.facets, flatSchema, strict);
|
|
30540
|
+
if (checksGroup && params.group !== void 0) requireDeclaredGroupFields(params.group.fields, flatSchema);
|
|
30541
|
+
}
|
|
30542
|
+
|
|
30543
|
+
// src/engine/query/shared.ts
|
|
30544
|
+
function partitionsFor(manager, partitionIds) {
|
|
30545
|
+
return partitionsIn(manager, partitionIds);
|
|
30546
|
+
}
|
|
30547
|
+
function coverageFor(manager, partitionIds) {
|
|
30548
|
+
if (partitionIds === void 0) {
|
|
30549
|
+
return {
|
|
30550
|
+
totalPartitions: manager.partitionCount,
|
|
30551
|
+
queriedPartitions: manager.partitionCount,
|
|
30552
|
+
timedOutPartitions: 0,
|
|
30553
|
+
failedPartitions: 0
|
|
30554
|
+
};
|
|
30555
|
+
}
|
|
30556
|
+
const queriedPartitions = partitionsFor(manager, partitionIds).length;
|
|
30557
|
+
return {
|
|
30558
|
+
totalPartitions: partitionIds.length,
|
|
30559
|
+
queriedPartitions,
|
|
30560
|
+
timedOutPartitions: 0,
|
|
30561
|
+
failedPartitions: partitionIds.length - queriedPartitions
|
|
30562
|
+
};
|
|
30563
|
+
}
|
|
30564
|
+
function partitionsForVectorSearch(manager, vecIndex, partitionIds) {
|
|
30565
|
+
if (partitionIds === void 0) {
|
|
30566
|
+
return void 0;
|
|
30567
|
+
}
|
|
30568
|
+
if (!vecIndex.partitionsKnown()) {
|
|
30569
|
+
vecIndex.assignPartitions((docId) => manager.partitionIdOf(docId));
|
|
30570
|
+
}
|
|
30571
|
+
return new Set(partitionIds);
|
|
30572
|
+
}
|
|
30573
|
+
function scoringConfigFor(params, context) {
|
|
30574
|
+
const scoringMode = params.scoring ?? context.config.defaultScoring ?? "local";
|
|
30575
|
+
if (scoringMode !== "broadcast") {
|
|
30576
|
+
return { scoringMode, partitionIds: context.partitionIds };
|
|
30577
|
+
}
|
|
30578
|
+
return { scoringMode, globalStats: context.broadcastStats?.(context.indexName), partitionIds: context.partitionIds };
|
|
30579
|
+
}
|
|
30580
|
+
function broadcastStatsForWorker(params, context, scoring) {
|
|
30581
|
+
if (scoring.scoringMode !== "broadcast" || scoring.globalStats === void 0) {
|
|
30582
|
+
return void 0;
|
|
30583
|
+
}
|
|
30584
|
+
const term = params.term;
|
|
30585
|
+
if (term === void 0 || term.trim().length === 0) {
|
|
30586
|
+
return void 0;
|
|
30587
|
+
}
|
|
30588
|
+
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
|
|
30589
|
+
}
|
|
30590
|
+
function searchOptionsFor(manager) {
|
|
30591
|
+
return {
|
|
30592
|
+
bm25Params: manager.config.bm25,
|
|
30593
|
+
stopWords: manager.analysis.stopWords,
|
|
30594
|
+
customTokenizer: manager.analysis.customTokenizer
|
|
30595
|
+
};
|
|
30596
|
+
}
|
|
30597
|
+
function collectFilterDocIds(manager, params, schema, partitionIds) {
|
|
30598
|
+
const filterDocIds = /* @__PURE__ */ new Set();
|
|
30599
|
+
if (!params.filters) return filterDocIds;
|
|
30600
|
+
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
30601
|
+
const partitionFiltered = partition.applyFilters(params.filters, schema);
|
|
30602
|
+
for (const docId of partitionFiltered) {
|
|
30603
|
+
filterDocIds.add(docId);
|
|
30604
|
+
}
|
|
30605
|
+
}
|
|
30606
|
+
return filterDocIds;
|
|
30607
|
+
}
|
|
30608
|
+
function vectorResultsToScored(results) {
|
|
30609
|
+
return results.map((r) => ({
|
|
30610
|
+
docId: r.docId,
|
|
30611
|
+
score: r.score,
|
|
30612
|
+
termFrequencies: {},
|
|
30613
|
+
fieldLengths: {},
|
|
30614
|
+
idf: {}
|
|
30615
|
+
}));
|
|
30616
|
+
}
|
|
30617
|
+
function vectorSearchersOf(context) {
|
|
30618
|
+
return context.vectorSearchers ?? context.manager.getVectorIndexes();
|
|
30619
|
+
}
|
|
30620
|
+
function resolveVectorIndex(context, fieldName) {
|
|
30621
|
+
return vectorSearchersOf(context).get(fieldName);
|
|
30622
|
+
}
|
|
30623
|
+
var SEARCH_MODES = /* @__PURE__ */ new Set(["fulltext", "vector", "hybrid"]);
|
|
30624
|
+
function requireKnownMode(params) {
|
|
30625
|
+
if (params.mode === void 0 || SEARCH_MODES.has(params.mode)) return;
|
|
30626
|
+
throw new NarsilError(
|
|
30627
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
30628
|
+
`A search takes the mode "fulltext", "vector", or "hybrid", while this query names "${String(params.mode)}"`,
|
|
30629
|
+
{ mode: String(params.mode), modes: [...SEARCH_MODES] }
|
|
30630
|
+
);
|
|
30631
|
+
}
|
|
30632
|
+
function requireValidQueryOptions(params, config) {
|
|
30633
|
+
const strict = config.strict === true;
|
|
30634
|
+
requireSortableFields(params.sort, config.schema, strict);
|
|
30635
|
+
requireValidQueryFilter(params.filters, config.schema, strict);
|
|
30636
|
+
requireUsableQueryFields(params, config.schema, strict);
|
|
30637
|
+
}
|
|
30638
|
+
function requireValidQueryFilter(filters, schema, strict = false) {
|
|
30639
|
+
if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema), strict);
|
|
30640
|
+
}
|
|
30641
|
+
function requireVectorSearchable(params, context, needsVector) {
|
|
30642
|
+
if (!needsVector) return;
|
|
30643
|
+
const field = params.vector?.field;
|
|
30644
|
+
if (field === void 0) {
|
|
30645
|
+
throw new NarsilError(
|
|
30646
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
30647
|
+
`A "${params.mode}" query needs a vector clause naming the field to search`,
|
|
30648
|
+
{ mode: params.mode }
|
|
30649
|
+
);
|
|
30650
|
+
}
|
|
30651
|
+
if (vectorSearchersOf(context).has(field)) return;
|
|
30652
|
+
throw new NarsilError(
|
|
30653
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
30654
|
+
`Field "${field}" has no vector index, so a vector search cannot use it`,
|
|
30655
|
+
{ field, vectorFields: [...vectorSearchersOf(context).keys()] }
|
|
30656
|
+
);
|
|
30657
|
+
}
|
|
30658
|
+
|
|
30659
|
+
// src/engine/list-documents/index.ts
|
|
30660
|
+
function clampListLimit(limit) {
|
|
30661
|
+
const clamped = Math.max(1, clampLimit(limit));
|
|
30662
|
+
requireWithinResultWindow(clamped, 0);
|
|
30663
|
+
return clamped;
|
|
30664
|
+
}
|
|
30665
|
+
function firstIdAfter(sorted, anchor) {
|
|
29606
30666
|
let low = 0;
|
|
29607
30667
|
let high = sorted.length;
|
|
29608
30668
|
while (low < high) {
|
|
@@ -29658,6 +30718,7 @@ function pageInSortOrder(partitions, cursor, filtered, limit, sort, signature, s
|
|
|
29658
30718
|
fields,
|
|
29659
30719
|
directions,
|
|
29660
30720
|
fieldTypes,
|
|
30721
|
+
modes: sortModesOf(normalized),
|
|
29661
30722
|
limit: limit + 1,
|
|
29662
30723
|
anchorKey: cursor === null ? null : cursor.sortKey,
|
|
29663
30724
|
anchorId: cursor === null ? null : cursor.anchor
|
|
@@ -29689,7 +30750,8 @@ function executeListDocuments(params, context) {
|
|
|
29689
30750
|
const startTime = now();
|
|
29690
30751
|
const limit = clampListLimit(params.limit);
|
|
29691
30752
|
const signature = sortSignatureOf(params.sort);
|
|
29692
|
-
requireSortableFields(params.sort, schema);
|
|
30753
|
+
requireSortableFields(params.sort, schema, context.strict === true);
|
|
30754
|
+
requireValidQueryFilter(params.filters, schema, context.strict === true);
|
|
29693
30755
|
const binding = listBindingOf(params.filters);
|
|
29694
30756
|
let cursor = null;
|
|
29695
30757
|
if (params.cursor !== void 0) {
|
|
@@ -29737,6 +30799,10 @@ function getNestedValue2(obj, path) {
|
|
|
29737
30799
|
}
|
|
29738
30800
|
return current;
|
|
29739
30801
|
}
|
|
30802
|
+
function hitsKeptPerGroup(maxPerGroup) {
|
|
30803
|
+
if (maxPerGroup === void 0 || !Number.isFinite(maxPerGroup) || maxPerGroup < 1) return DEFAULT_MAX_PER_GROUP;
|
|
30804
|
+
return Math.floor(maxPerGroup);
|
|
30805
|
+
}
|
|
29740
30806
|
function applyGrouping(hits, group, getDocument) {
|
|
29741
30807
|
if (!group.fields || group.fields.length === 0) {
|
|
29742
30808
|
return [{ values: {}, hits }];
|
|
@@ -29760,16 +30826,14 @@ function applyGrouping(hits, group, getDocument) {
|
|
|
29760
30826
|
}
|
|
29761
30827
|
}
|
|
29762
30828
|
const groups = [];
|
|
30829
|
+
const keptPerGroup = hitsKeptPerGroup(group.maxPerGroup);
|
|
29763
30830
|
for (const entry of groupMap.values()) {
|
|
29764
|
-
|
|
29765
|
-
|
|
29766
|
-
groupHits = groupHits.slice(0, group.maxPerGroup);
|
|
29767
|
-
}
|
|
29768
|
-
const result = { values: entry.values, hits: groupHits };
|
|
30831
|
+
const everyHit = entry.hits;
|
|
30832
|
+
const result = { values: entry.values, hits: everyHit.slice(0, keptPerGroup) };
|
|
29769
30833
|
if (group.reduce) {
|
|
29770
30834
|
const folded = foldGroupReducer(
|
|
29771
30835
|
group.reduce,
|
|
29772
|
-
|
|
30836
|
+
everyHit.map((hit) => ({ document: getDocument(hit.id), score: hit.score ?? 0 }))
|
|
29773
30837
|
);
|
|
29774
30838
|
if (folded.reducerError !== void 0) {
|
|
29775
30839
|
result.reducerError = folded.reducerError;
|
|
@@ -29820,23 +30884,27 @@ function dedupePinnedEntries(pinned) {
|
|
|
29820
30884
|
}
|
|
29821
30885
|
return deduped;
|
|
29822
30886
|
}
|
|
29823
|
-
function
|
|
30887
|
+
function placePinned(hits, pinned, resolveDoc) {
|
|
29824
30888
|
const result = hits.slice();
|
|
29825
30889
|
const deduped = dedupePinnedEntries(pinned);
|
|
29826
30890
|
const pinnedDocIds = new Set(deduped.map((entry) => entry.docId));
|
|
30891
|
+
const listedPins = /* @__PURE__ */ new Set();
|
|
29827
30892
|
for (let i = result.length - 1; i >= 0; i--) {
|
|
29828
30893
|
if (pinnedDocIds.has(result[i].id)) {
|
|
30894
|
+
listedPins.add(result[i].id);
|
|
29829
30895
|
result.splice(i, 1);
|
|
29830
30896
|
}
|
|
29831
30897
|
}
|
|
29832
30898
|
const sorted = deduped.sort((a, b) => a.position - b.position);
|
|
30899
|
+
let placedFromOutside = 0;
|
|
29833
30900
|
for (const entry of sorted) {
|
|
29834
30901
|
const doc = resolveDoc(entry.docId);
|
|
29835
30902
|
if (!doc) continue;
|
|
30903
|
+
if (!listedPins.has(entry.docId)) placedFromOutside++;
|
|
29836
30904
|
const pos = Math.max(0, Math.min(entry.position, result.length));
|
|
29837
30905
|
result.splice(pos, 0, doc);
|
|
29838
30906
|
}
|
|
29839
|
-
return result;
|
|
30907
|
+
return { hits: result, placedFromOutside };
|
|
29840
30908
|
}
|
|
29841
30909
|
|
|
29842
30910
|
// src/highlighting/highlighter.ts
|
|
@@ -29934,19 +31002,25 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
29934
31002
|
});
|
|
29935
31003
|
const matchedRanges = [];
|
|
29936
31004
|
const prefixToken = options?.prefixToken;
|
|
31005
|
+
const tolerance = options?.tolerance ?? 0;
|
|
31006
|
+
const prefixLength = options?.prefixLength ?? 0;
|
|
31007
|
+
const matchByFieldToken = /* @__PURE__ */ new Map();
|
|
31008
|
+
const matchesQuery = (fieldToken) => {
|
|
31009
|
+
const known = matchByFieldToken.get(fieldToken);
|
|
31010
|
+
if (known !== void 0) return known;
|
|
31011
|
+
const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
|
|
31012
|
+
const matches = stemmedQueryTokens.some(
|
|
31013
|
+
(stemmedQuery) => fuzzyTermMatches(stemmedQuery, stemmedField, tolerance, prefixLength)
|
|
31014
|
+
);
|
|
31015
|
+
matchByFieldToken.set(fieldToken, matches);
|
|
31016
|
+
return matches;
|
|
31017
|
+
};
|
|
29937
31018
|
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
29938
31019
|
if (i >= charOffsets.length) break;
|
|
29939
31020
|
const fieldToken = fieldResult.tokens[i].token;
|
|
29940
|
-
const
|
|
29941
|
-
if (
|
|
31021
|
+
const prefixMatch = prefixToken !== void 0 && fieldToken.startsWith(prefixToken);
|
|
31022
|
+
if (prefixMatch || matchesQuery(fieldToken)) {
|
|
29942
31023
|
matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
|
|
29943
|
-
continue;
|
|
29944
|
-
}
|
|
29945
|
-
for (const stemmedQuery of stemmedQueryTokens) {
|
|
29946
|
-
if (stemmedField === stemmedQuery) {
|
|
29947
|
-
matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
|
|
29948
|
-
break;
|
|
29949
|
-
}
|
|
29950
31024
|
}
|
|
29951
31025
|
}
|
|
29952
31026
|
const mergedPositions = mergeOverlapping(matchedRanges);
|
|
@@ -30016,7 +31090,9 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
30016
31090
|
preTag: params.highlight.preTag,
|
|
30017
31091
|
postTag: params.highlight.postTag,
|
|
30018
31092
|
maxSnippetLength: params.highlight.maxSnippetLength,
|
|
30019
|
-
prefixToken
|
|
31093
|
+
prefixToken,
|
|
31094
|
+
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
31095
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
30020
31096
|
});
|
|
30021
31097
|
}
|
|
30022
31098
|
}
|
|
@@ -30024,120 +31100,6 @@ function applyHighlights(hits, params, language, analysis, readStoredDocument) {
|
|
|
30024
31100
|
}
|
|
30025
31101
|
}
|
|
30026
31102
|
|
|
30027
|
-
// src/engine/query/shared.ts
|
|
30028
|
-
function partitionsFor(manager, partitionIds) {
|
|
30029
|
-
return partitionsIn(manager, partitionIds);
|
|
30030
|
-
}
|
|
30031
|
-
function coverageFor(manager, partitionIds) {
|
|
30032
|
-
if (partitionIds === void 0) {
|
|
30033
|
-
return {
|
|
30034
|
-
totalPartitions: manager.partitionCount,
|
|
30035
|
-
queriedPartitions: manager.partitionCount,
|
|
30036
|
-
timedOutPartitions: 0,
|
|
30037
|
-
failedPartitions: 0
|
|
30038
|
-
};
|
|
30039
|
-
}
|
|
30040
|
-
const queriedPartitions = partitionsFor(manager, partitionIds).length;
|
|
30041
|
-
return {
|
|
30042
|
-
totalPartitions: partitionIds.length,
|
|
30043
|
-
queriedPartitions,
|
|
30044
|
-
timedOutPartitions: 0,
|
|
30045
|
-
failedPartitions: partitionIds.length - queriedPartitions
|
|
30046
|
-
};
|
|
30047
|
-
}
|
|
30048
|
-
function partitionsForVectorSearch(manager, vecIndex, partitionIds) {
|
|
30049
|
-
if (partitionIds === void 0) {
|
|
30050
|
-
return void 0;
|
|
30051
|
-
}
|
|
30052
|
-
if (!vecIndex.partitionsKnown()) {
|
|
30053
|
-
vecIndex.assignPartitions((docId) => manager.partitionIdOf(docId));
|
|
30054
|
-
}
|
|
30055
|
-
return new Set(partitionIds);
|
|
30056
|
-
}
|
|
30057
|
-
function scoringConfigFor(params, context) {
|
|
30058
|
-
const scoringMode = params.scoring ?? context.config.defaultScoring ?? "local";
|
|
30059
|
-
if (scoringMode !== "broadcast") {
|
|
30060
|
-
return { scoringMode, partitionIds: context.partitionIds };
|
|
30061
|
-
}
|
|
30062
|
-
return { scoringMode, globalStats: context.broadcastStats?.(context.indexName), partitionIds: context.partitionIds };
|
|
30063
|
-
}
|
|
30064
|
-
function broadcastStatsForWorker(params, context, scoring) {
|
|
30065
|
-
if (scoring.scoringMode !== "broadcast" || scoring.globalStats === void 0) {
|
|
30066
|
-
return void 0;
|
|
30067
|
-
}
|
|
30068
|
-
const term = params.term;
|
|
30069
|
-
if (term === void 0 || term.trim().length === 0) {
|
|
30070
|
-
return void 0;
|
|
30071
|
-
}
|
|
30072
|
-
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
|
|
30073
|
-
}
|
|
30074
|
-
function searchOptionsFor(manager) {
|
|
30075
|
-
return {
|
|
30076
|
-
bm25Params: manager.config.bm25,
|
|
30077
|
-
stopWords: manager.analysis.stopWords,
|
|
30078
|
-
customTokenizer: manager.analysis.customTokenizer
|
|
30079
|
-
};
|
|
30080
|
-
}
|
|
30081
|
-
function collectFilterDocIds(manager, params, schema, partitionIds) {
|
|
30082
|
-
const filterDocIds = /* @__PURE__ */ new Set();
|
|
30083
|
-
if (!params.filters) return filterDocIds;
|
|
30084
|
-
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
30085
|
-
const partitionFiltered = partition.applyFilters(params.filters, schema);
|
|
30086
|
-
for (const docId of partitionFiltered) {
|
|
30087
|
-
filterDocIds.add(docId);
|
|
30088
|
-
}
|
|
30089
|
-
}
|
|
30090
|
-
return filterDocIds;
|
|
30091
|
-
}
|
|
30092
|
-
function vectorResultsToScored(results) {
|
|
30093
|
-
return results.map((r) => ({
|
|
30094
|
-
docId: r.docId,
|
|
30095
|
-
score: r.score,
|
|
30096
|
-
termFrequencies: {},
|
|
30097
|
-
fieldLengths: {},
|
|
30098
|
-
idf: {}
|
|
30099
|
-
}));
|
|
30100
|
-
}
|
|
30101
|
-
function vectorSearchersOf(context) {
|
|
30102
|
-
return context.vectorSearchers ?? context.manager.getVectorIndexes();
|
|
30103
|
-
}
|
|
30104
|
-
function resolveVectorIndex(context, fieldName) {
|
|
30105
|
-
return vectorSearchersOf(context).get(fieldName);
|
|
30106
|
-
}
|
|
30107
|
-
var SEARCH_MODES = /* @__PURE__ */ new Set(["fulltext", "vector", "hybrid"]);
|
|
30108
|
-
function requireKnownMode(params) {
|
|
30109
|
-
if (params.mode === void 0 || SEARCH_MODES.has(params.mode)) return;
|
|
30110
|
-
throw new NarsilError(
|
|
30111
|
-
ErrorCodes.SEARCH_INVALID_MODE,
|
|
30112
|
-
`A search takes the mode "fulltext", "vector", or "hybrid", while this query names "${String(params.mode)}"`,
|
|
30113
|
-
{ mode: String(params.mode), modes: [...SEARCH_MODES] }
|
|
30114
|
-
);
|
|
30115
|
-
}
|
|
30116
|
-
function requireVectorSearchable(params, context, needsVector) {
|
|
30117
|
-
if (!needsVector) return;
|
|
30118
|
-
const field = params.vector?.field;
|
|
30119
|
-
if (field === void 0) {
|
|
30120
|
-
throw new NarsilError(
|
|
30121
|
-
ErrorCodes.SEARCH_INVALID_MODE,
|
|
30122
|
-
`A "${params.mode}" query needs a vector clause naming the field to search`,
|
|
30123
|
-
{ mode: params.mode }
|
|
30124
|
-
);
|
|
30125
|
-
}
|
|
30126
|
-
if (vectorSearchersOf(context).has(field)) return;
|
|
30127
|
-
throw new NarsilError(
|
|
30128
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
30129
|
-
`Field "${field}" has no vector index, so a vector search cannot use it`,
|
|
30130
|
-
{ field, vectorFields: [...vectorSearchersOf(context).keys()] }
|
|
30131
|
-
);
|
|
30132
|
-
}
|
|
30133
|
-
function clampAlpha(alpha) {
|
|
30134
|
-
if (alpha === void 0) return 0.5;
|
|
30135
|
-
if (!Number.isFinite(alpha)) return 0.5;
|
|
30136
|
-
if (alpha < 0) return 0;
|
|
30137
|
-
if (alpha > 1) return 1;
|
|
30138
|
-
return alpha;
|
|
30139
|
-
}
|
|
30140
|
-
|
|
30141
31103
|
// src/engine/query/sorted.ts
|
|
30142
31104
|
function sortsWithoutScores(params) {
|
|
30143
31105
|
return params.includeScores !== true && params.includeScoreComponents !== true && params.minScore === void 0 && params.group === void 0 && params.pinned === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
|
|
@@ -30158,19 +31120,21 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
30158
31120
|
const directions = normalized.map((entry) => entry.direction);
|
|
30159
31121
|
const flatSchema = flattenSchema(schema);
|
|
30160
31122
|
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
31123
|
+
const modes = sortModesOf(normalized);
|
|
30161
31124
|
const options = searchOptionsFor(manager);
|
|
30162
31125
|
const partitions = partitionsFor(manager, context.partitionIds);
|
|
30163
31126
|
const partitionLimit = offset + limit + 1;
|
|
30164
31127
|
const merged = [];
|
|
30165
31128
|
const partitionFacets = [];
|
|
31129
|
+
const partitionFacetConfig = params.facets !== void 0 && partitions.length > 1 ? everyValueFacetConfig(params.facets) : params.facets;
|
|
30166
31130
|
let count = 0;
|
|
30167
31131
|
for (const partition of partitions) {
|
|
30168
31132
|
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
30169
|
-
if (
|
|
31133
|
+
if (partitionFacetConfig !== void 0) {
|
|
30170
31134
|
partitionFacets.push(
|
|
30171
31135
|
partition.computeFacets(
|
|
30172
31136
|
matches === null ? /* @__PURE__ */ new Set() : { ordinalBitset: matches.ordinalBitset() },
|
|
30173
|
-
|
|
31137
|
+
partitionFacetConfig,
|
|
30174
31138
|
schema
|
|
30175
31139
|
)
|
|
30176
31140
|
);
|
|
@@ -30181,6 +31145,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
30181
31145
|
fields,
|
|
30182
31146
|
directions,
|
|
30183
31147
|
fieldTypes,
|
|
31148
|
+
modes,
|
|
30184
31149
|
limit: partitionLimit,
|
|
30185
31150
|
anchorKey,
|
|
30186
31151
|
anchorId,
|
|
@@ -30206,13 +31171,42 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
30206
31171
|
hits: page.map((entry) => ({ id: entry.id, document: void 0 })),
|
|
30207
31172
|
count,
|
|
30208
31173
|
cursor,
|
|
30209
|
-
facets: partitionFacets.length > 0 ? mergeFacets(partitionFacets) : void 0
|
|
31174
|
+
facets: params.facets !== void 0 && partitionFacets.length > 0 ? mergeFacets(partitionFacets, params.facets) : void 0
|
|
30210
31175
|
};
|
|
30211
31176
|
}
|
|
30212
31177
|
|
|
30213
31178
|
// src/search/fusion.ts
|
|
31179
|
+
function rankConstantOf(k) {
|
|
31180
|
+
if (k === void 0 || k === null) return DEFAULT_RANK_CONSTANT;
|
|
31181
|
+
if (!Number.isInteger(k) || k < MIN_RANK_CONSTANT) {
|
|
31182
|
+
throw new NarsilError(
|
|
31183
|
+
ErrorCodes.CONFIG_INVALID,
|
|
31184
|
+
`hybrid.k must be a whole number of at least ${MIN_RANK_CONSTANT}`,
|
|
31185
|
+
{ k }
|
|
31186
|
+
);
|
|
31187
|
+
}
|
|
31188
|
+
return k;
|
|
31189
|
+
}
|
|
31190
|
+
function linearAlphaOf(alpha) {
|
|
31191
|
+
if (alpha === void 0 || alpha === null) return DEFAULT_LINEAR_ALPHA;
|
|
31192
|
+
if (!Number.isFinite(alpha) || alpha < MIN_LINEAR_ALPHA || alpha > MAX_LINEAR_ALPHA) {
|
|
31193
|
+
throw new NarsilError(
|
|
31194
|
+
ErrorCodes.CONFIG_INVALID,
|
|
31195
|
+
`hybrid.alpha must be a number from ${MIN_LINEAR_ALPHA} to ${MAX_LINEAR_ALPHA}`,
|
|
31196
|
+
{ alpha }
|
|
31197
|
+
);
|
|
31198
|
+
}
|
|
31199
|
+
return alpha;
|
|
31200
|
+
}
|
|
31201
|
+
function resolveHybridFusion(hybrid) {
|
|
31202
|
+
const strategy = hybrid?.strategy ?? "rrf";
|
|
31203
|
+
if (strategy !== "rrf" && strategy !== "linear") {
|
|
31204
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `hybrid.strategy must be "rrf" or "linear"`, { strategy });
|
|
31205
|
+
}
|
|
31206
|
+
return { strategy, k: rankConstantOf(hybrid?.k), alpha: linearAlphaOf(hybrid?.alpha) };
|
|
31207
|
+
}
|
|
30214
31208
|
function reciprocalRankFusion(lists, options) {
|
|
30215
|
-
const k = options.k
|
|
31209
|
+
const k = options.k;
|
|
30216
31210
|
const scores = /* @__PURE__ */ new Map();
|
|
30217
31211
|
const docData = /* @__PURE__ */ new Map();
|
|
30218
31212
|
for (const list of lists) {
|
|
@@ -30304,7 +31298,18 @@ function oversampleOf(vectorConfig) {
|
|
|
30304
31298
|
}
|
|
30305
31299
|
return oversample;
|
|
30306
31300
|
}
|
|
30307
|
-
|
|
31301
|
+
function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
31302
|
+
const reach = cursorDepth + offset + limit;
|
|
31303
|
+
if (reach > RESULT_WINDOW) {
|
|
31304
|
+
throw new NarsilError(
|
|
31305
|
+
ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
|
|
31306
|
+
`The engine pages a vector search through the first ${RESULT_WINDOW} results, and this page ends at ${reach}. Narrow the search with a filter or a similarity floor`,
|
|
31307
|
+
{ limit, offset, cursorDepth, window: RESULT_WINDOW }
|
|
31308
|
+
);
|
|
31309
|
+
}
|
|
31310
|
+
return Math.min(reach + 1, RESULT_WINDOW);
|
|
31311
|
+
}
|
|
31312
|
+
async function executeVectorSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
30308
31313
|
const { manager, config, partitionIds } = context;
|
|
30309
31314
|
const vectorConfig = params.vector;
|
|
30310
31315
|
if (!vectorConfig || !vectorConfig.value) {
|
|
@@ -30325,8 +31330,8 @@ async function executeVectorSearch(params, context, limit, offset) {
|
|
|
30325
31330
|
filterPartitions = partitionsForVectorSearch(manager, vecIndex, partitionIds);
|
|
30326
31331
|
}
|
|
30327
31332
|
const queryVec = new Float32Array(vectorConfig.value);
|
|
30328
|
-
const k = limit
|
|
30329
|
-
const
|
|
31333
|
+
const k = vectorFetchDepth(limit, offset, cursorDepth);
|
|
31334
|
+
const outcome = await vecIndex.searchParallel(queryVec, k, {
|
|
30330
31335
|
metric: vectorConfig.metric ?? "cosine",
|
|
30331
31336
|
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
30332
31337
|
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
@@ -30334,8 +31339,8 @@ async function executeVectorSearch(params, context, limit, offset) {
|
|
|
30334
31339
|
efSearch: vectorConfig.efSearch,
|
|
30335
31340
|
oversample: oversampleOf(vectorConfig)
|
|
30336
31341
|
});
|
|
30337
|
-
const scored = vectorResultsToScored(results);
|
|
30338
|
-
return { scored, totalMatched:
|
|
31342
|
+
const scored = vectorResultsToScored(outcome.results);
|
|
31343
|
+
return { scored, totalMatched: outcome.matched, matchedExact: outcome.matchedExact };
|
|
30339
31344
|
}
|
|
30340
31345
|
async function textLeg(textOnlyParams, context) {
|
|
30341
31346
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
@@ -30354,9 +31359,9 @@ async function textLeg(textOnlyParams, context) {
|
|
|
30354
31359
|
async function vectorLeg(vectorConfig, queryVector, context, filterDocIds, k) {
|
|
30355
31360
|
const { manager } = context;
|
|
30356
31361
|
const vecIndex = resolveVectorIndex(context, vectorConfig.field);
|
|
30357
|
-
if (!vecIndex) return [];
|
|
31362
|
+
if (!vecIndex) return { scored: [], holdsEveryMatch: true };
|
|
30358
31363
|
const filterPartitions = filterDocIds === void 0 ? partitionsForVectorSearch(manager, vecIndex, context.partitionIds) : void 0;
|
|
30359
|
-
const
|
|
31364
|
+
const outcome = await vecIndex.searchParallel(queryVector, k, {
|
|
30360
31365
|
metric: vectorConfig.metric ?? "cosine",
|
|
30361
31366
|
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
30362
31367
|
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
@@ -30364,11 +31369,15 @@ async function vectorLeg(vectorConfig, queryVector, context, filterDocIds, k) {
|
|
|
30364
31369
|
efSearch: vectorConfig.efSearch,
|
|
30365
31370
|
oversample: oversampleOf(vectorConfig)
|
|
30366
31371
|
});
|
|
30367
|
-
return
|
|
31372
|
+
return {
|
|
31373
|
+
scored: vectorResultsToScored(outcome.results),
|
|
31374
|
+
holdsEveryMatch: outcome.matchedExact && outcome.results.length === outcome.matched
|
|
31375
|
+
};
|
|
30368
31376
|
}
|
|
30369
|
-
async function executeHybridSearch(params, context, limit, offset) {
|
|
31377
|
+
async function executeHybridSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
30370
31378
|
const { manager, config } = context;
|
|
30371
31379
|
const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
|
|
31380
|
+
const fusion = resolveHybridFusion(params.hybrid);
|
|
30372
31381
|
let filterDocIds;
|
|
30373
31382
|
if (params.filters) {
|
|
30374
31383
|
filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds);
|
|
@@ -30385,31 +31394,36 @@ async function executeHybridSearch(params, context, limit, offset) {
|
|
|
30385
31394
|
new Float32Array(vectorConfig.value),
|
|
30386
31395
|
context,
|
|
30387
31396
|
filterDocIds,
|
|
30388
|
-
limit
|
|
31397
|
+
vectorFetchDepth(limit, offset, cursorDepth)
|
|
30389
31398
|
);
|
|
30390
|
-
const [textFanOutResult,
|
|
30391
|
-
const
|
|
30392
|
-
const strategy = hybridConfig.strategy ?? "rrf";
|
|
31399
|
+
const [textFanOutResult, vectorOutcome] = await Promise.all([textLeg(textOnlyParams, context), vectorPending]);
|
|
31400
|
+
const vectorScored = vectorOutcome.scored;
|
|
30393
31401
|
let fusedScored;
|
|
30394
|
-
if (strategy === "rrf") {
|
|
30395
|
-
|
|
30396
|
-
fusedScored = reciprocalRankFusion([textFanOutResult.scored, vectorScored], { k: rrfK });
|
|
31402
|
+
if (fusion.strategy === "rrf") {
|
|
31403
|
+
fusedScored = reciprocalRankFusion([textFanOutResult.scored, vectorScored], { k: fusion.k });
|
|
30397
31404
|
} else {
|
|
30398
|
-
|
|
30399
|
-
fusedScored = linearCombination(textFanOutResult.scored, vectorScored, { alpha });
|
|
31405
|
+
fusedScored = linearCombination(textFanOutResult.scored, vectorScored, { alpha: fusion.alpha });
|
|
30400
31406
|
}
|
|
30401
31407
|
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
30402
31408
|
const threshold = params.minScore;
|
|
30403
31409
|
fusedScored = fusedScored.filter((doc) => doc.score >= threshold);
|
|
30404
31410
|
}
|
|
31411
|
+
const textHoldsEveryMatch = textFanOutResult.scored.length === textFanOutResult.totalMatched;
|
|
30405
31412
|
return {
|
|
30406
31413
|
scored: fusedScored,
|
|
30407
31414
|
totalMatched: fusedScored.length,
|
|
31415
|
+
matchedExact: vectorOutcome.holdsEveryMatch && textHoldsEveryMatch,
|
|
30408
31416
|
facets: textFanOutResult.facets
|
|
30409
31417
|
};
|
|
30410
31418
|
}
|
|
30411
31419
|
|
|
30412
31420
|
// src/engine/query/index.ts
|
|
31421
|
+
function requireCursorDepth(cursor, signature, binding) {
|
|
31422
|
+
if (cursor === void 0) return 0;
|
|
31423
|
+
const decoded = decodePageCursor(cursor);
|
|
31424
|
+
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
31425
|
+
return decoded.depth ?? 0;
|
|
31426
|
+
}
|
|
30413
31427
|
async function executeQuery(params, context) {
|
|
30414
31428
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
30415
31429
|
const startTime = now();
|
|
@@ -30430,11 +31444,12 @@ async function executeQuery(params, context) {
|
|
|
30430
31444
|
{ sort: sortFields.map((entry) => entry.field) }
|
|
30431
31445
|
);
|
|
30432
31446
|
}
|
|
30433
|
-
|
|
31447
|
+
requireValidQueryOptions(params, config);
|
|
30434
31448
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
30435
31449
|
let paginated;
|
|
30436
31450
|
let nextCursor;
|
|
30437
31451
|
let count;
|
|
31452
|
+
let countExact = true;
|
|
30438
31453
|
let facets;
|
|
30439
31454
|
let groups;
|
|
30440
31455
|
if (sortSignature !== null && hasTerm && !isVectorOnly && !isHybridMode && sortsWithoutScores(params)) {
|
|
@@ -30448,9 +31463,11 @@ async function executeQuery(params, context) {
|
|
|
30448
31463
|
const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
|
|
30449
31464
|
let fanOutResult;
|
|
30450
31465
|
if (isVectorOnly && hasGlobalVectorIndex) {
|
|
30451
|
-
|
|
31466
|
+
const depth = requireCursorDepth(params.searchAfter, sortSignature, context.cursorBinding);
|
|
31467
|
+
fanOutResult = await executeVectorSearch(params, context, limit, offset, depth);
|
|
30452
31468
|
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
30453
|
-
|
|
31469
|
+
const depth = requireCursorDepth(params.searchAfter, sortSignature, context.cursorBinding);
|
|
31470
|
+
fanOutResult = await executeHybridSearch(params, context, limit, offset, depth);
|
|
30454
31471
|
} else {
|
|
30455
31472
|
const scoring = scoringConfigFor(params, context);
|
|
30456
31473
|
const workerResult = workerSearch ? await workerSearch(indexName, params, broadcastStatsForWorker(params, context, scoring), context.partitionIds) : null;
|
|
@@ -30485,11 +31502,12 @@ async function executeQuery(params, context) {
|
|
|
30485
31502
|
const sortFieldNames = sortFields.map((entry) => entry.field);
|
|
30486
31503
|
const sortFlatSchema = sortFieldNames.length === 0 ? {} : flattenSchema(config.schema);
|
|
30487
31504
|
const sortFieldTypes = sortFieldNames.map((field) => sortFlatSchema[field]);
|
|
31505
|
+
const sortModes = sortModesOf(sortFields);
|
|
30488
31506
|
const sortKeyCache = /* @__PURE__ */ new Map();
|
|
30489
31507
|
const sortKeyOf = (docId) => {
|
|
30490
31508
|
let key = sortKeyCache.get(docId);
|
|
30491
31509
|
if (key === void 0) {
|
|
30492
|
-
key = manager.sortValues(docId, sortFieldNames, sortFieldTypes);
|
|
31510
|
+
key = manager.sortValues(docId, sortFieldNames, sortFieldTypes, sortModes);
|
|
30493
31511
|
sortKeyCache.set(docId, key);
|
|
30494
31512
|
}
|
|
30495
31513
|
return key;
|
|
@@ -30500,12 +31518,15 @@ async function executeQuery(params, context) {
|
|
|
30500
31518
|
if (params.group) {
|
|
30501
31519
|
groups = applyGrouping(hits, params.group, (docId) => manager.getRef(docId));
|
|
30502
31520
|
}
|
|
31521
|
+
let pinsFromOutside = 0;
|
|
30503
31522
|
if (params.pinned && params.searchAfter === void 0 && context.partitionIds === void 0) {
|
|
30504
|
-
|
|
31523
|
+
const placement = placePinned(hits, params.pinned, (docId) => {
|
|
30505
31524
|
const doc = manager.getRef(docId);
|
|
30506
31525
|
if (!doc) return void 0;
|
|
30507
31526
|
return { id: docId, score: 0, document: doc };
|
|
30508
31527
|
});
|
|
31528
|
+
hits = placement.hits;
|
|
31529
|
+
pinsFromOutside = placement.placedFromOutside;
|
|
30509
31530
|
}
|
|
30510
31531
|
let sortContext;
|
|
30511
31532
|
if (sortSignature !== null) {
|
|
@@ -30527,7 +31548,9 @@ async function executeQuery(params, context) {
|
|
|
30527
31548
|
);
|
|
30528
31549
|
paginated = paged.paginated;
|
|
30529
31550
|
nextCursor = paged.nextCursor;
|
|
30530
|
-
|
|
31551
|
+
const holdsEveryMatch = fanOutResult.matchedExact !== false && fanOutResult.scored.length === fanOutResult.totalMatched;
|
|
31552
|
+
count = fanOutResult.totalMatched + (holdsEveryMatch ? pinsFromOutside : 0);
|
|
31553
|
+
countExact = fanOutResult.matchedExact !== false && (holdsEveryMatch || pinsFromOutside === 0);
|
|
30531
31554
|
facets = fanOutResult.facets;
|
|
30532
31555
|
if (sortSignature !== null && params.includeScores !== true) {
|
|
30533
31556
|
for (const hit of hits) hit.score = void 0;
|
|
@@ -30563,6 +31586,7 @@ async function executeQuery(params, context) {
|
|
|
30563
31586
|
return {
|
|
30564
31587
|
hits: paginated,
|
|
30565
31588
|
count,
|
|
31589
|
+
countExact,
|
|
30566
31590
|
elapsed,
|
|
30567
31591
|
cursor: nextCursor,
|
|
30568
31592
|
facets,
|
|
@@ -30574,6 +31598,7 @@ async function executePreflight(params, context) {
|
|
|
30574
31598
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
30575
31599
|
const startTime = now();
|
|
30576
31600
|
requireKnownMode(params);
|
|
31601
|
+
requireValidQueryOptions(params, config);
|
|
30577
31602
|
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
30578
31603
|
const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
|
|
30579
31604
|
const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
|
|
@@ -30582,14 +31607,17 @@ async function executePreflight(params, context) {
|
|
|
30582
31607
|
const requestedVectorField = params.vector?.field;
|
|
30583
31608
|
const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
|
|
30584
31609
|
let totalMatched;
|
|
31610
|
+
let countExact = true;
|
|
30585
31611
|
const preflightLimit = 1e3;
|
|
30586
31612
|
const preflightOffset = 0;
|
|
30587
31613
|
if (isVectorOnly && hasGlobalVectorIndex) {
|
|
30588
31614
|
const result = await executeVectorSearch(params, context, preflightLimit, preflightOffset);
|
|
30589
31615
|
totalMatched = result.totalMatched;
|
|
31616
|
+
countExact = result.matchedExact !== false;
|
|
30590
31617
|
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
30591
31618
|
const result = await executeHybridSearch(params, context, preflightLimit, preflightOffset);
|
|
30592
31619
|
totalMatched = result.totalMatched;
|
|
31620
|
+
countExact = result.matchedExact !== false;
|
|
30593
31621
|
} else if (countsWithoutScores(params)) {
|
|
30594
31622
|
totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
|
|
30595
31623
|
searchOptions: searchOptionsFor(manager),
|
|
@@ -30613,7 +31641,7 @@ async function executePreflight(params, context) {
|
|
|
30613
31641
|
}
|
|
30614
31642
|
}
|
|
30615
31643
|
const elapsed = now() - startTime;
|
|
30616
|
-
return { count: totalMatched, elapsed };
|
|
31644
|
+
return { count: totalMatched, countExact, elapsed };
|
|
30617
31645
|
}
|
|
30618
31646
|
|
|
30619
31647
|
// src/engine/resolve-vector-text.ts
|
|
@@ -30640,7 +31668,16 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
|
|
|
30640
31668
|
if (params.vector.text.length === 0) {
|
|
30641
31669
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Vector query 'text' must not be empty");
|
|
30642
31670
|
}
|
|
30643
|
-
|
|
31671
|
+
let raw;
|
|
31672
|
+
try {
|
|
31673
|
+
raw = await embeddingAdapter.embed(params.vector.text, "query", signal);
|
|
31674
|
+
} catch (err) {
|
|
31675
|
+
if (err instanceof NarsilError) throw err;
|
|
31676
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
31677
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, `The embedding adapter cannot embed the query: ${message}`, {
|
|
31678
|
+
cause: message
|
|
31679
|
+
});
|
|
31680
|
+
}
|
|
30644
31681
|
if (!(raw instanceof Float32Array)) {
|
|
30645
31682
|
throw new NarsilError(
|
|
30646
31683
|
ErrorCodes.EMBEDDING_FAILED,
|
|
@@ -30707,6 +31744,10 @@ function pickDisplaySurface(surfaceOccurrences) {
|
|
|
30707
31744
|
}
|
|
30708
31745
|
|
|
30709
31746
|
// src/narsil/reads.ts
|
|
31747
|
+
async function enterRead(core, indexName) {
|
|
31748
|
+
core.guardShutdown();
|
|
31749
|
+
if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
31750
|
+
}
|
|
30710
31751
|
function scopedParams(params, options) {
|
|
30711
31752
|
{
|
|
30712
31753
|
return params;
|
|
@@ -30720,7 +31761,7 @@ function broadcastStatsFor(core, options) {
|
|
|
30720
31761
|
return (name) => invalidation.broadcastStats(name);
|
|
30721
31762
|
}
|
|
30722
31763
|
async function runEngineQuery(core, indexName, params, options) {
|
|
30723
|
-
core
|
|
31764
|
+
await enterRead(core, indexName);
|
|
30724
31765
|
const release = await core.indexState.acquire(indexName);
|
|
30725
31766
|
try {
|
|
30726
31767
|
const entry = core.requireIndex(indexName);
|
|
@@ -30761,7 +31802,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
30761
31802
|
}
|
|
30762
31803
|
}
|
|
30763
31804
|
async function runEnginePreflight(core, indexName, params, options) {
|
|
30764
|
-
core
|
|
31805
|
+
await enterRead(core, indexName);
|
|
30765
31806
|
const release = await core.indexState.acquire(indexName);
|
|
30766
31807
|
try {
|
|
30767
31808
|
const entry = core.requireIndex(indexName);
|
|
@@ -30794,7 +31835,7 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
30794
31835
|
}
|
|
30795
31836
|
}
|
|
30796
31837
|
async function runEngineSuggest(core, indexName, params, partitionIds) {
|
|
30797
|
-
core
|
|
31838
|
+
await enterRead(core, indexName);
|
|
30798
31839
|
const release = await core.indexState.acquire(indexName);
|
|
30799
31840
|
try {
|
|
30800
31841
|
const result = executeSuggest(
|
|
@@ -30812,13 +31853,14 @@ async function runEngineSuggest(core, indexName, params, partitionIds) {
|
|
|
30812
31853
|
}
|
|
30813
31854
|
}
|
|
30814
31855
|
async function runEngineListDocuments(core, indexName, params, partitionIds) {
|
|
30815
|
-
core
|
|
31856
|
+
await enterRead(core, indexName);
|
|
30816
31857
|
const release = await core.indexState.acquire(indexName);
|
|
30817
31858
|
try {
|
|
30818
31859
|
const entry = core.requireIndex(indexName);
|
|
30819
31860
|
return executeListDocuments(params, {
|
|
30820
31861
|
manager: core.requireManager(indexName),
|
|
30821
31862
|
schema: entry.config.schema,
|
|
31863
|
+
strict: entry.config.strict,
|
|
30822
31864
|
partitionIds
|
|
30823
31865
|
});
|
|
30824
31866
|
} finally {
|
|
@@ -31125,16 +32167,11 @@ function createNarsilFromCore(core, config) {
|
|
|
31125
32167
|
|
|
31126
32168
|
// src/narsil/index.ts
|
|
31127
32169
|
async function createNarsil(config) {
|
|
31128
|
-
|
|
31129
|
-
|
|
31130
|
-
|
|
31131
|
-
|
|
31132
|
-
|
|
31133
|
-
if (core.invalidation) {
|
|
31134
|
-
await core.invalidation.start();
|
|
31135
|
-
}
|
|
31136
|
-
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
31137
|
-
return createNarsilFromCore(core, config);
|
|
32170
|
+
requireKnownConfig(config);
|
|
32171
|
+
const runnableConfig = await resolveRunnableConfig(config);
|
|
32172
|
+
const core = createEngineCore(runnableConfig);
|
|
32173
|
+
await startEngineCore(core, runnableConfig);
|
|
32174
|
+
return createNarsilFromCore(core, runnableConfig);
|
|
31138
32175
|
}
|
|
31139
32176
|
|
|
31140
32177
|
// src/index.ts
|