@delali/narsil 0.3.1-next.71 → 0.3.1-next.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-Dt-7B4Wz.d.ts → capabilities-D6gjjQyA.d.ts} +2 -2
- package/dist/{chunk-P4DSQ6M4.mjs → chunk-2X2ZQOAB.mjs} +69 -28
- package/dist/chunk-3HQPUCCT.mjs +785 -0
- package/dist/{chunk-IOR2K26O.mjs → chunk-7QMTA7AA.mjs} +146 -37
- package/dist/{chunk-FVLQ63B7.mjs → chunk-CVPA5Q7R.mjs} +24 -3
- package/dist/{chunk-2STVUTNA.mjs → chunk-EDMCZLSJ.mjs} +4 -3
- package/dist/{chunk-M76V3OGJ.mjs → chunk-HVJYTKK7.mjs} +940 -647
- package/dist/{chunk-HWQLTLNG.mjs → chunk-KZTI25D3.mjs} +1 -1
- package/dist/{chunk-YJDQ7NLA.mjs → chunk-LCCL6EEH.mjs} +3 -3
- package/dist/{chunk-OI42GWLR.mjs → chunk-LJYTW7GT.mjs} +57 -11
- package/dist/{chunk-653BNMAF.mjs → chunk-M6XF5HDE.mjs} +57 -17
- package/dist/{chunk-YNVIUGF5.mjs → chunk-P2ZZN32C.mjs} +1 -1
- package/dist/{chunk-LMO5TJUS.mjs → chunk-QR4OKRVY.mjs} +637 -196
- package/dist/{chunk-USKFU7J5.mjs → chunk-RV77FR2F.mjs} +1 -0
- package/dist/{chunk-GIVAGD3Q.mjs → chunk-XZ4PAREY.mjs} +1 -1
- package/dist/{chunk-3JGWULDF.mjs → chunk-Y52WZWAU.mjs} +5 -5
- package/dist/chunk-ZKRV5M2P.mjs +75 -0
- package/dist/{client-COek5GXK.d.ts → client-C2TiW29y.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +1 -0
- package/dist/{config-rBpbgTw8.d.ts → config-BLguB-om.d.ts} +2 -2
- package/dist/{config-Dfanyqqt.d.ts → config-C7Ip8Ife.d.ts} +17 -2
- package/dist/direct-executor-SVCHXR74.mjs +14 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +58 -35
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-igk0aMpQ.d.ts → engine-Dm4446j1.d.ts} +2 -2
- package/dist/index.browser.mjs +2183 -1087
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +14 -12
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-C7UMCMH3.d.ts → memory-CKHTU1CL.d.ts} +66 -25
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +7 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +17 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -12
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +1 -0
- package/dist/{schema-Bgzr4g7b.d.ts → schema-BBOCdhND.d.ts} +9 -7
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +13 -12
- package/dist/{tasks-BZt_zes6.d.ts → tasks-BECd0b__.d.ts} +1 -1
- package/dist/{thread-JEVAYETR.mjs → thread-IBSIZZ6W.mjs} +22 -13
- package/dist/{types-gOUqhs2F.d.ts → types-BiUmaiWT.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +4 -4
- package/package.json +15 -7
- package/dist/direct-executor-GYJEKNV5.mjs +0 -12
|
@@ -1,18 +1,20 @@
|
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-2X2ZQOAB.mjs';
|
|
1
2
|
import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
|
|
2
|
-
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, admitInsert, asBatchInsertError, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import { requireSearchBackendSettings } from './chunk-YNVIUGF5.mjs';
|
|
3
|
+
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-CVPA5Q7R.mjs';
|
|
4
|
+
import { BATCH_CHUNK_SIZE, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION } from './chunk-XZ4PAREY.mjs';
|
|
5
|
+
import { generateId, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, requirePatternSearchFor, createCompositePartition, deserializePayloadV2, createSharedFrozenSegment, createPartitionManager, buildSegmentPayload, resolvePartitionInsertOptions, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, resolveIndexAnalysis, deserializePayloadV1, decodeVectorIndexParts, nodeThreadId, resolveRequestThreadCount, createPartitionIndex, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, isCompositePartition, workerResourceLimits, deserializeMetadata, isNodeMainThread, spawnNodeWorker, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, serializeMetadata, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, decodeVectorFilePayload, documentsOfPayloadV2, countDead, decodeVectorGraphPayload, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createFrozenSegment } from './chunk-HVJYTKK7.mjs';
|
|
6
|
+
import { getNestedValue, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, kWayMerge, mergeFacets } from './chunk-QR4OKRVY.mjs';
|
|
7
|
+
import { getLanguage, detectRuntime } from './chunk-KZTI25D3.mjs';
|
|
8
|
+
import { extractVectorFieldsFromSchema, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema, isSingleStringFieldType, requireValidFieldTypes, schemaFieldsOf, validateRequiredFields, validateDocument, validateDocumentStrict } from './chunk-M6XF5HDE.mjs';
|
|
9
|
+
import { releaseLocksHeldBy } from './chunk-LCCL6EEH.mjs';
|
|
10
|
+
import { requireSearchBackendSettings } from './chunk-P2ZZN32C.mjs';
|
|
11
11
|
import { crc32, nativeCrc32, crc32Final, crc32Init, crc32Update } from './chunk-3L4U6EJQ.mjs';
|
|
12
12
|
import { CHECKSUM_TIMEOUT_MS, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-IFBKLBGB.mjs';
|
|
13
|
-
import {
|
|
13
|
+
import { widestPatternSplit } from './chunk-3HQPUCCT.mjs';
|
|
14
|
+
import { DEFAULT_PATTERN_WORK_CAP, PATTERN_TEST_OPERATORS } from './chunk-ZKRV5M2P.mjs';
|
|
15
|
+
import { compareCodePoints, foldedCodePointCount, codePointCount } from './chunk-LJYTW7GT.mjs';
|
|
14
16
|
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
|
|
15
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
17
|
+
import { ErrorCodes, NarsilError } from './chunk-RV77FR2F.mjs';
|
|
16
18
|
import { encode, decode } from '@msgpack/msgpack';
|
|
17
19
|
|
|
18
20
|
// src/engine/config-keys.ts
|
|
@@ -26,7 +28,8 @@ var NARSIL_CONFIG_KEYS = {
|
|
|
26
28
|
embeddingAdapters: true,
|
|
27
29
|
durability: true,
|
|
28
30
|
analysis: true,
|
|
29
|
-
lifecycle: true
|
|
31
|
+
lifecycle: true,
|
|
32
|
+
patternWorkCap: true
|
|
30
33
|
};
|
|
31
34
|
var DURABILITY_CONFIG_KEYS = {
|
|
32
35
|
tier: true,
|
|
@@ -91,6 +94,16 @@ function requireKnownConfig(config) {
|
|
|
91
94
|
requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
|
|
92
95
|
requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
|
|
93
96
|
requireNamedPlugins(config.plugins);
|
|
97
|
+
requireValidPatternWorkCap(config.patternWorkCap);
|
|
98
|
+
}
|
|
99
|
+
function requireValidPatternWorkCap(cap) {
|
|
100
|
+
if (cap === void 0) return;
|
|
101
|
+
if (typeof cap === "number" && Number.isSafeInteger(cap) && cap >= 1) return;
|
|
102
|
+
throw new NarsilError(
|
|
103
|
+
ErrorCodes.CONFIG_INVALID,
|
|
104
|
+
`The configuration sets "patternWorkCap" to ${String(cap)}, while the cap must be a whole number from 1 to ${Number.MAX_SAFE_INTEGER}`,
|
|
105
|
+
{ setting: "patternWorkCap" }
|
|
106
|
+
);
|
|
94
107
|
}
|
|
95
108
|
|
|
96
109
|
// src/engine/orchestration/turn.ts
|
|
@@ -8174,6 +8187,49 @@ function stopIdleSweep(state) {
|
|
|
8174
8187
|
state.idleSweep = null;
|
|
8175
8188
|
}
|
|
8176
8189
|
|
|
8190
|
+
// src/core/pattern-index/query-positions.ts
|
|
8191
|
+
function isRecord3(value) {
|
|
8192
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
8193
|
+
}
|
|
8194
|
+
function textPositions(text, caseFold) {
|
|
8195
|
+
if (typeof text !== "string") return 0;
|
|
8196
|
+
return caseFold ? foldedCodePointCount(text) : codePointCount(text);
|
|
8197
|
+
}
|
|
8198
|
+
function larger(left, right) {
|
|
8199
|
+
if (left === null) return right;
|
|
8200
|
+
if (right === null) return left;
|
|
8201
|
+
return Math.max(left, right);
|
|
8202
|
+
}
|
|
8203
|
+
function largestTestOf(filter) {
|
|
8204
|
+
const caseFold = filter.caseFold === true;
|
|
8205
|
+
let largest = null;
|
|
8206
|
+
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
8207
|
+
const operand = filter[operator];
|
|
8208
|
+
if (operand === void 0) continue;
|
|
8209
|
+
const texts = Array.isArray(operand) ? operand : [operand];
|
|
8210
|
+
for (const text of texts) largest = larger(largest, textPositions(text, caseFold));
|
|
8211
|
+
}
|
|
8212
|
+
return largest;
|
|
8213
|
+
}
|
|
8214
|
+
function largestIn(expression, patternFields) {
|
|
8215
|
+
let largest = null;
|
|
8216
|
+
if (isRecord3(expression.fields)) {
|
|
8217
|
+
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
8218
|
+
if (patternFields.has(fieldPath) && isRecord3(filter)) largest = larger(largest, largestTestOf(filter));
|
|
8219
|
+
}
|
|
8220
|
+
}
|
|
8221
|
+
const nested = [...expression.and ?? [], ...expression.or ?? []];
|
|
8222
|
+
if (expression.not !== void 0) nested.push(expression.not);
|
|
8223
|
+
for (const child of nested) largest = larger(largest, largestIn(child, patternFields));
|
|
8224
|
+
return largest;
|
|
8225
|
+
}
|
|
8226
|
+
function largestPatternTest(expression, schema) {
|
|
8227
|
+
if (expression === void 0) return null;
|
|
8228
|
+
const patternFields = /* @__PURE__ */ new Set();
|
|
8229
|
+
for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
|
|
8230
|
+
return patternFields.size === 0 ? null : largestIn(expression, patternFields);
|
|
8231
|
+
}
|
|
8232
|
+
|
|
8177
8233
|
// src/engine/orchestration/search.ts
|
|
8178
8234
|
function partitionsPerLease(scope, leaseCount) {
|
|
8179
8235
|
const assignments = Array.from({ length: leaseCount }, () => []);
|
|
@@ -8185,15 +8241,18 @@ function partitionsPerLease(scope, leaseCount) {
|
|
|
8185
8241
|
function mergeWorkerResults(results, facetConfig) {
|
|
8186
8242
|
const merged = kWayMerge(results.map((result) => result.scored));
|
|
8187
8243
|
let totalMatched = 0;
|
|
8244
|
+
let patternWorkSpent = 0;
|
|
8188
8245
|
const workerFacets = [];
|
|
8189
8246
|
for (const result of results) {
|
|
8190
8247
|
totalMatched += result.totalMatched;
|
|
8248
|
+
patternWorkSpent += result.patternWorkSpent ?? 0;
|
|
8191
8249
|
if (result.facets !== void 0) workerFacets.push(result.facets);
|
|
8192
8250
|
}
|
|
8193
8251
|
return {
|
|
8194
8252
|
scored: merged,
|
|
8195
8253
|
totalMatched,
|
|
8196
|
-
facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0
|
|
8254
|
+
facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0,
|
|
8255
|
+
patternWorkSpent
|
|
8197
8256
|
};
|
|
8198
8257
|
}
|
|
8199
8258
|
function scoresPerPartition(state, indexName, params, globalStats) {
|
|
@@ -8201,26 +8260,45 @@ function scoresPerPartition(state, indexName, params, globalStats) {
|
|
|
8201
8260
|
if (mode === "local") return true;
|
|
8202
8261
|
return mode === "broadcast" && globalStats !== void 0;
|
|
8203
8262
|
}
|
|
8204
|
-
function queryAction(indexName, params, globalStats, partitionIds) {
|
|
8263
|
+
function queryAction(indexName, params, globalStats, partitionIds, patternWork) {
|
|
8205
8264
|
return {
|
|
8206
8265
|
type: "query",
|
|
8207
8266
|
indexName,
|
|
8208
8267
|
params,
|
|
8209
8268
|
requestId: createRequestId(),
|
|
8269
|
+
patternWork,
|
|
8210
8270
|
...partitionIds !== void 0 ? { partitionIds } : {},
|
|
8211
8271
|
...globalStats !== void 0 ? { globalStats } : {}
|
|
8212
8272
|
};
|
|
8213
8273
|
}
|
|
8214
|
-
async function runSplit(leases, scope, indexName, params, globalStats) {
|
|
8274
|
+
async function runSplit(leases, scope, indexName, params, globalStats, patternWork) {
|
|
8215
8275
|
const assignments = partitionsPerLease(scope, leases.length);
|
|
8216
8276
|
const workerParams = params.facets !== void 0 ? { ...params, facets: oversampledFacetConfig(params.facets) } : params;
|
|
8217
|
-
const
|
|
8277
|
+
const outcomes = await Promise.allSettled(
|
|
8218
8278
|
assignments.map(
|
|
8219
|
-
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds)).finally(() => leases[at].release())
|
|
8279
|
+
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork)).finally(() => leases[at].release())
|
|
8220
8280
|
)
|
|
8221
8281
|
);
|
|
8282
|
+
const results = [];
|
|
8283
|
+
const failures = [];
|
|
8284
|
+
for (const outcome of outcomes) {
|
|
8285
|
+
if (outcome.status === "fulfilled") results.push(outcome.value);
|
|
8286
|
+
else failures.push(outcome.reason);
|
|
8287
|
+
}
|
|
8288
|
+
if (failures.length > 0) throw failures.find(isWorkCapError) ?? failures[0];
|
|
8222
8289
|
return mergeWorkerResults(results, params.facets);
|
|
8223
8290
|
}
|
|
8291
|
+
function isWorkCapError(err) {
|
|
8292
|
+
return err instanceof NarsilError && err.code === ErrorCodes.SEARCH_WORK_CAP_EXCEEDED;
|
|
8293
|
+
}
|
|
8294
|
+
function patternSplitWidth(state, indexName, params) {
|
|
8295
|
+
const schema = state.indexRegistry.get(indexName)?.config.schema;
|
|
8296
|
+
if (schema === void 0) return Number.POSITIVE_INFINITY;
|
|
8297
|
+
const largest = largestPatternTest(params.filters, schema);
|
|
8298
|
+
if (largest === null) return Number.POSITIVE_INFINITY;
|
|
8299
|
+
if (typeof SharedArrayBuffer !== "function") return 1;
|
|
8300
|
+
return widestPatternSplit(largest);
|
|
8301
|
+
}
|
|
8224
8302
|
function takeMainCopyTurn(state) {
|
|
8225
8303
|
if (state.mainCopyQueries === "none" || state.mainCopyTurnTaken) return false;
|
|
8226
8304
|
state.mainCopyTurnTaken = true;
|
|
@@ -8234,7 +8312,7 @@ function answersFasterOnMainCopy(state, pool, manager) {
|
|
|
8234
8312
|
if (manager.countDocuments() > MAIN_COPY_LONE_QUERY_DOCUMENTS) return false;
|
|
8235
8313
|
return takeMainCopyTurn(state);
|
|
8236
8314
|
}
|
|
8237
|
-
async function searchViaWorker(state, indexName, params, globalStats, partitionIds) {
|
|
8315
|
+
async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork) {
|
|
8238
8316
|
const pool = state.workerPool;
|
|
8239
8317
|
if (!pool) return null;
|
|
8240
8318
|
if (!state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) return null;
|
|
@@ -8242,19 +8320,30 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
|
|
|
8242
8320
|
if (!manager) return null;
|
|
8243
8321
|
if (answersFasterOnMainCopy(state, pool, manager)) return null;
|
|
8244
8322
|
const scope = partitionIds ?? Array.from({ length: manager.partitionCount }, (_, partitionId) => partitionId);
|
|
8245
|
-
const idle = pool.leaseIdle(scope.length);
|
|
8323
|
+
const idle = pool.leaseIdle(Math.min(scope.length, patternSplitWidth(state, indexName, params)));
|
|
8246
8324
|
const leases = [];
|
|
8325
|
+
let fork = null;
|
|
8247
8326
|
try {
|
|
8248
8327
|
if (idle.length >= 2 && scoresPerPartition(state, indexName, params, globalStats)) {
|
|
8249
8328
|
leases.push(...idle);
|
|
8250
|
-
|
|
8329
|
+
fork = patternWork.fork(leases.length);
|
|
8330
|
+
const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared);
|
|
8331
|
+
fork.settle(merged.patternWorkSpent ?? 0);
|
|
8332
|
+
return merged;
|
|
8251
8333
|
}
|
|
8252
8334
|
for (const lease2 of idle.slice(1)) lease2.release();
|
|
8253
8335
|
const lease = idle[0] ?? (takeMainCopyTurn(state) ? null : pool.leaseLeastBusy());
|
|
8254
8336
|
if (lease === null) return null;
|
|
8255
8337
|
leases.push(lease);
|
|
8256
|
-
|
|
8338
|
+
fork = patternWork.fork(1);
|
|
8339
|
+
const result = await lease.executor.execute(
|
|
8340
|
+
queryAction(indexName, params, globalStats, partitionIds, fork.shared)
|
|
8341
|
+
);
|
|
8342
|
+
fork.settle(result.patternWorkSpent ?? 0);
|
|
8343
|
+
return result;
|
|
8257
8344
|
} catch (err) {
|
|
8345
|
+
if (isWorkCapError(err)) throw err;
|
|
8346
|
+
fork?.settle(0);
|
|
8258
8347
|
console.warn("Worker search failed, falling back to local:", err);
|
|
8259
8348
|
return null;
|
|
8260
8349
|
} finally {
|
|
@@ -8483,7 +8572,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
8483
8572
|
segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
|
|
8484
8573
|
holdUnbroadcastSegments: (indexName, segmentIds) => holdUnbroadcastSegments(state, indexName, segmentIds),
|
|
8485
8574
|
releaseUnbroadcastSegments: (indexName, segmentIds) => releaseUnbroadcastSegments(state, indexName, segmentIds),
|
|
8486
|
-
searchViaWorker: (indexName, params, globalStats, partitionIds) => searchViaWorker(state, indexName, params, globalStats, partitionIds),
|
|
8575
|
+
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork),
|
|
8487
8576
|
hasWorkerPool: () => state.workerPool !== null,
|
|
8488
8577
|
mainCopyQueries: () => state.mainCopyQueries ?? "lone",
|
|
8489
8578
|
shareMainThread: () => {
|
|
@@ -8772,7 +8861,8 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
8772
8861
|
analysisRebuild,
|
|
8773
8862
|
indexState,
|
|
8774
8863
|
mutationCtx,
|
|
8775
|
-
rebalanceCtx
|
|
8864
|
+
rebalanceCtx,
|
|
8865
|
+
patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
|
|
8776
8866
|
};
|
|
8777
8867
|
}
|
|
8778
8868
|
|
|
@@ -8872,6 +8962,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
8872
8962
|
}
|
|
8873
8963
|
const schema = validateSchema(requestedConfig.schema);
|
|
8874
8964
|
const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
8965
|
+
requirePatternSearchFor(schema);
|
|
8875
8966
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
8876
8967
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
8877
8968
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
@@ -9043,15 +9134,8 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
9043
9134
|
async function insertDocument(ctx, indexName, document, docId, options) {
|
|
9044
9135
|
ctx.guardShutdown();
|
|
9045
9136
|
const entry = ctx.requireIndex(indexName);
|
|
9046
|
-
|
|
9047
|
-
|
|
9048
|
-
throw new NarsilError(
|
|
9049
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
9050
|
-
`The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
|
|
9051
|
-
{ docId, documentId: embeddedDocId }
|
|
9052
|
-
);
|
|
9053
|
-
}
|
|
9054
|
-
const resolvedDocId = docId ?? embeddedDocId ?? ctx.idGenerator();
|
|
9137
|
+
if (docId !== void 0) requireMatchingDocId(docId, document);
|
|
9138
|
+
const resolvedDocId = docId ?? providedDocId(document) ?? ctx.idGenerator();
|
|
9055
9139
|
validateDocId(resolvedDocId);
|
|
9056
9140
|
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document });
|
|
9057
9141
|
if (entry.config.required && entry.config.required.length > 0) {
|
|
@@ -9288,6 +9372,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
|
|
|
9288
9372
|
if (chunkFailedIndexes.has(i)) continue;
|
|
9289
9373
|
const docId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
9290
9374
|
try {
|
|
9375
|
+
requireDocumentObject(documents[i]);
|
|
9291
9376
|
validateDocId(docId);
|
|
9292
9377
|
if (hasBeforeHook) {
|
|
9293
9378
|
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
|
|
@@ -9786,6 +9871,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
9786
9871
|
if (chunkFailedIndexes.has(i)) continue;
|
|
9787
9872
|
const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
9788
9873
|
try {
|
|
9874
|
+
requireDocumentObject(documents[i]);
|
|
9789
9875
|
validateDocId(batchDocId);
|
|
9790
9876
|
if (hasBeforeHook) {
|
|
9791
9877
|
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
|
|
@@ -10008,6 +10094,7 @@ async function settleUpdate(ctx, indexName, prepared) {
|
|
|
10008
10094
|
async function prepareUpdate(ctx, indexName, entry, docId, document) {
|
|
10009
10095
|
ctx.guardShutdown();
|
|
10010
10096
|
validateDocId(docId);
|
|
10097
|
+
requireMatchingDocId(docId, document);
|
|
10011
10098
|
const manager = ctx.requireManager(indexName);
|
|
10012
10099
|
if (ctx.pluginRegistry.hasHooks("beforeUpdate")) {
|
|
10013
10100
|
await ctx.pluginRegistry.runHook("beforeUpdate", {
|
|
@@ -10117,6 +10204,22 @@ async function updateDocument(ctx, indexName, docId, document, options) {
|
|
|
10117
10204
|
ctx.checkHeapPressure(indexName);
|
|
10118
10205
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
10119
10206
|
}
|
|
10207
|
+
function reportedDocIdOf(update) {
|
|
10208
|
+
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
10209
|
+
return typeof docId === "string" ? docId : "";
|
|
10210
|
+
}
|
|
10211
|
+
function requireUpdateEntry(update) {
|
|
10212
|
+
if (update !== null && typeof update === "object" && !Array.isArray(update)) {
|
|
10213
|
+
const { docId, document } = update;
|
|
10214
|
+
if (typeof docId === "string" && document !== null && typeof document === "object" && !Array.isArray(document)) {
|
|
10215
|
+
return { document };
|
|
10216
|
+
}
|
|
10217
|
+
}
|
|
10218
|
+
throw new NarsilError(
|
|
10219
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
10220
|
+
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
10221
|
+
);
|
|
10222
|
+
}
|
|
10120
10223
|
async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
10121
10224
|
ctx.guardShutdown();
|
|
10122
10225
|
const entry = ctx.requireIndex(indexName);
|
|
@@ -10129,10 +10232,13 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
10129
10232
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
10130
10233
|
const prepared = [];
|
|
10131
10234
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
10235
|
+
const update = updates[i];
|
|
10236
|
+
const docId = reportedDocIdOf(update);
|
|
10132
10237
|
try {
|
|
10133
|
-
|
|
10238
|
+
const { document } = requireUpdateEntry(update);
|
|
10239
|
+
prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document));
|
|
10134
10240
|
} catch (err) {
|
|
10135
|
-
failed.push({ docId
|
|
10241
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
10136
10242
|
}
|
|
10137
10243
|
}
|
|
10138
10244
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
@@ -10770,7 +10876,8 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
10770
10876
|
indexName,
|
|
10771
10877
|
broadcastStats: broadcastStatsFor(core, options),
|
|
10772
10878
|
partitionIds: options?.partitionIds,
|
|
10773
|
-
cursorBinding: queryBindingOf(params)
|
|
10879
|
+
cursorBinding: queryBindingOf(params),
|
|
10880
|
+
patternWorkCap: core.patternWorkCap
|
|
10774
10881
|
});
|
|
10775
10882
|
if (core.pluginRegistry.hasHooks("afterSearch")) {
|
|
10776
10883
|
const observed = { ...result, hits: [...result.hits] };
|
|
@@ -10811,7 +10918,8 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
10811
10918
|
indexName,
|
|
10812
10919
|
broadcastStats: broadcastStatsFor(core, options),
|
|
10813
10920
|
partitionIds: options?.partitionIds,
|
|
10814
|
-
cursorBinding: queryBindingOf(params)
|
|
10921
|
+
cursorBinding: queryBindingOf(params),
|
|
10922
|
+
patternWorkCap: core.patternWorkCap
|
|
10815
10923
|
});
|
|
10816
10924
|
if (core.analysisRebuild.isStale(indexName)) {
|
|
10817
10925
|
result.analysisStale = true;
|
|
@@ -10873,7 +10981,8 @@ async function runEngineListDocuments(core, indexName, params, partitionIds) {
|
|
|
10873
10981
|
manager: core.requireManager(indexName),
|
|
10874
10982
|
schema: entry.config.schema,
|
|
10875
10983
|
strict: entry.config.strict,
|
|
10876
|
-
partitionIds
|
|
10984
|
+
partitionIds,
|
|
10985
|
+
patternWorkCap: core.patternWorkCap
|
|
10877
10986
|
});
|
|
10878
10987
|
} finally {
|
|
10879
10988
|
release();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { validateRequiredFields } from './chunk-
|
|
2
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
1
|
+
import { validateRequiredFields } from './chunk-M6XF5HDE.mjs';
|
|
2
|
+
import { ErrorCodes, NarsilError } from './chunk-RV77FR2F.mjs';
|
|
3
3
|
|
|
4
4
|
// src/narsil/internals.ts
|
|
5
5
|
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
@@ -239,9 +239,30 @@ async function embedBatchDocumentFields(documents, embeddingConfig, adapter, sig
|
|
|
239
239
|
|
|
240
240
|
// src/engine/mutations/insert-admission.ts
|
|
241
241
|
function providedDocId(document) {
|
|
242
|
+
if (document === null || typeof document !== "object") return void 0;
|
|
242
243
|
const id = document.id;
|
|
243
244
|
return typeof id === "string" && id.length > 0 ? id : void 0;
|
|
244
245
|
}
|
|
246
|
+
function requireDocumentObject(document) {
|
|
247
|
+
if (document !== null && typeof document === "object" && !Array.isArray(document)) return;
|
|
248
|
+
const received = Array.isArray(document) ? "a list" : document === null ? "null" : `a ${typeof document}`;
|
|
249
|
+
throw new NarsilError(
|
|
250
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
251
|
+
`Each document must be an object, while this entry is ${received}`,
|
|
252
|
+
{
|
|
253
|
+
received
|
|
254
|
+
}
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
function requireMatchingDocId(docId, document) {
|
|
258
|
+
const embeddedDocId = providedDocId(document);
|
|
259
|
+
if (embeddedDocId === void 0 || embeddedDocId === docId) return;
|
|
260
|
+
throw new NarsilError(
|
|
261
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
262
|
+
`The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
|
|
263
|
+
{ docId, documentId: embeddedDocId }
|
|
264
|
+
);
|
|
265
|
+
}
|
|
245
266
|
function asBatchInsertError(err) {
|
|
246
267
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
247
268
|
}
|
|
@@ -566,4 +587,4 @@ function restoredConfigFields(envelope) {
|
|
|
566
587
|
};
|
|
567
588
|
}
|
|
568
589
|
|
|
569
|
-
export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };
|
|
590
|
+
export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, requireDocumentObject, requireMatchingDocId, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { MAX_SORT_FIELDS } from './chunk-
|
|
2
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-
|
|
3
|
-
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-
|
|
1
|
+
import { MAX_SORT_FIELDS } from './chunk-M6XF5HDE.mjs';
|
|
2
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-P2ZZN32C.mjs';
|
|
3
|
+
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-RV77FR2F.mjs';
|
|
4
4
|
|
|
5
5
|
// src/server/capabilities.ts
|
|
6
6
|
var ASYNC_IMPORT_CAPABILITY = "documents.import.async";
|
|
@@ -113,6 +113,7 @@ var STATUS_BY_CODE = {
|
|
|
113
113
|
SEARCH_INVALID_MODE: 400,
|
|
114
114
|
SEARCH_INVALID_CURSOR: 400,
|
|
115
115
|
SEARCH_RESULT_WINDOW_EXCEEDED: 400,
|
|
116
|
+
SEARCH_WORK_CAP_EXCEEDED: 400,
|
|
116
117
|
LANGUAGE_NOT_SUPPORTED: 400,
|
|
117
118
|
CONFIG_INVALID: 400,
|
|
118
119
|
EMBEDDING_DIMENSION_MISMATCH: 400,
|