@delali/narsil 0.4.1-next.215 → 0.4.2-next.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-DPRCAVC2.mjs} +14 -8
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-WRRXR57N.mjs} +1 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-XBWABEL3.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-Y5CSZDOH.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-FEPCLXXY.mjs} +7 -1
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-HVPPDHSY.mjs} +259 -104
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-DBC4EJWB.mjs → chunk-33ZMFOAR.mjs} +1 -1
- package/dist/{chunk-NCIGNQJE.mjs → chunk-4GCHJ3XD.mjs} +930 -387
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-4WQNWAQE.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-5E3TJ4P3.mjs} +5 -5
- package/dist/{chunk-GXCDWLQI.mjs → chunk-5GYGHPCH.mjs} +7 -1
- package/dist/{chunk-SMQPZHPS.mjs → chunk-ATQ6JR3I.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-BBIAUHAB.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-DA32WXBT.mjs} +2 -2
- package/dist/{chunk-MSJD74GN.mjs → chunk-EALKCTWH.mjs} +5 -5
- package/dist/{chunk-755NM4PL.mjs → chunk-FOGTBAYN.mjs} +183 -21
- package/dist/{chunk-6GHX35ZM.mjs → chunk-GTBJSNN2.mjs} +1 -1
- package/dist/{chunk-DDUXEJXW.mjs → chunk-H6LWGMPQ.mjs} +45 -22
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-IL5DRCNU.mjs → chunk-KQWTS7AM.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-KX3OP6VP.mjs} +2 -2
- package/dist/{chunk-D456ZFPQ.mjs → chunk-MIIJ5O3O.mjs} +2 -2
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-MS746XNY.mjs} +285 -118
- package/dist/{chunk-WM5KJSQV.mjs → chunk-NFA3WHJU.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-OETLM4EH.mjs} +1 -1
- package/dist/{chunk-MYN4K5IS.mjs → chunk-OLXFIUIZ.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-X3EQCQLP.mjs → chunk-Q4NLL66U.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-QJFKL4NK.mjs} +4 -4
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-3IE3A74H.mjs → chunk-SSITKQU6.mjs} +3 -3
- package/dist/{chunk-B3MO37YF.mjs → chunk-TANKI7TG.mjs} +1 -1
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-TI4G3SIG.mjs} +63 -144
- package/dist/{chunk-OGYSPNL2.mjs → chunk-TT3UDDL3.mjs} +7 -7
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-VHIOGDVD.mjs} +1 -1
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-W3JH2IK6.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-WIT6B5OB.mjs} +2 -2
- package/dist/{chunk-MI6UZGUR.mjs → chunk-XGVSG6ZU.mjs} +3 -3
- package/dist/{chunk-6I6OO6B2.mjs → chunk-XWPNRHTG.mjs} +4 -4
- package/dist/{chunk-WTMCGZ74.mjs → chunk-YHZSSPG3.mjs} +2 -2
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-PCBB6FCO.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-4ALYIIC4.mjs} +14 -14
- package/dist/msgpack-leaf-6ZBZWFRD.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-2OK4TOLY.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-Y5OAKNRB.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-UQVQZFIJ.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-4OOUY2K7.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-5MYHYBET.mjs} +286 -118
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AVCMTWET.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-GS6LGXY3.mjs} +1 -1
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-LE4DTMDL.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-VIO5IPF7.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-VRI5XH2M.mjs} +7 -1
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-XGS2UYVI.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-XW6GZQWP.mjs} +4 -4
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-Z44S6WW5.mjs} +2 -2
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-4MIGXWZT.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-LZDHB7FX.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-J7SB263E.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PEKX7TBL.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-I2EC7H6R.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-RQXM7HBO.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-EZGCLSLM.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-JNGH2JYC.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-2OWLLBCV.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-TQ4SU76L.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-W7U2X6V3.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-WGP4FBXF.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-FYWJQSPL.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-GRG44RQA.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-YDJVH73G.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-5UFJV7SC.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-6YLVL2NC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-IP4L7KEW.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -10
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -1866,7 +1866,7 @@ function epochStatesOfMetadata(metadata) {
|
|
|
1866
1866
|
|
|
1867
1867
|
// src/engine/index-from-metadata.ts
|
|
1868
1868
|
function isRecoveryConfigError(err) {
|
|
1869
|
-
return err instanceof NarsilError && (err.code === ErrorCodes.CONFIG_INVALID || err.code === ErrorCodes.SCHEMA_INVALID_TYPE || err.code === ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION);
|
|
1869
|
+
return err instanceof NarsilError && (err.code === ErrorCodes.CONFIG_INVALID || err.code === ErrorCodes.LANGUAGE_NOT_SUPPORTED || err.code === ErrorCodes.SCHEMA_INVALID_TYPE || err.code === ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION);
|
|
1870
1870
|
}
|
|
1871
1871
|
function resolveEmbeddingAdapter(deps, metadata, schema) {
|
|
1872
1872
|
const name = metadata.embedding?.adapter ?? null;
|
|
@@ -1936,8 +1936,9 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1936
1936
|
return;
|
|
1937
1937
|
}
|
|
1938
1938
|
const indexConfig = reconstructSchemaFromMetadata(metadata);
|
|
1939
|
-
|
|
1939
|
+
let language;
|
|
1940
1940
|
try {
|
|
1941
|
+
language = getLanguage(indexConfig.language ?? "english");
|
|
1941
1942
|
indexConfig.schema = validateSchema(indexConfig.schema);
|
|
1942
1943
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
1943
1944
|
requireSurfaceFormsForPartialFields(indexConfig.schema, indexConfig.surfaceForms);
|
|
@@ -2002,6 +2003,51 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
2002
2003
|
}
|
|
2003
2004
|
}
|
|
2004
2005
|
|
|
2006
|
+
// src/engine/mutations/batch-shutdown.ts
|
|
2007
|
+
function isFailureThatShutdownCauses(code) {
|
|
2008
|
+
return code === ErrorCodes.INSTANCE_SHUT_DOWN || code === ErrorCodes.INDEX_NOT_FOUND || code === ErrorCodes.EMBEDDING_FAILED;
|
|
2009
|
+
}
|
|
2010
|
+
function shutDownStarted(ctx) {
|
|
2011
|
+
return ctx.abortController.signal.aborted;
|
|
2012
|
+
}
|
|
2013
|
+
function shutDownError() {
|
|
2014
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
2015
|
+
}
|
|
2016
|
+
function causedByShutdown(ctx, err) {
|
|
2017
|
+
const { signal } = ctx.abortController;
|
|
2018
|
+
if (!signal.aborted) return false;
|
|
2019
|
+
if (err === signal.reason) return true;
|
|
2020
|
+
if (err instanceof NarsilError) return isFailureThatShutdownCauses(err.code);
|
|
2021
|
+
return err instanceof Error && err.name === "AbortError";
|
|
2022
|
+
}
|
|
2023
|
+
function batchFailureOf(ctx, err, fallbackCode) {
|
|
2024
|
+
if (causedByShutdown(ctx, err)) {
|
|
2025
|
+
return err instanceof NarsilError && err.code === ErrorCodes.INSTANCE_SHUT_DOWN ? err : shutDownError();
|
|
2026
|
+
}
|
|
2027
|
+
return err instanceof NarsilError ? err : new NarsilError(fallbackCode, String(err));
|
|
2028
|
+
}
|
|
2029
|
+
function docIdOf(entry) {
|
|
2030
|
+
return entry.docId;
|
|
2031
|
+
}
|
|
2032
|
+
function failAsShutDown(failed, entries, reportedDocIdOf2, start = 0, end = entries.length) {
|
|
2033
|
+
if (start >= end) return;
|
|
2034
|
+
const error = shutDownError();
|
|
2035
|
+
for (let i = start; i < end; i++) failed.push({ docId: reportedDocIdOf2(entries[i]), error });
|
|
2036
|
+
}
|
|
2037
|
+
function applyUnlessShutDown(ctx, apply) {
|
|
2038
|
+
return () => {
|
|
2039
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
2040
|
+
return apply();
|
|
2041
|
+
};
|
|
2042
|
+
}
|
|
2043
|
+
async function unlessShutDown(ctx, work) {
|
|
2044
|
+
try {
|
|
2045
|
+
await work();
|
|
2046
|
+
} catch (err) {
|
|
2047
|
+
if (!shutDownStarted(ctx)) throw err;
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2005
2051
|
// src/engine/mutations/batch-admissions.ts
|
|
2006
2052
|
function createPendingAdmissions() {
|
|
2007
2053
|
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
@@ -2012,8 +2058,11 @@ function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
|
2012
2058
|
const partitionId = manager.routePartition(docId);
|
|
2013
2059
|
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
2014
2060
|
}
|
|
2015
|
-
function asBatchInsertError(err) {
|
|
2016
|
-
return
|
|
2061
|
+
function asBatchInsertError(ctx, err) {
|
|
2062
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_VALIDATION_FAILED);
|
|
2063
|
+
}
|
|
2064
|
+
function providedDocIdOrEmpty(document) {
|
|
2065
|
+
return providedDocId(document) ?? "";
|
|
2017
2066
|
}
|
|
2018
2067
|
|
|
2019
2068
|
// src/engine/mutations/outside-hold.ts
|
|
@@ -2243,7 +2292,11 @@ async function prepareInsertDocuments(ctx, indexName, documents, start, end, hol
|
|
|
2243
2292
|
const pending = [];
|
|
2244
2293
|
for (let i = start; i < end; i++) {
|
|
2245
2294
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2246
|
-
if (ctx
|
|
2295
|
+
if (shutDownStarted(ctx)) {
|
|
2296
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
2297
|
+
failAsShutDown(failed, documents, providedDocIdOrEmpty, i, end);
|
|
2298
|
+
return null;
|
|
2299
|
+
}
|
|
2247
2300
|
const document = documents[i];
|
|
2248
2301
|
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
2249
2302
|
try {
|
|
@@ -2252,17 +2305,26 @@ async function prepareInsertDocuments(ctx, indexName, documents, start, end, hol
|
|
|
2252
2305
|
validateDocId(docId);
|
|
2253
2306
|
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
2254
2307
|
} catch (err) {
|
|
2255
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2308
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
2256
2309
|
}
|
|
2257
2310
|
}
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2311
|
+
let prepared;
|
|
2312
|
+
try {
|
|
2313
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
2314
|
+
prepared = await prepareDocumentWrites(
|
|
2315
|
+
ctx,
|
|
2316
|
+
indexName,
|
|
2317
|
+
pending.map((item) => item.preparation),
|
|
2318
|
+
hold,
|
|
2319
|
+
true,
|
|
2320
|
+
earlier
|
|
2321
|
+
);
|
|
2322
|
+
} catch (err) {
|
|
2323
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
2324
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
2325
|
+
return null;
|
|
2326
|
+
}
|
|
2327
|
+
const { entry, embedding } = prepared;
|
|
2266
2328
|
const candidates = [];
|
|
2267
2329
|
for (const item of pending) {
|
|
2268
2330
|
const { document, failure } = item.preparation;
|
|
@@ -2270,22 +2332,19 @@ async function prepareInsertDocuments(ctx, indexName, documents, start, end, hol
|
|
|
2270
2332
|
candidates.push({ docId: item.docId, document });
|
|
2271
2333
|
continue;
|
|
2272
2334
|
}
|
|
2273
|
-
const docId = item.refusedByHook ? item.docId :
|
|
2274
|
-
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
2335
|
+
const docId = item.refusedByHook ? item.docId : providedDocIdOrEmpty(document);
|
|
2336
|
+
failed.push({ docId, error: asBatchInsertError(ctx, failure) });
|
|
2275
2337
|
}
|
|
2276
2338
|
return { entry, candidates, embedding };
|
|
2277
2339
|
}
|
|
2278
2340
|
async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
|
|
2279
|
-
const
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
true,
|
|
2287
|
-
failed
|
|
2288
|
-
);
|
|
2341
|
+
const prepared = await prepareInsertDocuments(ctx, indexName, documents, 0, documents.length, hold, true, failed);
|
|
2342
|
+
if (prepared === null) return { admitted: [], embedding: null };
|
|
2343
|
+
const { entry, candidates, embedding } = prepared;
|
|
2344
|
+
if (shutDownStarted(ctx)) {
|
|
2345
|
+
failAsShutDown(failed, candidates, docIdOf);
|
|
2346
|
+
return { admitted: [], embedding };
|
|
2347
|
+
}
|
|
2289
2348
|
const manager = ctx.requireManager(indexName);
|
|
2290
2349
|
const vecIndexes = manager.getVectorIndexes();
|
|
2291
2350
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
@@ -2295,7 +2354,11 @@ async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
|
|
|
2295
2354
|
const pending = createPendingAdmissions();
|
|
2296
2355
|
for (let position = 0; position < candidates.length; position++) {
|
|
2297
2356
|
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
2298
|
-
if (ctx
|
|
2357
|
+
if (shutDownStarted(ctx)) {
|
|
2358
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
2359
|
+
failAsShutDown(failed, candidates, docIdOf, position);
|
|
2360
|
+
return { admitted: [], embedding };
|
|
2361
|
+
}
|
|
2299
2362
|
const { docId, document } = candidates[position];
|
|
2300
2363
|
try {
|
|
2301
2364
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
@@ -2317,7 +2380,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
|
|
|
2317
2380
|
recordPendingAdmission(ctx, pending, manager, docId);
|
|
2318
2381
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2319
2382
|
} catch (err) {
|
|
2320
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2383
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
2321
2384
|
}
|
|
2322
2385
|
}
|
|
2323
2386
|
return { admitted, embedding };
|
|
@@ -2467,9 +2530,9 @@ async function recordChunk(ctx, indexName, mutations, applies) {
|
|
|
2467
2530
|
function applyInsertDocument(ctx, indexName, doc, options, progress) {
|
|
2468
2531
|
const manager = ctx.requireManager(indexName);
|
|
2469
2532
|
const insertIntoIndex = () => {
|
|
2470
|
-
if (ctx
|
|
2533
|
+
if (shutDownStarted(ctx)) {
|
|
2471
2534
|
progress.skipped = true;
|
|
2472
|
-
throw
|
|
2535
|
+
throw shutDownError();
|
|
2473
2536
|
}
|
|
2474
2537
|
admitInsert(ctx, indexName, manager, doc.docId);
|
|
2475
2538
|
const queued = ctx.bufferIfRebalancing(indexName, {
|
|
@@ -2525,11 +2588,11 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
2525
2588
|
const progress = docs.map(() => ({ inserted: false, buffered: false, skipped: false }));
|
|
2526
2589
|
const applies = docs.map((doc, i) => applyInsertDocument(ctx, indexName, doc, options, progress[i]));
|
|
2527
2590
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2528
|
-
await ctx.orchestrator.settlePublications(indexName);
|
|
2591
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName));
|
|
2529
2592
|
const applications = [];
|
|
2530
2593
|
for (let i = 0; i < docs.length; i++) {
|
|
2531
2594
|
if (progress[i].skipped) {
|
|
2532
|
-
applications.push({ status: "
|
|
2595
|
+
applications.push({ status: "failed", error: shutDownError() });
|
|
2533
2596
|
continue;
|
|
2534
2597
|
}
|
|
2535
2598
|
const failure = failures[i];
|
|
@@ -2562,27 +2625,33 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
2562
2625
|
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
2563
2626
|
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
2564
2627
|
const manager = ctx.executor.getManager(indexName);
|
|
2565
|
-
await
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2628
|
+
await unlessShutDown(
|
|
2629
|
+
ctx,
|
|
2630
|
+
() => runOutside(hold, async () => {
|
|
2631
|
+
for (const doc of inserted) {
|
|
2632
|
+
try {
|
|
2633
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2634
|
+
} catch (err) {
|
|
2635
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2636
|
+
}
|
|
2571
2637
|
}
|
|
2572
|
-
}
|
|
2573
|
-
|
|
2574
|
-
return ctx.executor.getManager(indexName) !== manager;
|
|
2638
|
+
})
|
|
2639
|
+
);
|
|
2640
|
+
return shutDownStarted(ctx) || ctx.executor.getManager(indexName) !== manager;
|
|
2575
2641
|
}
|
|
2576
2642
|
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
2577
2643
|
const succeeded = [];
|
|
2578
2644
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2579
2645
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2580
|
-
if (ctx
|
|
2646
|
+
if (shutDownStarted(ctx)) {
|
|
2647
|
+
failAsShutDown(failed, admitted, docIdOf, chunkStart);
|
|
2648
|
+
break;
|
|
2649
|
+
}
|
|
2581
2650
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
2582
2651
|
try {
|
|
2583
2652
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2584
2653
|
} catch (err) {
|
|
2585
|
-
const error = asBatchInsertError(err);
|
|
2654
|
+
const error = asBatchInsertError(ctx, err);
|
|
2586
2655
|
for (let i = chunkStart; i < admitted.length; i++) failed.push({ docId: admitted[i].docId, error });
|
|
2587
2656
|
break;
|
|
2588
2657
|
}
|
|
@@ -2591,9 +2660,8 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
2591
2660
|
for (let i = 0; i < chunk.length; i++) {
|
|
2592
2661
|
const doc = chunk[i];
|
|
2593
2662
|
const application = applications[i];
|
|
2594
|
-
if (application.status === "skipped") continue;
|
|
2595
2663
|
if (application.status === "failed") {
|
|
2596
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2664
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
2597
2665
|
continue;
|
|
2598
2666
|
}
|
|
2599
2667
|
succeeded.push(doc.docId);
|
|
@@ -2608,7 +2676,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
2608
2676
|
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
2609
2677
|
}
|
|
2610
2678
|
if (applied < admitted.length) {
|
|
2611
|
-
ctx.checkHeapPressure(indexName);
|
|
2679
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
2612
2680
|
await new Promise((r) => setTimeout(r, 0));
|
|
2613
2681
|
}
|
|
2614
2682
|
}
|
|
@@ -2617,7 +2685,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
2617
2685
|
|
|
2618
2686
|
// src/engine/mutations/insert-batch-chunks.ts
|
|
2619
2687
|
async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
2620
|
-
const
|
|
2688
|
+
const prepared = await prepareInsertDocuments(
|
|
2621
2689
|
ctx,
|
|
2622
2690
|
indexName,
|
|
2623
2691
|
documents,
|
|
@@ -2628,13 +2696,24 @@ async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold,
|
|
|
2628
2696
|
failed,
|
|
2629
2697
|
earlier
|
|
2630
2698
|
);
|
|
2699
|
+
if (prepared === null) return [];
|
|
2700
|
+
const { entry, candidates } = prepared;
|
|
2701
|
+
const admitted = [];
|
|
2702
|
+
if (shutDownStarted(ctx)) {
|
|
2703
|
+
failAsShutDown(failed, candidates, docIdOf);
|
|
2704
|
+
return admitted;
|
|
2705
|
+
}
|
|
2631
2706
|
const manager = ctx.requireManager(indexName);
|
|
2632
2707
|
const vecIndexes = manager.getVectorIndexes();
|
|
2633
2708
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2634
|
-
|
|
2635
|
-
for (const { docId, document } of candidates) {
|
|
2709
|
+
for (let position = 0; position < candidates.length; position++) {
|
|
2636
2710
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2637
|
-
if (ctx
|
|
2711
|
+
if (shutDownStarted(ctx)) {
|
|
2712
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
2713
|
+
failAsShutDown(failed, candidates, docIdOf, position);
|
|
2714
|
+
return [];
|
|
2715
|
+
}
|
|
2716
|
+
const { docId, document } = candidates[position];
|
|
2638
2717
|
try {
|
|
2639
2718
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2640
2719
|
document,
|
|
@@ -2646,7 +2725,7 @@ async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold,
|
|
|
2646
2725
|
admitInsert(ctx, indexName, manager, docId);
|
|
2647
2726
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2648
2727
|
} catch (err) {
|
|
2649
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2728
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
2650
2729
|
}
|
|
2651
2730
|
}
|
|
2652
2731
|
return admitted;
|
|
@@ -2655,17 +2734,25 @@ async function insertInChunks(ctx, indexName, documents, options, hold, hooked,
|
|
|
2655
2734
|
const succeeded = [];
|
|
2656
2735
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2657
2736
|
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2658
|
-
if (ctx
|
|
2737
|
+
if (shutDownStarted(ctx)) {
|
|
2738
|
+
failAsShutDown(failed, documents, providedDocIdOrEmpty, chunkStart);
|
|
2739
|
+
break;
|
|
2740
|
+
}
|
|
2659
2741
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2660
2742
|
const earlier = embeddingOfDocuments(embedded, documents.slice(chunkStart, chunkEnd));
|
|
2661
2743
|
const prepared = await admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
2744
|
+
if (shutDownStarted(ctx)) {
|
|
2745
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
2746
|
+
failAsShutDown(failed, documents, providedDocIdOrEmpty, chunkEnd);
|
|
2747
|
+
break;
|
|
2748
|
+
}
|
|
2662
2749
|
try {
|
|
2663
2750
|
if (prepared.length > 0) ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2664
2751
|
} catch (err) {
|
|
2665
|
-
const error = asBatchInsertError(err);
|
|
2752
|
+
const error = asBatchInsertError(ctx, err);
|
|
2666
2753
|
for (const doc of prepared) failed.push({ docId: doc.docId, error });
|
|
2667
2754
|
for (let i = chunkEnd; i < documents.length; i++) {
|
|
2668
|
-
failed.push({ docId:
|
|
2755
|
+
failed.push({ docId: providedDocIdOrEmpty(documents[i]), error });
|
|
2669
2756
|
}
|
|
2670
2757
|
break;
|
|
2671
2758
|
}
|
|
@@ -2675,9 +2762,8 @@ async function insertInChunks(ctx, indexName, documents, options, hold, hooked,
|
|
|
2675
2762
|
for (let i = 0; i < prepared.length; i++) {
|
|
2676
2763
|
const doc = prepared[i];
|
|
2677
2764
|
const application = applications[i];
|
|
2678
|
-
if (application.status === "skipped") continue;
|
|
2679
2765
|
if (application.status === "failed") {
|
|
2680
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2766
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
2681
2767
|
continue;
|
|
2682
2768
|
}
|
|
2683
2769
|
succeeded.push(doc.docId);
|
|
@@ -2688,21 +2774,24 @@ async function insertInChunks(ctx, indexName, documents, options, hold, hooked,
|
|
|
2688
2774
|
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
2689
2775
|
}
|
|
2690
2776
|
if (chunkEnd < documents.length) {
|
|
2691
|
-
ctx.checkHeapPressure(indexName);
|
|
2777
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
2692
2778
|
await new Promise((r) => setTimeout(r, 0));
|
|
2693
2779
|
}
|
|
2694
2780
|
}
|
|
2695
2781
|
return { succeeded, touchedVectorFields };
|
|
2696
2782
|
}
|
|
2697
2783
|
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
2784
|
+
if (shutDownStarted(ctx)) return;
|
|
2698
2785
|
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
2699
2786
|
for (const fieldPath of inserts.touchedVectorFields) {
|
|
2700
2787
|
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
2701
2788
|
}
|
|
2702
2789
|
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
2703
2790
|
ctx.checkHeapPressure(indexName);
|
|
2704
|
-
await ctx
|
|
2705
|
-
|
|
2791
|
+
await unlessShutDown(ctx, async () => {
|
|
2792
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
2793
|
+
if (options?.wait === true && !shutDownStarted(ctx)) await awaitWriteVisibility(ctx, indexName);
|
|
2794
|
+
});
|
|
2706
2795
|
}
|
|
2707
2796
|
|
|
2708
2797
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -2711,7 +2800,7 @@ function applySegment(ctx, indexName, attachment, doc, options) {
|
|
|
2711
2800
|
const publish = async () => {
|
|
2712
2801
|
if (attachment.failure !== void 0) throw attachment.failure;
|
|
2713
2802
|
if (attachment.attached) return;
|
|
2714
|
-
if (ctx
|
|
2803
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
2715
2804
|
if (ctx.isRebalancing(indexName) || manager.partitionCount !== attachment.partitionCount) {
|
|
2716
2805
|
attachment.individual = true;
|
|
2717
2806
|
}
|
|
@@ -2865,6 +2954,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2865
2954
|
for (let index = 0; index < chunks.length; index++) {
|
|
2866
2955
|
const { docs, attachmentOf } = chunks[index];
|
|
2867
2956
|
try {
|
|
2957
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
2868
2958
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2869
2959
|
} catch (error) {
|
|
2870
2960
|
for (const later of chunks.slice(index)) {
|
|
@@ -2878,7 +2968,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2878
2968
|
return applySegment(ctx, indexName, attachment, doc, options);
|
|
2879
2969
|
});
|
|
2880
2970
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2881
|
-
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
2971
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName, "segments"));
|
|
2882
2972
|
const inserted = [];
|
|
2883
2973
|
for (let i = 0; i < docs.length; i++) {
|
|
2884
2974
|
const doc = docs[i];
|
|
@@ -2919,7 +3009,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2919
3009
|
break;
|
|
2920
3010
|
}
|
|
2921
3011
|
if (index + 1 < chunks.length) {
|
|
2922
|
-
ctx.checkHeapPressure(indexName);
|
|
3012
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
2923
3013
|
await new Promise((r) => setTimeout(r, 0));
|
|
2924
3014
|
}
|
|
2925
3015
|
}
|
|
@@ -2930,7 +3020,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2930
3020
|
continue;
|
|
2931
3021
|
}
|
|
2932
3022
|
if (!errorOf.has(doc.docId)) continue;
|
|
2933
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
3023
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, errorOf.get(doc.docId)) });
|
|
2934
3024
|
}
|
|
2935
3025
|
return { succeeded, touchedVectorFields, unrecorded };
|
|
2936
3026
|
}
|
|
@@ -2944,8 +3034,11 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
2944
3034
|
);
|
|
2945
3035
|
}
|
|
2946
3036
|
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
2947
|
-
|
|
2948
|
-
|
|
3037
|
+
const nothingIngested = () => ({ succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] });
|
|
3038
|
+
if (admitted.length === 0) return nothingIngested();
|
|
3039
|
+
if (shutDownStarted(ctx)) {
|
|
3040
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
3041
|
+
return nothingIngested();
|
|
2949
3042
|
}
|
|
2950
3043
|
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
2951
3044
|
const manager = ctx.requireManager(indexName);
|
|
@@ -2963,6 +3056,10 @@ async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
|
2963
3056
|
} catch {
|
|
2964
3057
|
built = null;
|
|
2965
3058
|
}
|
|
3059
|
+
if (shutDownStarted(ctx)) {
|
|
3060
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
3061
|
+
return nothingIngested();
|
|
3062
|
+
}
|
|
2966
3063
|
if (built === null) {
|
|
2967
3064
|
return applyIndividually();
|
|
2968
3065
|
}
|
|
@@ -3013,7 +3110,9 @@ async function insertBatchViaSegments(ctx, indexName, documents, options, hold)
|
|
|
3013
3110
|
const failed = [];
|
|
3014
3111
|
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents, failed, hold);
|
|
3015
3112
|
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
3016
|
-
if (
|
|
3113
|
+
if (shutDownStarted(ctx)) {
|
|
3114
|
+
failAsShutDown(failed, outcome.unrecorded, docIdOf);
|
|
3115
|
+
} else if (outcome.unrecorded.length > 0) {
|
|
3017
3116
|
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
3018
3117
|
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
3019
3118
|
outcome.succeeded.push(...rest.succeeded);
|
|
@@ -3028,20 +3127,24 @@ async function insertDocumentBatch(ctx, indexName, documents, options, hold) {
|
|
|
3028
3127
|
ctx.guardShutdown();
|
|
3029
3128
|
ctx.requireIndex(indexName);
|
|
3030
3129
|
validateBatchList(documents, "documents");
|
|
3031
|
-
|
|
3130
|
+
const failed = [];
|
|
3131
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length));
|
|
3132
|
+
if (shutDownStarted(ctx)) {
|
|
3133
|
+
failAsShutDown(failed, documents, providedDocIdOrEmpty);
|
|
3134
|
+
return { succeeded: [], failed };
|
|
3135
|
+
}
|
|
3032
3136
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
3033
3137
|
if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
3034
3138
|
return insertBatchViaSegments(ctx, indexName, documents, options, hold);
|
|
3035
3139
|
}
|
|
3036
|
-
const failed = [];
|
|
3037
3140
|
const inserts = await insertInChunks(ctx, indexName, documents, options, hold, true, failed);
|
|
3038
3141
|
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
3039
3142
|
return { succeeded: inserts.succeeded, failed };
|
|
3040
3143
|
}
|
|
3041
3144
|
|
|
3042
3145
|
// src/engine/mutations/remove.ts
|
|
3043
|
-
function asRemoveError(err) {
|
|
3044
|
-
return
|
|
3146
|
+
function asRemoveError(ctx, err) {
|
|
3147
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
3045
3148
|
}
|
|
3046
3149
|
function validateRemoval(ctx, docId) {
|
|
3047
3150
|
ctx.guardShutdown();
|
|
@@ -3149,18 +3252,31 @@ async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, h
|
|
|
3149
3252
|
}
|
|
3150
3253
|
}
|
|
3151
3254
|
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
3152
|
-
await
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3255
|
+
await unlessShutDown(
|
|
3256
|
+
ctx,
|
|
3257
|
+
() => runOutside(hold, async () => {
|
|
3258
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3259
|
+
if (refused.has(i)) continue;
|
|
3260
|
+
try {
|
|
3261
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
3262
|
+
} catch (err) {
|
|
3263
|
+
refused.set(i, err);
|
|
3264
|
+
}
|
|
3159
3265
|
}
|
|
3160
|
-
}
|
|
3161
|
-
|
|
3266
|
+
})
|
|
3267
|
+
);
|
|
3162
3268
|
return refused;
|
|
3163
3269
|
}
|
|
3270
|
+
function failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused) {
|
|
3271
|
+
const error = shutDownError();
|
|
3272
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3273
|
+
failed.push({ docId: docIds[i], error: refused.has(i) ? asRemoveError(ctx, refused.get(i)) : error });
|
|
3274
|
+
}
|
|
3275
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
3276
|
+
}
|
|
3277
|
+
function removalDocIdOf(docId) {
|
|
3278
|
+
return docId;
|
|
3279
|
+
}
|
|
3164
3280
|
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
3165
3281
|
ctx.guardShutdown();
|
|
3166
3282
|
ctx.requireIndex(indexName);
|
|
@@ -3172,8 +3288,16 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
3172
3288
|
let manager = first;
|
|
3173
3289
|
try {
|
|
3174
3290
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
3291
|
+
if (shutDownStarted(ctx)) {
|
|
3292
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkStart);
|
|
3293
|
+
break;
|
|
3294
|
+
}
|
|
3175
3295
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
3176
3296
|
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
3297
|
+
if (shutDownStarted(ctx)) {
|
|
3298
|
+
failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused);
|
|
3299
|
+
break;
|
|
3300
|
+
}
|
|
3177
3301
|
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3178
3302
|
const removed = [];
|
|
3179
3303
|
const prepared = [];
|
|
@@ -3183,10 +3307,15 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
3183
3307
|
if (refused.has(i)) throw refused.get(i);
|
|
3184
3308
|
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
3185
3309
|
} catch (err) {
|
|
3186
|
-
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
3310
|
+
failed.push({ docId: docIds[i], error: asRemoveError(ctx, err) });
|
|
3187
3311
|
}
|
|
3188
3312
|
}
|
|
3189
|
-
|
|
3313
|
+
if (shutDownStarted(ctx)) {
|
|
3314
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
3315
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
3316
|
+
break;
|
|
3317
|
+
}
|
|
3318
|
+
const applies = prepared.map((removal) => applyUnlessShutDown(ctx, applyOfRemoval(ctx, indexName, removal)));
|
|
3190
3319
|
try {
|
|
3191
3320
|
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
3192
3321
|
for (let i = 0; i < prepared.length; i++) {
|
|
@@ -3207,7 +3336,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
3207
3336
|
error = rollbackError;
|
|
3208
3337
|
}
|
|
3209
3338
|
removal.releaseVectorHolds?.();
|
|
3210
|
-
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
3339
|
+
failed.push({ docId: removal.docId, error: asRemoveError(ctx, error) });
|
|
3211
3340
|
continue;
|
|
3212
3341
|
}
|
|
3213
3342
|
removal.releaseVectorHolds?.();
|
|
@@ -3217,18 +3346,19 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
3217
3346
|
} finally {
|
|
3218
3347
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
3219
3348
|
}
|
|
3220
|
-
await settleRemovals(ctx, indexName, removed, hold);
|
|
3221
|
-
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3349
|
+
await unlessShutDown(ctx, () => settleRemovals(ctx, indexName, removed, hold));
|
|
3350
|
+
if (!shutDownStarted(ctx)) manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3222
3351
|
if (chunkEnd < docIds.length) {
|
|
3223
3352
|
await new Promise((r) => setTimeout(r, 0));
|
|
3224
3353
|
}
|
|
3225
3354
|
}
|
|
3226
3355
|
} finally {
|
|
3227
3356
|
const ending = manager;
|
|
3228
|
-
await ending?.publication.write(() => ending.endBatchRemove());
|
|
3357
|
+
await unlessShutDown(ctx, () => ending?.publication.write(() => ending.endBatchRemove()));
|
|
3229
3358
|
}
|
|
3359
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
3230
3360
|
ctx.checkHeapPressure(indexName);
|
|
3231
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3361
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
3232
3362
|
return { succeeded, failed };
|
|
3233
3363
|
}
|
|
3234
3364
|
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
@@ -3268,8 +3398,8 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
3268
3398
|
}
|
|
3269
3399
|
return { partitionDoc };
|
|
3270
3400
|
}
|
|
3271
|
-
function asUpdateError(err) {
|
|
3272
|
-
return
|
|
3401
|
+
function asUpdateError(ctx, err) {
|
|
3402
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
3273
3403
|
}
|
|
3274
3404
|
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
3275
3405
|
for (const update of updates) {
|
|
@@ -3466,24 +3596,39 @@ async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold,
|
|
|
3466
3596
|
const { document } = requireUpdateEntry(update);
|
|
3467
3597
|
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
3468
3598
|
} catch (err) {
|
|
3469
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
3599
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
3470
3600
|
}
|
|
3471
3601
|
}
|
|
3472
|
-
|
|
3473
|
-
|
|
3474
|
-
|
|
3475
|
-
|
|
3476
|
-
|
|
3477
|
-
|
|
3478
|
-
|
|
3602
|
+
let entry;
|
|
3603
|
+
try {
|
|
3604
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
3605
|
+
const writes = await prepareDocumentWrites(
|
|
3606
|
+
ctx,
|
|
3607
|
+
indexName,
|
|
3608
|
+
preparations.map((pending) => pending.preparation),
|
|
3609
|
+
hold,
|
|
3610
|
+
false
|
|
3611
|
+
);
|
|
3612
|
+
entry = writes.entry;
|
|
3613
|
+
} catch (err) {
|
|
3614
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
3615
|
+
failAsShutDown(failed, preparations, docIdOf);
|
|
3616
|
+
return [];
|
|
3617
|
+
}
|
|
3479
3618
|
const prepared = [];
|
|
3480
|
-
for (
|
|
3619
|
+
for (let position = 0; position < preparations.length; position++) {
|
|
3481
3620
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3621
|
+
if (shutDownStarted(ctx)) {
|
|
3622
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
3623
|
+
failAsShutDown(failed, preparations, docIdOf, position);
|
|
3624
|
+
return [];
|
|
3625
|
+
}
|
|
3626
|
+
const { docId, preparation } = preparations[position];
|
|
3482
3627
|
try {
|
|
3483
3628
|
if (preparation.failure !== void 0) throw preparation.failure;
|
|
3484
3629
|
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
3485
3630
|
} catch (err) {
|
|
3486
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
3631
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
3487
3632
|
}
|
|
3488
3633
|
}
|
|
3489
3634
|
return prepared;
|
|
@@ -3496,17 +3641,26 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
3496
3641
|
const failed = [];
|
|
3497
3642
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
3498
3643
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
3644
|
+
if (shutDownStarted(ctx)) {
|
|
3645
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkStart);
|
|
3646
|
+
break;
|
|
3647
|
+
}
|
|
3499
3648
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
3500
3649
|
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
3650
|
+
if (shutDownStarted(ctx)) {
|
|
3651
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
3652
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkEnd);
|
|
3653
|
+
break;
|
|
3654
|
+
}
|
|
3501
3655
|
try {
|
|
3502
3656
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
3503
3657
|
} catch (err) {
|
|
3504
|
-
const error = asUpdateError(err);
|
|
3658
|
+
const error = asUpdateError(ctx, err);
|
|
3505
3659
|
for (const update of prepared) failed.push({ docId: update.docId, error });
|
|
3506
3660
|
for (let i = chunkEnd; i < updates.length; i++) failed.push({ docId: reportedDocIdOf(updates[i]), error });
|
|
3507
3661
|
break;
|
|
3508
3662
|
}
|
|
3509
|
-
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
3663
|
+
const applies = prepared.map((update) => applyUnlessShutDown(ctx, applyOfUpdate(ctx, indexName, update)));
|
|
3510
3664
|
let failures;
|
|
3511
3665
|
try {
|
|
3512
3666
|
failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
@@ -3537,7 +3691,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
3537
3691
|
const update = prepared[i];
|
|
3538
3692
|
const failure = failures[i];
|
|
3539
3693
|
if (failure !== null) {
|
|
3540
|
-
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
3694
|
+
failed.push({ docId: update.docId, error: asUpdateError(ctx, failure.error) });
|
|
3541
3695
|
continue;
|
|
3542
3696
|
}
|
|
3543
3697
|
if (!update.buffered) settled.push(update);
|
|
@@ -3546,18 +3700,19 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
3546
3700
|
}
|
|
3547
3701
|
succeeded.push(update.docId);
|
|
3548
3702
|
}
|
|
3549
|
-
await settleUpdates(ctx, indexName, settled, hold);
|
|
3703
|
+
await unlessShutDown(ctx, () => settleUpdates(ctx, indexName, settled, hold));
|
|
3550
3704
|
if (chunkEnd < updates.length) {
|
|
3551
|
-
ctx.checkHeapPressure(indexName);
|
|
3705
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
3552
3706
|
await new Promise((r) => setTimeout(r, 0));
|
|
3553
3707
|
}
|
|
3554
3708
|
}
|
|
3709
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
3555
3710
|
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
3556
3711
|
for (const fieldPath of touchedVectorFields) {
|
|
3557
3712
|
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
3558
3713
|
}
|
|
3559
3714
|
ctx.checkHeapPressure(indexName);
|
|
3560
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3715
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
3561
3716
|
return { succeeded, failed };
|
|
3562
3717
|
}
|
|
3563
3718
|
|