@delali/narsil 0.4.1 → 0.4.2-next.273
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-56UNHLAF.mjs} +1 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-EKISQBV2.mjs} +283 -244
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-VAPCFSMX.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-VIVG3OXM.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-76Q5SZ5S.mjs} +23 -10
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-3HKIVZFU.mjs} +295 -106
- package/dist/browser/{segment-codec-leaf-A427QJUX.mjs → segment-codec-leaf-GXSRX5VT.mjs} +47 -30
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-35RWLUVY.mjs} +63 -144
- package/dist/{chunk-6GHX35ZM.mjs → chunk-43T3P3BE.mjs} +1 -1
- package/dist/{chunk-IL5DRCNU.mjs → chunk-4JLHHEBN.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-4YIVY3PB.mjs} +2 -2
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-5SHWA6PE.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-6LVQ3GLT.mjs} +3 -3
- package/dist/{chunk-DDUXEJXW.mjs → chunk-6SIEVWMG.mjs} +45 -22
- package/dist/{chunk-WM5KJSQV.mjs → chunk-ACNHJOGC.mjs} +1 -1
- package/dist/{chunk-755NM4PL.mjs → chunk-BWPVDIRN.mjs} +183 -21
- package/dist/{chunk-SMQPZHPS.mjs → chunk-C57GWAXN.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-DRZP5MSG.mjs} +5 -5
- package/dist/{chunk-MYN4K5IS.mjs → chunk-DZMD4OMD.mjs} +1 -1
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-EY5U4P5V.mjs} +1 -1
- package/dist/{chunk-X3EQCQLP.mjs → chunk-EZJ6HBG7.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-FE2XILWN.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-G3KVEHIW.mjs} +2 -2
- package/dist/{chunk-B3MO37YF.mjs → chunk-GTGQLCEI.mjs} +1 -1
- package/dist/{chunk-MSJD74GN.mjs → chunk-H23FB267.mjs} +5 -5
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-WTMCGZ74.mjs → chunk-KXA77DPZ.mjs} +2 -2
- package/dist/{chunk-6I6OO6B2.mjs → chunk-LRIJ7U3T.mjs} +4 -4
- package/dist/{chunk-3IE3A74H.mjs → chunk-PNFP5THV.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-PQWK7KXS.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-OGYSPNL2.mjs → chunk-QNZJYMCF.mjs} +7 -7
- package/dist/{chunk-DBC4EJWB.mjs → chunk-R3PFPFME.mjs} +1 -1
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-MI6UZGUR.mjs → chunk-SPA7GXST.mjs} +3 -3
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-WYH3QHBK.mjs} +698 -451
- package/dist/{chunk-D456ZFPQ.mjs → chunk-X5V25BPN.mjs} +2 -2
- package/dist/{chunk-GXCDWLQI.mjs → chunk-Y7GXGJFZ.mjs} +23 -10
- package/dist/{chunk-NCIGNQJE.mjs → chunk-YGRJLI2Z.mjs} +930 -387
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-ZO7VLAIT.mjs} +2 -2
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-ZTB7AWTW.mjs} +4 -4
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-7G57AYXH.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-KPN7NSOC.mjs} +14 -14
- package/dist/msgpack-leaf-LIT7TKZW.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-CSULLISE.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-BKQADXOT.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-HTSJKZRU.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AOLLV37A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-DN4FT62B.mjs} +23 -10
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-EF4Y5C3Y.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-ESVBC3AT.mjs} +1 -1
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-H6LK5WCF.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-JEYZQA4Z.mjs} +699 -451
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-KJQ3BUVM.mjs} +4 -4
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-QYVRD6C5.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-X3VY2YT4.mjs} +2 -2
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-ZF3VSGEI.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-FOUPTT7J.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-MGFFGKYQ.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-MVJTP3E4.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PSWHL55A.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-WOJWSDLW.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-ZUN6Y44X.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-K5XG46FV.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-KDPM3P27.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-FVFMBBH6.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-3QEJNMFH.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-Q76Q4S35.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-BSDDR7DQ.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-6TT66HC5.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-R5JQ2X2M.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-NCGETZYY.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-TX3GTD6S.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-VLVPAUWC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-NZSBQZKG.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -12
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -1649,6 +1649,7 @@ var getVectorFieldPaths;
|
|
|
1649
1649
|
var indexChangeOutsideExternalWaits;
|
|
1650
1650
|
var insertDocument;
|
|
1651
1651
|
var insertDocumentVectors;
|
|
1652
|
+
var isCompositePartition;
|
|
1652
1653
|
var isSyncEpoch;
|
|
1653
1654
|
var loadEmbedCode;
|
|
1654
1655
|
var loadSchemaCodeFor;
|
|
@@ -1665,6 +1666,7 @@ var requireDocumentObject;
|
|
|
1665
1666
|
var requireDocumentWithinSizeLimit;
|
|
1666
1667
|
var requireMatchingDocId;
|
|
1667
1668
|
var requireSurfaceFormsForPartialFields;
|
|
1669
|
+
var setNestedValue;
|
|
1668
1670
|
var storedBM25Params;
|
|
1669
1671
|
var turnIsSpent;
|
|
1670
1672
|
var turnsAreBudgeted;
|
|
@@ -1721,6 +1723,7 @@ function bindOptionalWritesHost(host, segmentCodec) {
|
|
|
1721
1723
|
indexChangeOutsideExternalWaits,
|
|
1722
1724
|
insertDocument,
|
|
1723
1725
|
insertDocumentVectors,
|
|
1726
|
+
isCompositePartition,
|
|
1724
1727
|
isSyncEpoch,
|
|
1725
1728
|
loadEmbedCode,
|
|
1726
1729
|
loadSchemaCodeFor,
|
|
@@ -1737,6 +1740,7 @@ function bindOptionalWritesHost(host, segmentCodec) {
|
|
|
1737
1740
|
requireDocumentWithinSizeLimit,
|
|
1738
1741
|
requireMatchingDocId,
|
|
1739
1742
|
requireSurfaceFormsForPartialFields,
|
|
1743
|
+
setNestedValue,
|
|
1740
1744
|
storedBM25Params,
|
|
1741
1745
|
turnIsSpent,
|
|
1742
1746
|
turnsAreBudgeted,
|
|
@@ -1866,7 +1870,7 @@ function epochStatesOfMetadata(metadata) {
|
|
|
1866
1870
|
|
|
1867
1871
|
// src/engine/index-from-metadata.ts
|
|
1868
1872
|
function isRecoveryConfigError(err) {
|
|
1869
|
-
return err instanceof NarsilError && (err.code === ErrorCodes.CONFIG_INVALID || err.code === ErrorCodes.SCHEMA_INVALID_TYPE || err.code === ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION);
|
|
1873
|
+
return err instanceof NarsilError && (err.code === ErrorCodes.CONFIG_INVALID || err.code === ErrorCodes.LANGUAGE_NOT_SUPPORTED || err.code === ErrorCodes.SCHEMA_INVALID_TYPE || err.code === ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION);
|
|
1870
1874
|
}
|
|
1871
1875
|
function resolveEmbeddingAdapter(deps, metadata, schema) {
|
|
1872
1876
|
const name = metadata.embedding?.adapter ?? null;
|
|
@@ -1936,8 +1940,9 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1936
1940
|
return;
|
|
1937
1941
|
}
|
|
1938
1942
|
const indexConfig = reconstructSchemaFromMetadata(metadata);
|
|
1939
|
-
|
|
1943
|
+
let language;
|
|
1940
1944
|
try {
|
|
1945
|
+
language = getLanguage(indexConfig.language ?? "english");
|
|
1941
1946
|
indexConfig.schema = validateSchema(indexConfig.schema);
|
|
1942
1947
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
1943
1948
|
requireSurfaceFormsForPartialFields(indexConfig.schema, indexConfig.surfaceForms);
|
|
@@ -2002,6 +2007,51 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
2002
2007
|
}
|
|
2003
2008
|
}
|
|
2004
2009
|
|
|
2010
|
+
// src/engine/mutations/batch-shutdown.ts
|
|
2011
|
+
function isFailureThatShutdownCauses(code) {
|
|
2012
|
+
return code === ErrorCodes.INSTANCE_SHUT_DOWN || code === ErrorCodes.INDEX_NOT_FOUND || code === ErrorCodes.EMBEDDING_FAILED;
|
|
2013
|
+
}
|
|
2014
|
+
function shutDownStarted(ctx) {
|
|
2015
|
+
return ctx.abortController.signal.aborted;
|
|
2016
|
+
}
|
|
2017
|
+
function shutDownError() {
|
|
2018
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
2019
|
+
}
|
|
2020
|
+
function causedByShutdown(ctx, err) {
|
|
2021
|
+
const { signal } = ctx.abortController;
|
|
2022
|
+
if (!signal.aborted) return false;
|
|
2023
|
+
if (err === signal.reason) return true;
|
|
2024
|
+
if (err instanceof NarsilError) return isFailureThatShutdownCauses(err.code);
|
|
2025
|
+
return err instanceof Error && err.name === "AbortError";
|
|
2026
|
+
}
|
|
2027
|
+
function batchFailureOf(ctx, err, fallbackCode) {
|
|
2028
|
+
if (causedByShutdown(ctx, err)) {
|
|
2029
|
+
return err instanceof NarsilError && err.code === ErrorCodes.INSTANCE_SHUT_DOWN ? err : shutDownError();
|
|
2030
|
+
}
|
|
2031
|
+
return err instanceof NarsilError ? err : new NarsilError(fallbackCode, String(err));
|
|
2032
|
+
}
|
|
2033
|
+
function docIdOf(entry) {
|
|
2034
|
+
return entry.docId;
|
|
2035
|
+
}
|
|
2036
|
+
function failAsShutDown(failed, entries, reportedDocIdOf2, start = 0, end = entries.length) {
|
|
2037
|
+
if (start >= end) return;
|
|
2038
|
+
const error = shutDownError();
|
|
2039
|
+
for (let i = start; i < end; i++) failed.push({ docId: reportedDocIdOf2(entries[i]), error });
|
|
2040
|
+
}
|
|
2041
|
+
function applyUnlessShutDown(ctx, apply) {
|
|
2042
|
+
return () => {
|
|
2043
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
2044
|
+
return apply();
|
|
2045
|
+
};
|
|
2046
|
+
}
|
|
2047
|
+
async function unlessShutDown(ctx, work) {
|
|
2048
|
+
try {
|
|
2049
|
+
await work();
|
|
2050
|
+
} catch (err) {
|
|
2051
|
+
if (!shutDownStarted(ctx)) throw err;
|
|
2052
|
+
}
|
|
2053
|
+
}
|
|
2054
|
+
|
|
2005
2055
|
// src/engine/mutations/batch-admissions.ts
|
|
2006
2056
|
function createPendingAdmissions() {
|
|
2007
2057
|
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
@@ -2012,8 +2062,11 @@ function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
|
2012
2062
|
const partitionId = manager.routePartition(docId);
|
|
2013
2063
|
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
2014
2064
|
}
|
|
2015
|
-
function asBatchInsertError(err) {
|
|
2016
|
-
return
|
|
2065
|
+
function asBatchInsertError(ctx, err) {
|
|
2066
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_VALIDATION_FAILED);
|
|
2067
|
+
}
|
|
2068
|
+
function providedDocIdOrEmpty(document) {
|
|
2069
|
+
return providedDocId(document) ?? "";
|
|
2017
2070
|
}
|
|
2018
2071
|
|
|
2019
2072
|
// src/engine/mutations/outside-hold.ts
|
|
@@ -2243,7 +2296,11 @@ async function prepareInsertDocuments(ctx, indexName, documents, start, end, hol
|
|
|
2243
2296
|
const pending = [];
|
|
2244
2297
|
for (let i = start; i < end; i++) {
|
|
2245
2298
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2246
|
-
if (ctx
|
|
2299
|
+
if (shutDownStarted(ctx)) {
|
|
2300
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
2301
|
+
failAsShutDown(failed, documents, providedDocIdOrEmpty, i, end);
|
|
2302
|
+
return null;
|
|
2303
|
+
}
|
|
2247
2304
|
const document = documents[i];
|
|
2248
2305
|
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
2249
2306
|
try {
|
|
@@ -2252,17 +2309,26 @@ async function prepareInsertDocuments(ctx, indexName, documents, start, end, hol
|
|
|
2252
2309
|
validateDocId(docId);
|
|
2253
2310
|
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
2254
2311
|
} catch (err) {
|
|
2255
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2312
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
2256
2313
|
}
|
|
2257
2314
|
}
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2315
|
+
let prepared;
|
|
2316
|
+
try {
|
|
2317
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
2318
|
+
prepared = await prepareDocumentWrites(
|
|
2319
|
+
ctx,
|
|
2320
|
+
indexName,
|
|
2321
|
+
pending.map((item) => item.preparation),
|
|
2322
|
+
hold,
|
|
2323
|
+
true,
|
|
2324
|
+
earlier
|
|
2325
|
+
);
|
|
2326
|
+
} catch (err) {
|
|
2327
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
2328
|
+
failAsShutDown(failed, pending, docIdOf);
|
|
2329
|
+
return null;
|
|
2330
|
+
}
|
|
2331
|
+
const { entry, embedding } = prepared;
|
|
2266
2332
|
const candidates = [];
|
|
2267
2333
|
for (const item of pending) {
|
|
2268
2334
|
const { document, failure } = item.preparation;
|
|
@@ -2270,22 +2336,19 @@ async function prepareInsertDocuments(ctx, indexName, documents, start, end, hol
|
|
|
2270
2336
|
candidates.push({ docId: item.docId, document });
|
|
2271
2337
|
continue;
|
|
2272
2338
|
}
|
|
2273
|
-
const docId = item.refusedByHook ? item.docId :
|
|
2274
|
-
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
2339
|
+
const docId = item.refusedByHook ? item.docId : providedDocIdOrEmpty(document);
|
|
2340
|
+
failed.push({ docId, error: asBatchInsertError(ctx, failure) });
|
|
2275
2341
|
}
|
|
2276
2342
|
return { entry, candidates, embedding };
|
|
2277
2343
|
}
|
|
2278
2344
|
async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
|
|
2279
|
-
const
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
true,
|
|
2287
|
-
failed
|
|
2288
|
-
);
|
|
2345
|
+
const prepared = await prepareInsertDocuments(ctx, indexName, documents, 0, documents.length, hold, true, failed);
|
|
2346
|
+
if (prepared === null) return { admitted: [], embedding: null };
|
|
2347
|
+
const { entry, candidates, embedding } = prepared;
|
|
2348
|
+
if (shutDownStarted(ctx)) {
|
|
2349
|
+
failAsShutDown(failed, candidates, docIdOf);
|
|
2350
|
+
return { admitted: [], embedding };
|
|
2351
|
+
}
|
|
2289
2352
|
const manager = ctx.requireManager(indexName);
|
|
2290
2353
|
const vecIndexes = manager.getVectorIndexes();
|
|
2291
2354
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
@@ -2295,7 +2358,11 @@ async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
|
|
|
2295
2358
|
const pending = createPendingAdmissions();
|
|
2296
2359
|
for (let position = 0; position < candidates.length; position++) {
|
|
2297
2360
|
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
2298
|
-
if (ctx
|
|
2361
|
+
if (shutDownStarted(ctx)) {
|
|
2362
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
2363
|
+
failAsShutDown(failed, candidates, docIdOf, position);
|
|
2364
|
+
return { admitted: [], embedding };
|
|
2365
|
+
}
|
|
2299
2366
|
const { docId, document } = candidates[position];
|
|
2300
2367
|
try {
|
|
2301
2368
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
@@ -2317,7 +2384,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
|
|
|
2317
2384
|
recordPendingAdmission(ctx, pending, manager, docId);
|
|
2318
2385
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2319
2386
|
} catch (err) {
|
|
2320
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2387
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
2321
2388
|
}
|
|
2322
2389
|
}
|
|
2323
2390
|
return { admitted, embedding };
|
|
@@ -2382,7 +2449,7 @@ async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
2382
2449
|
await manager.publication.write(
|
|
2383
2450
|
() => changeIndexOutsideExternalWaits(async () => {
|
|
2384
2451
|
if (manager.has(docId)) return;
|
|
2385
|
-
const previous = prepareDocumentVectors(restoreDoc, ctx.requireIndex(indexName).vectorFieldPaths);
|
|
2452
|
+
const previous = prepareDocumentVectors(restoreDoc(), ctx.requireIndex(indexName).vectorFieldPaths);
|
|
2386
2453
|
manager.insert(docId, previous.partitionDoc, {
|
|
2387
2454
|
preparedText: { after: preparedText?.before }
|
|
2388
2455
|
});
|
|
@@ -2467,9 +2534,9 @@ async function recordChunk(ctx, indexName, mutations, applies) {
|
|
|
2467
2534
|
function applyInsertDocument(ctx, indexName, doc, options, progress) {
|
|
2468
2535
|
const manager = ctx.requireManager(indexName);
|
|
2469
2536
|
const insertIntoIndex = () => {
|
|
2470
|
-
if (ctx
|
|
2537
|
+
if (shutDownStarted(ctx)) {
|
|
2471
2538
|
progress.skipped = true;
|
|
2472
|
-
throw
|
|
2539
|
+
throw shutDownError();
|
|
2473
2540
|
}
|
|
2474
2541
|
admitInsert(ctx, indexName, manager, doc.docId);
|
|
2475
2542
|
const queued = ctx.bufferIfRebalancing(indexName, {
|
|
@@ -2525,11 +2592,11 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
2525
2592
|
const progress = docs.map(() => ({ inserted: false, buffered: false, skipped: false }));
|
|
2526
2593
|
const applies = docs.map((doc, i) => applyInsertDocument(ctx, indexName, doc, options, progress[i]));
|
|
2527
2594
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2528
|
-
await ctx.orchestrator.settlePublications(indexName);
|
|
2595
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName));
|
|
2529
2596
|
const applications = [];
|
|
2530
2597
|
for (let i = 0; i < docs.length; i++) {
|
|
2531
2598
|
if (progress[i].skipped) {
|
|
2532
|
-
applications.push({ status: "
|
|
2599
|
+
applications.push({ status: "failed", error: shutDownError() });
|
|
2533
2600
|
continue;
|
|
2534
2601
|
}
|
|
2535
2602
|
const failure = failures[i];
|
|
@@ -2562,27 +2629,33 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
2562
2629
|
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
2563
2630
|
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
2564
2631
|
const manager = ctx.executor.getManager(indexName);
|
|
2565
|
-
await
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2632
|
+
await unlessShutDown(
|
|
2633
|
+
ctx,
|
|
2634
|
+
() => runOutside(hold, async () => {
|
|
2635
|
+
for (const doc of inserted) {
|
|
2636
|
+
try {
|
|
2637
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2638
|
+
} catch (err) {
|
|
2639
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2640
|
+
}
|
|
2571
2641
|
}
|
|
2572
|
-
}
|
|
2573
|
-
|
|
2574
|
-
return ctx.executor.getManager(indexName) !== manager;
|
|
2642
|
+
})
|
|
2643
|
+
);
|
|
2644
|
+
return shutDownStarted(ctx) || ctx.executor.getManager(indexName) !== manager;
|
|
2575
2645
|
}
|
|
2576
2646
|
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
2577
2647
|
const succeeded = [];
|
|
2578
2648
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2579
2649
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2580
|
-
if (ctx
|
|
2650
|
+
if (shutDownStarted(ctx)) {
|
|
2651
|
+
failAsShutDown(failed, admitted, docIdOf, chunkStart);
|
|
2652
|
+
break;
|
|
2653
|
+
}
|
|
2581
2654
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
2582
2655
|
try {
|
|
2583
2656
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2584
2657
|
} catch (err) {
|
|
2585
|
-
const error = asBatchInsertError(err);
|
|
2658
|
+
const error = asBatchInsertError(ctx, err);
|
|
2586
2659
|
for (let i = chunkStart; i < admitted.length; i++) failed.push({ docId: admitted[i].docId, error });
|
|
2587
2660
|
break;
|
|
2588
2661
|
}
|
|
@@ -2591,9 +2664,8 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
2591
2664
|
for (let i = 0; i < chunk.length; i++) {
|
|
2592
2665
|
const doc = chunk[i];
|
|
2593
2666
|
const application = applications[i];
|
|
2594
|
-
if (application.status === "skipped") continue;
|
|
2595
2667
|
if (application.status === "failed") {
|
|
2596
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2668
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
2597
2669
|
continue;
|
|
2598
2670
|
}
|
|
2599
2671
|
succeeded.push(doc.docId);
|
|
@@ -2608,7 +2680,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
2608
2680
|
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
2609
2681
|
}
|
|
2610
2682
|
if (applied < admitted.length) {
|
|
2611
|
-
ctx.checkHeapPressure(indexName);
|
|
2683
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
2612
2684
|
await new Promise((r) => setTimeout(r, 0));
|
|
2613
2685
|
}
|
|
2614
2686
|
}
|
|
@@ -2617,7 +2689,7 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, f
|
|
|
2617
2689
|
|
|
2618
2690
|
// src/engine/mutations/insert-batch-chunks.ts
|
|
2619
2691
|
async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
2620
|
-
const
|
|
2692
|
+
const prepared = await prepareInsertDocuments(
|
|
2621
2693
|
ctx,
|
|
2622
2694
|
indexName,
|
|
2623
2695
|
documents,
|
|
@@ -2628,13 +2700,24 @@ async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold,
|
|
|
2628
2700
|
failed,
|
|
2629
2701
|
earlier
|
|
2630
2702
|
);
|
|
2703
|
+
if (prepared === null) return [];
|
|
2704
|
+
const { entry, candidates } = prepared;
|
|
2705
|
+
const admitted = [];
|
|
2706
|
+
if (shutDownStarted(ctx)) {
|
|
2707
|
+
failAsShutDown(failed, candidates, docIdOf);
|
|
2708
|
+
return admitted;
|
|
2709
|
+
}
|
|
2631
2710
|
const manager = ctx.requireManager(indexName);
|
|
2632
2711
|
const vecIndexes = manager.getVectorIndexes();
|
|
2633
2712
|
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2634
|
-
|
|
2635
|
-
for (const { docId, document } of candidates) {
|
|
2713
|
+
for (let position = 0; position < candidates.length; position++) {
|
|
2636
2714
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2637
|
-
if (ctx
|
|
2715
|
+
if (shutDownStarted(ctx)) {
|
|
2716
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
2717
|
+
failAsShutDown(failed, candidates, docIdOf, position);
|
|
2718
|
+
return [];
|
|
2719
|
+
}
|
|
2720
|
+
const { docId, document } = candidates[position];
|
|
2638
2721
|
try {
|
|
2639
2722
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2640
2723
|
document,
|
|
@@ -2646,7 +2729,7 @@ async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold,
|
|
|
2646
2729
|
admitInsert(ctx, indexName, manager, docId);
|
|
2647
2730
|
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2648
2731
|
} catch (err) {
|
|
2649
|
-
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2732
|
+
failed.push({ docId, error: asBatchInsertError(ctx, err) });
|
|
2650
2733
|
}
|
|
2651
2734
|
}
|
|
2652
2735
|
return admitted;
|
|
@@ -2655,17 +2738,25 @@ async function insertInChunks(ctx, indexName, documents, options, hold, hooked,
|
|
|
2655
2738
|
const succeeded = [];
|
|
2656
2739
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2657
2740
|
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2658
|
-
if (ctx
|
|
2741
|
+
if (shutDownStarted(ctx)) {
|
|
2742
|
+
failAsShutDown(failed, documents, providedDocIdOrEmpty, chunkStart);
|
|
2743
|
+
break;
|
|
2744
|
+
}
|
|
2659
2745
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2660
2746
|
const earlier = embeddingOfDocuments(embedded, documents.slice(chunkStart, chunkEnd));
|
|
2661
2747
|
const prepared = await admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
2748
|
+
if (shutDownStarted(ctx)) {
|
|
2749
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
2750
|
+
failAsShutDown(failed, documents, providedDocIdOrEmpty, chunkEnd);
|
|
2751
|
+
break;
|
|
2752
|
+
}
|
|
2662
2753
|
try {
|
|
2663
2754
|
if (prepared.length > 0) ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2664
2755
|
} catch (err) {
|
|
2665
|
-
const error = asBatchInsertError(err);
|
|
2756
|
+
const error = asBatchInsertError(ctx, err);
|
|
2666
2757
|
for (const doc of prepared) failed.push({ docId: doc.docId, error });
|
|
2667
2758
|
for (let i = chunkEnd; i < documents.length; i++) {
|
|
2668
|
-
failed.push({ docId:
|
|
2759
|
+
failed.push({ docId: providedDocIdOrEmpty(documents[i]), error });
|
|
2669
2760
|
}
|
|
2670
2761
|
break;
|
|
2671
2762
|
}
|
|
@@ -2675,9 +2766,8 @@ async function insertInChunks(ctx, indexName, documents, options, hold, hooked,
|
|
|
2675
2766
|
for (let i = 0; i < prepared.length; i++) {
|
|
2676
2767
|
const doc = prepared[i];
|
|
2677
2768
|
const application = applications[i];
|
|
2678
|
-
if (application.status === "skipped") continue;
|
|
2679
2769
|
if (application.status === "failed") {
|
|
2680
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2770
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, application.error) });
|
|
2681
2771
|
continue;
|
|
2682
2772
|
}
|
|
2683
2773
|
succeeded.push(doc.docId);
|
|
@@ -2688,21 +2778,24 @@ async function insertInChunks(ctx, indexName, documents, options, hold, hooked,
|
|
|
2688
2778
|
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
2689
2779
|
}
|
|
2690
2780
|
if (chunkEnd < documents.length) {
|
|
2691
|
-
ctx.checkHeapPressure(indexName);
|
|
2781
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
2692
2782
|
await new Promise((r) => setTimeout(r, 0));
|
|
2693
2783
|
}
|
|
2694
2784
|
}
|
|
2695
2785
|
return { succeeded, touchedVectorFields };
|
|
2696
2786
|
}
|
|
2697
2787
|
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
2788
|
+
if (shutDownStarted(ctx)) return;
|
|
2698
2789
|
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
2699
2790
|
for (const fieldPath of inserts.touchedVectorFields) {
|
|
2700
2791
|
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
2701
2792
|
}
|
|
2702
2793
|
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
2703
2794
|
ctx.checkHeapPressure(indexName);
|
|
2704
|
-
await ctx
|
|
2705
|
-
|
|
2795
|
+
await unlessShutDown(ctx, async () => {
|
|
2796
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
2797
|
+
if (options?.wait === true && !shutDownStarted(ctx)) await awaitWriteVisibility(ctx, indexName);
|
|
2798
|
+
});
|
|
2706
2799
|
}
|
|
2707
2800
|
|
|
2708
2801
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -2711,7 +2804,7 @@ function applySegment(ctx, indexName, attachment, doc, options) {
|
|
|
2711
2804
|
const publish = async () => {
|
|
2712
2805
|
if (attachment.failure !== void 0) throw attachment.failure;
|
|
2713
2806
|
if (attachment.attached) return;
|
|
2714
|
-
if (ctx
|
|
2807
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
2715
2808
|
if (ctx.isRebalancing(indexName) || manager.partitionCount !== attachment.partitionCount) {
|
|
2716
2809
|
attachment.individual = true;
|
|
2717
2810
|
}
|
|
@@ -2865,6 +2958,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2865
2958
|
for (let index = 0; index < chunks.length; index++) {
|
|
2866
2959
|
const { docs, attachmentOf } = chunks[index];
|
|
2867
2960
|
try {
|
|
2961
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
2868
2962
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2869
2963
|
} catch (error) {
|
|
2870
2964
|
for (const later of chunks.slice(index)) {
|
|
@@ -2878,7 +2972,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2878
2972
|
return applySegment(ctx, indexName, attachment, doc, options);
|
|
2879
2973
|
});
|
|
2880
2974
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2881
|
-
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
2975
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.settlePublications(indexName, "segments"));
|
|
2882
2976
|
const inserted = [];
|
|
2883
2977
|
for (let i = 0; i < docs.length; i++) {
|
|
2884
2978
|
const doc = docs[i];
|
|
@@ -2919,7 +3013,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2919
3013
|
break;
|
|
2920
3014
|
}
|
|
2921
3015
|
if (index + 1 < chunks.length) {
|
|
2922
|
-
ctx.checkHeapPressure(indexName);
|
|
3016
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
2923
3017
|
await new Promise((r) => setTimeout(r, 0));
|
|
2924
3018
|
}
|
|
2925
3019
|
}
|
|
@@ -2930,7 +3024,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2930
3024
|
continue;
|
|
2931
3025
|
}
|
|
2932
3026
|
if (!errorOf.has(doc.docId)) continue;
|
|
2933
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
3027
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(ctx, errorOf.get(doc.docId)) });
|
|
2934
3028
|
}
|
|
2935
3029
|
return { succeeded, touchedVectorFields, unrecorded };
|
|
2936
3030
|
}
|
|
@@ -2944,8 +3038,11 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
2944
3038
|
);
|
|
2945
3039
|
}
|
|
2946
3040
|
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
2947
|
-
|
|
2948
|
-
|
|
3041
|
+
const nothingIngested = () => ({ succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] });
|
|
3042
|
+
if (admitted.length === 0) return nothingIngested();
|
|
3043
|
+
if (shutDownStarted(ctx)) {
|
|
3044
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
3045
|
+
return nothingIngested();
|
|
2949
3046
|
}
|
|
2950
3047
|
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
2951
3048
|
const manager = ctx.requireManager(indexName);
|
|
@@ -2963,6 +3060,10 @@ async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
|
2963
3060
|
} catch {
|
|
2964
3061
|
built = null;
|
|
2965
3062
|
}
|
|
3063
|
+
if (shutDownStarted(ctx)) {
|
|
3064
|
+
failAsShutDown(failed, admitted, docIdOf);
|
|
3065
|
+
return nothingIngested();
|
|
3066
|
+
}
|
|
2966
3067
|
if (built === null) {
|
|
2967
3068
|
return applyIndividually();
|
|
2968
3069
|
}
|
|
@@ -3013,7 +3114,9 @@ async function insertBatchViaSegments(ctx, indexName, documents, options, hold)
|
|
|
3013
3114
|
const failed = [];
|
|
3014
3115
|
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents, failed, hold);
|
|
3015
3116
|
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
3016
|
-
if (
|
|
3117
|
+
if (shutDownStarted(ctx)) {
|
|
3118
|
+
failAsShutDown(failed, outcome.unrecorded, docIdOf);
|
|
3119
|
+
} else if (outcome.unrecorded.length > 0) {
|
|
3017
3120
|
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
3018
3121
|
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
3019
3122
|
outcome.succeeded.push(...rest.succeeded);
|
|
@@ -3028,20 +3131,54 @@ async function insertDocumentBatch(ctx, indexName, documents, options, hold) {
|
|
|
3028
3131
|
ctx.guardShutdown();
|
|
3029
3132
|
ctx.requireIndex(indexName);
|
|
3030
3133
|
validateBatchList(documents, "documents");
|
|
3031
|
-
|
|
3134
|
+
const failed = [];
|
|
3135
|
+
await unlessShutDown(ctx, () => ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length));
|
|
3136
|
+
if (shutDownStarted(ctx)) {
|
|
3137
|
+
failAsShutDown(failed, documents, providedDocIdOrEmpty);
|
|
3138
|
+
return { succeeded: [], failed };
|
|
3139
|
+
}
|
|
3032
3140
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
3033
3141
|
if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
3034
3142
|
return insertBatchViaSegments(ctx, indexName, documents, options, hold);
|
|
3035
3143
|
}
|
|
3036
|
-
const failed = [];
|
|
3037
3144
|
const inserts = await insertInChunks(ctx, indexName, documents, options, hold, true, failed);
|
|
3038
3145
|
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
3039
3146
|
return { succeeded: inserts.succeeded, failed };
|
|
3040
3147
|
}
|
|
3041
3148
|
|
|
3149
|
+
// src/engine/mutations/rollback-copy.ts
|
|
3150
|
+
function frozenRollbackCopy(manager, docId) {
|
|
3151
|
+
const partitionId = manager.partitionIdOf(docId);
|
|
3152
|
+
if (partitionId === void 0) return void 0;
|
|
3153
|
+
const partition = manager.getPartition(partitionId);
|
|
3154
|
+
if (!isCompositePartition(partition)) return void 0;
|
|
3155
|
+
for (const segment of partition.frozenSegments()) {
|
|
3156
|
+
const ordinal = segment.docStore.getInternalId(docId);
|
|
3157
|
+
if (ordinal !== void 0) return segment.documentSource.copyForRollbackAt?.(ordinal);
|
|
3158
|
+
}
|
|
3159
|
+
return void 0;
|
|
3160
|
+
}
|
|
3161
|
+
function rollbackCopyOf(manager, docId) {
|
|
3162
|
+
const frozenCopy = frozenRollbackCopy(manager, docId);
|
|
3163
|
+
if (frozenCopy === void 0) {
|
|
3164
|
+
const document = manager.get(docId);
|
|
3165
|
+
return document === void 0 ? void 0 : () => document;
|
|
3166
|
+
}
|
|
3167
|
+
const vectors = [];
|
|
3168
|
+
for (const [fieldPath, vectorIndex] of manager.getVectorIndexes()) {
|
|
3169
|
+
const vector = vectorIndex.getVector(docId);
|
|
3170
|
+
if (vector) vectors.push([fieldPath, vector]);
|
|
3171
|
+
}
|
|
3172
|
+
return () => {
|
|
3173
|
+
const document = frozenCopy();
|
|
3174
|
+
for (const [fieldPath, vector] of vectors) setNestedValue(document, fieldPath, vector);
|
|
3175
|
+
return document;
|
|
3176
|
+
};
|
|
3177
|
+
}
|
|
3178
|
+
|
|
3042
3179
|
// src/engine/mutations/remove.ts
|
|
3043
|
-
function asRemoveError(err) {
|
|
3044
|
-
return
|
|
3180
|
+
function asRemoveError(ctx, err) {
|
|
3181
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
3045
3182
|
}
|
|
3046
3183
|
function validateRemoval(ctx, docId) {
|
|
3047
3184
|
ctx.guardShutdown();
|
|
@@ -3077,7 +3214,7 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
3077
3214
|
prepared.buffered = true;
|
|
3078
3215
|
return queued;
|
|
3079
3216
|
}
|
|
3080
|
-
prepared.restoreDoc = ctx.durability ? manager
|
|
3217
|
+
prepared.restoreDoc = ctx.durability ? rollbackCopyOf(manager, docId) : void 0;
|
|
3081
3218
|
if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
|
|
3082
3219
|
manager.remove(docId, void 0, {
|
|
3083
3220
|
onRemovedSpellings: (spellings) => {
|
|
@@ -3149,18 +3286,31 @@ async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, h
|
|
|
3149
3286
|
}
|
|
3150
3287
|
}
|
|
3151
3288
|
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
3152
|
-
await
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3289
|
+
await unlessShutDown(
|
|
3290
|
+
ctx,
|
|
3291
|
+
() => runOutside(hold, async () => {
|
|
3292
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3293
|
+
if (refused.has(i)) continue;
|
|
3294
|
+
try {
|
|
3295
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
3296
|
+
} catch (err) {
|
|
3297
|
+
refused.set(i, err);
|
|
3298
|
+
}
|
|
3159
3299
|
}
|
|
3160
|
-
}
|
|
3161
|
-
|
|
3300
|
+
})
|
|
3301
|
+
);
|
|
3162
3302
|
return refused;
|
|
3163
3303
|
}
|
|
3304
|
+
function failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused) {
|
|
3305
|
+
const error = shutDownError();
|
|
3306
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3307
|
+
failed.push({ docId: docIds[i], error: refused.has(i) ? asRemoveError(ctx, refused.get(i)) : error });
|
|
3308
|
+
}
|
|
3309
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
3310
|
+
}
|
|
3311
|
+
function removalDocIdOf(docId) {
|
|
3312
|
+
return docId;
|
|
3313
|
+
}
|
|
3164
3314
|
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
3165
3315
|
ctx.guardShutdown();
|
|
3166
3316
|
ctx.requireIndex(indexName);
|
|
@@ -3172,8 +3322,16 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
3172
3322
|
let manager = first;
|
|
3173
3323
|
try {
|
|
3174
3324
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
3325
|
+
if (shutDownStarted(ctx)) {
|
|
3326
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkStart);
|
|
3327
|
+
break;
|
|
3328
|
+
}
|
|
3175
3329
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
3176
3330
|
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
3331
|
+
if (shutDownStarted(ctx)) {
|
|
3332
|
+
failChunkAsShutDown(ctx, failed, docIds, chunkStart, chunkEnd, refused);
|
|
3333
|
+
break;
|
|
3334
|
+
}
|
|
3177
3335
|
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3178
3336
|
const removed = [];
|
|
3179
3337
|
const prepared = [];
|
|
@@ -3183,10 +3341,15 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
3183
3341
|
if (refused.has(i)) throw refused.get(i);
|
|
3184
3342
|
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
3185
3343
|
} catch (err) {
|
|
3186
|
-
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
3344
|
+
failed.push({ docId: docIds[i], error: asRemoveError(ctx, err) });
|
|
3187
3345
|
}
|
|
3188
3346
|
}
|
|
3189
|
-
|
|
3347
|
+
if (shutDownStarted(ctx)) {
|
|
3348
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
3349
|
+
failAsShutDown(failed, docIds, removalDocIdOf, chunkEnd);
|
|
3350
|
+
break;
|
|
3351
|
+
}
|
|
3352
|
+
const applies = prepared.map((removal) => applyUnlessShutDown(ctx, applyOfRemoval(ctx, indexName, removal)));
|
|
3190
3353
|
try {
|
|
3191
3354
|
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
3192
3355
|
for (let i = 0; i < prepared.length; i++) {
|
|
@@ -3207,7 +3370,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
3207
3370
|
error = rollbackError;
|
|
3208
3371
|
}
|
|
3209
3372
|
removal.releaseVectorHolds?.();
|
|
3210
|
-
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
3373
|
+
failed.push({ docId: removal.docId, error: asRemoveError(ctx, error) });
|
|
3211
3374
|
continue;
|
|
3212
3375
|
}
|
|
3213
3376
|
removal.releaseVectorHolds?.();
|
|
@@ -3217,18 +3380,19 @@ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
|
3217
3380
|
} finally {
|
|
3218
3381
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
3219
3382
|
}
|
|
3220
|
-
await settleRemovals(ctx, indexName, removed, hold);
|
|
3221
|
-
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3383
|
+
await unlessShutDown(ctx, () => settleRemovals(ctx, indexName, removed, hold));
|
|
3384
|
+
if (!shutDownStarted(ctx)) manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3222
3385
|
if (chunkEnd < docIds.length) {
|
|
3223
3386
|
await new Promise((r) => setTimeout(r, 0));
|
|
3224
3387
|
}
|
|
3225
3388
|
}
|
|
3226
3389
|
} finally {
|
|
3227
3390
|
const ending = manager;
|
|
3228
|
-
await ending?.publication.write(() => ending.endBatchRemove());
|
|
3391
|
+
await unlessShutDown(ctx, () => ending?.publication.write(() => ending.endBatchRemove()));
|
|
3229
3392
|
}
|
|
3393
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
3230
3394
|
ctx.checkHeapPressure(indexName);
|
|
3231
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3395
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
3232
3396
|
return { succeeded, failed };
|
|
3233
3397
|
}
|
|
3234
3398
|
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
@@ -3268,8 +3432,8 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
3268
3432
|
}
|
|
3269
3433
|
return { partitionDoc };
|
|
3270
3434
|
}
|
|
3271
|
-
function asUpdateError(err) {
|
|
3272
|
-
return
|
|
3435
|
+
function asUpdateError(ctx, err) {
|
|
3436
|
+
return batchFailureOf(ctx, err, ErrorCodes.DOC_NOT_FOUND);
|
|
3273
3437
|
}
|
|
3274
3438
|
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
3275
3439
|
for (const update of updates) {
|
|
@@ -3466,24 +3630,39 @@ async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold,
|
|
|
3466
3630
|
const { document } = requireUpdateEntry(update);
|
|
3467
3631
|
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
3468
3632
|
} catch (err) {
|
|
3469
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
3633
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
3470
3634
|
}
|
|
3471
3635
|
}
|
|
3472
|
-
|
|
3473
|
-
|
|
3474
|
-
|
|
3475
|
-
|
|
3476
|
-
|
|
3477
|
-
|
|
3478
|
-
|
|
3636
|
+
let entry;
|
|
3637
|
+
try {
|
|
3638
|
+
if (shutDownStarted(ctx)) throw shutDownError();
|
|
3639
|
+
const writes = await prepareDocumentWrites(
|
|
3640
|
+
ctx,
|
|
3641
|
+
indexName,
|
|
3642
|
+
preparations.map((pending) => pending.preparation),
|
|
3643
|
+
hold,
|
|
3644
|
+
false
|
|
3645
|
+
);
|
|
3646
|
+
entry = writes.entry;
|
|
3647
|
+
} catch (err) {
|
|
3648
|
+
if (!causedByShutdown(ctx, err)) throw err;
|
|
3649
|
+
failAsShutDown(failed, preparations, docIdOf);
|
|
3650
|
+
return [];
|
|
3651
|
+
}
|
|
3479
3652
|
const prepared = [];
|
|
3480
|
-
for (
|
|
3653
|
+
for (let position = 0; position < preparations.length; position++) {
|
|
3481
3654
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3655
|
+
if (shutDownStarted(ctx)) {
|
|
3656
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
3657
|
+
failAsShutDown(failed, preparations, docIdOf, position);
|
|
3658
|
+
return [];
|
|
3659
|
+
}
|
|
3660
|
+
const { docId, preparation } = preparations[position];
|
|
3482
3661
|
try {
|
|
3483
3662
|
if (preparation.failure !== void 0) throw preparation.failure;
|
|
3484
3663
|
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
3485
3664
|
} catch (err) {
|
|
3486
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
3665
|
+
failed.push({ docId, error: asUpdateError(ctx, err) });
|
|
3487
3666
|
}
|
|
3488
3667
|
}
|
|
3489
3668
|
return prepared;
|
|
@@ -3496,17 +3675,26 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
3496
3675
|
const failed = [];
|
|
3497
3676
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
3498
3677
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
3678
|
+
if (shutDownStarted(ctx)) {
|
|
3679
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkStart);
|
|
3680
|
+
break;
|
|
3681
|
+
}
|
|
3499
3682
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
3500
3683
|
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
3684
|
+
if (shutDownStarted(ctx)) {
|
|
3685
|
+
failAsShutDown(failed, prepared, docIdOf);
|
|
3686
|
+
failAsShutDown(failed, updates, reportedDocIdOf, chunkEnd);
|
|
3687
|
+
break;
|
|
3688
|
+
}
|
|
3501
3689
|
try {
|
|
3502
3690
|
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
3503
3691
|
} catch (err) {
|
|
3504
|
-
const error = asUpdateError(err);
|
|
3692
|
+
const error = asUpdateError(ctx, err);
|
|
3505
3693
|
for (const update of prepared) failed.push({ docId: update.docId, error });
|
|
3506
3694
|
for (let i = chunkEnd; i < updates.length; i++) failed.push({ docId: reportedDocIdOf(updates[i]), error });
|
|
3507
3695
|
break;
|
|
3508
3696
|
}
|
|
3509
|
-
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
3697
|
+
const applies = prepared.map((update) => applyUnlessShutDown(ctx, applyOfUpdate(ctx, indexName, update)));
|
|
3510
3698
|
let failures;
|
|
3511
3699
|
try {
|
|
3512
3700
|
failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
@@ -3537,7 +3725,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
3537
3725
|
const update = prepared[i];
|
|
3538
3726
|
const failure = failures[i];
|
|
3539
3727
|
if (failure !== null) {
|
|
3540
|
-
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
3728
|
+
failed.push({ docId: update.docId, error: asUpdateError(ctx, failure.error) });
|
|
3541
3729
|
continue;
|
|
3542
3730
|
}
|
|
3543
3731
|
if (!update.buffered) settled.push(update);
|
|
@@ -3546,18 +3734,19 @@ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
|
3546
3734
|
}
|
|
3547
3735
|
succeeded.push(update.docId);
|
|
3548
3736
|
}
|
|
3549
|
-
await settleUpdates(ctx, indexName, settled, hold);
|
|
3737
|
+
await unlessShutDown(ctx, () => settleUpdates(ctx, indexName, settled, hold));
|
|
3550
3738
|
if (chunkEnd < updates.length) {
|
|
3551
|
-
ctx.checkHeapPressure(indexName);
|
|
3739
|
+
if (!shutDownStarted(ctx)) ctx.checkHeapPressure(indexName);
|
|
3552
3740
|
await new Promise((r) => setTimeout(r, 0));
|
|
3553
3741
|
}
|
|
3554
3742
|
}
|
|
3743
|
+
if (shutDownStarted(ctx)) return { succeeded, failed };
|
|
3555
3744
|
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
3556
3745
|
for (const fieldPath of touchedVectorFields) {
|
|
3557
3746
|
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
3558
3747
|
}
|
|
3559
3748
|
ctx.checkHeapPressure(indexName);
|
|
3560
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3749
|
+
if (options?.wait === true) await unlessShutDown(ctx, () => awaitWriteVisibility(ctx, indexName));
|
|
3561
3750
|
return { succeeded, failed };
|
|
3562
3751
|
}
|
|
3563
3752
|
|