@delali/narsil 0.3.1-next.183 → 0.3.1-next.185
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-ZZUXMJIK.mjs → chunk-CPSHS46N.mjs} +129 -165
- package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
- package/dist/browser/{optional-writes-IQFTXANH.mjs → optional-writes-ZTBHYQMN.mjs} +619 -314
- package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
- package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
- package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
- package/dist/browser/{storage-leaf-2CO6SQ7R.mjs → storage-leaf-4XMCITJ6.mjs} +27 -9
- package/dist/{chunk-GGG5R4CU.mjs → chunk-2GONI5HL.mjs} +5 -3
- package/dist/{chunk-PK3BGGHO.mjs → chunk-2KCCQK4R.mjs} +1 -1
- package/dist/{chunk-JVR5KMO4.mjs → chunk-2YWWHMVC.mjs} +2 -2
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/{react-native/chunks/chunk-G4YYO5TC.mjs → chunk-4JBTALUC.mjs} +1 -1
- package/dist/{chunk-PXEGKQHS.mjs → chunk-5BUEBIXR.mjs} +2 -2
- package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
- package/dist/{chunk-NPXORGMK.mjs → chunk-7WXGUTWK.mjs} +3 -3
- package/dist/{chunk-2DK74OQL.mjs → chunk-A3OAFUZI.mjs} +1 -1
- package/dist/{chunk-632ABEIR.mjs → chunk-BVZWWMOU.mjs} +1 -1
- package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
- package/dist/{chunk-F2UOTRY7.mjs → chunk-CODGHJVW.mjs} +4 -4
- package/dist/{chunk-6ICOWB6I.mjs → chunk-CYCVPTTR.mjs} +1 -1
- package/dist/{chunk-WUWSUJ2X.mjs → chunk-DU23J5JG.mjs} +5 -5
- package/dist/{chunk-C77OBNPP.mjs → chunk-EDEE7ETQ.mjs} +1 -1
- package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
- package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
- package/dist/{chunk-V7F5DOSC.mjs → chunk-FBQYC5OX.mjs} +3 -3
- package/dist/{chunk-EGWBZSHK.mjs → chunk-FCQMZDRI.mjs} +2 -2
- package/dist/{chunk-MCAJOLVK.mjs → chunk-G3XRHAJA.mjs} +2 -2
- package/dist/{chunk-OQANB7SJ.mjs → chunk-H4QYQ5DE.mjs} +1 -1
- package/dist/{chunk-CKS4QQEC.mjs → chunk-II6XF6M4.mjs} +5 -5
- package/dist/{chunk-C3EINJNJ.mjs → chunk-JDG6CCXM.mjs} +1 -1
- package/dist/{chunk-3OEXOSWI.mjs → chunk-LR6JJVML.mjs} +4 -4
- package/dist/{chunk-TTTBE4TB.mjs → chunk-M5I64AQW.mjs} +3 -3
- package/dist/{chunk-FIXUVFUC.mjs → chunk-MHBG33R5.mjs} +7 -6
- package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
- package/dist/{chunk-M3H7CTIC.mjs → chunk-O337MDRJ.mjs} +5 -5
- package/dist/{chunk-CWXR7OP4.mjs → chunk-SL53DHVE.mjs} +804 -527
- package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
- package/dist/{chunk-SA7U67XF.mjs → chunk-URVH6BLO.mjs} +2 -2
- package/dist/{chunk-T3YI4KN6.mjs → chunk-VYSWVQCE.mjs} +1 -1
- package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
- package/dist/{chunk-ZVZ2V2PZ.mjs → chunk-WSIU6YXQ.mjs} +39 -20
- package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
- package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
- package/dist/{direct-executor-JXYIL4OQ.mjs → direct-executor-37IH3ISS.mjs} +5 -4
- package/dist/distribution.mjs +254 -76
- package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +5 -4
- package/dist/index.react-native.mjs +4 -3
- package/dist/{long-query-thread-HPKGRYWM.mjs → long-query-thread-DIPJBMV7.mjs} +15 -14
- package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
- package/dist/{optional-writes-QE7QUI45.mjs → optional-writes-YIVJMQ4D.mjs} +7 -6
- package/dist/{paged-partition-archive-IK2UOY2G.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +5 -4
- package/dist/persistence/durability/checkpoint-worker.mjs +13 -12
- package/dist/{query-actions-COEFVUH3.mjs → query-actions-7EOKT3WD.mjs} +5 -4
- package/dist/query-suggestions/merge-worker.mjs +8 -7
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.mjs +6 -5
- package/dist/react-native/chunks/{chunk-DZ4FQW5K.mjs → chunk-7TPO4FLE.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-6A5XE2MA.mjs → chunk-CBU4M3EU.mjs} +803 -526
- package/dist/react-native/chunks/{chunk-DN2GVVR4.mjs → chunk-IEY4DARA.mjs} +1 -1
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
- package/dist/react-native/chunks/{chunk-UPN4NUXU.mjs → chunk-T5HF3FW3.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XNIH3WIB.mjs → chunk-ZQPSXN6C.mjs} +1 -1
- package/dist/{chunk-YWSIRKKQ.mjs → react-native/chunks/chunk-ZUFOYZ5P.mjs} +1 -1
- package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
- package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
- package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
- package/dist/react-native/chunks/{optional-writes-YZ7WTR7V.mjs → optional-writes-QETIM6BI.mjs} +5 -4
- package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
- package/dist/react-native/chunks/{restore-queue-IGKD6Q2Y.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
- package/dist/react-native/chunks/{runtime-BUCSAK4U.mjs → runtime-UOHD2XUJ.mjs} +23 -3
- package/dist/react-native/chunks/{segment-codec-leaf-JTTGDT3G.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
- package/dist/react-native/chunks/{segment-merge-leaf-D5ZFJRUB.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
- package/dist/react-native/chunks/{storage-leaf-ZKIM6IF6.mjs → storage-leaf-LNOLKWZB.mjs} +36 -17
- package/dist/react-native/chunks/{store-leaf-PHBOTT5J.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
- package/dist/react-native/chunks/{suggestions-leaf-R7MIH6Q3.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
- package/dist/react-native/chunks/{worker-copies-U35KPVJ7.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
- package/dist/react-native/partial-words.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +4 -3
- package/dist/react-native/sync.mjs +4 -3
- package/dist/react-native/synonyms.mjs +4 -3
- package/dist/{rerank-LYL3D7LU.mjs → rerank-KTV5L7MZ.mjs} +5 -4
- package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
- package/dist/{restore-queue-LF4DTVJC.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
- package/dist/{runtime-4GKUIAKV.mjs → runtime-USQYKN3I.mjs} +23 -3
- package/dist/{segment-codec-leaf-ACALKS2C.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
- package/dist/{segment-merge-leaf-SKMPASFK.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
- package/dist/server.mjs +22 -21
- package/dist/{storage-leaf-OJ55R4C5.mjs → storage-leaf-ZI3H2WJY.mjs} +26 -25
- package/dist/{store-leaf-CWHHEDRK.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
- package/dist/{suggestions-leaf-ZTU3ZTWJ.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
- package/dist/sync/filesystem.mjs +10 -9
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.mjs +5 -4
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +5 -4
- package/dist/{thread-BEGFRULV.mjs → thread-NQRNW3IH.mjs} +14 -13
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/{worker-copies-373P6IYZ.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
- package/dist/workers/entry.mjs +11 -10
- package/package.json +7 -7
- package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
- package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
|
@@ -1628,19 +1628,17 @@ function removeDocument(input, generation, ordinal, fields) {
|
|
|
1628
1628
|
// src/engine/optional-writes-host.browser.ts
|
|
1629
1629
|
var BATCH_CHUNK_SIZE;
|
|
1630
1630
|
var ErrorCodes;
|
|
1631
|
+
var MAX_CALLS_ACROSS_INDEX_CHANGES;
|
|
1631
1632
|
var MIN_DOCUMENTS_FOR_SEGMENTS;
|
|
1632
1633
|
var NarsilError;
|
|
1633
1634
|
var admitBufferedUpsert;
|
|
1634
1635
|
var admitInsert;
|
|
1635
1636
|
var arrayPositionsAreStale;
|
|
1636
|
-
var asBatchInsertError;
|
|
1637
1637
|
var awaitWriteVisibility;
|
|
1638
|
+
var capsEachPartition;
|
|
1638
1639
|
var changeIndexOutsideExternalWaits;
|
|
1639
|
-
var collectRequiredFieldFailures;
|
|
1640
1640
|
var createFrozenSegment;
|
|
1641
|
-
var createPendingAdmissions;
|
|
1642
1641
|
var deleteNestedValue;
|
|
1643
|
-
var embedChunkDocuments;
|
|
1644
1642
|
var emitSyncChange;
|
|
1645
1643
|
var extractVectorFromDoc;
|
|
1646
1644
|
var fnv1a;
|
|
@@ -1662,7 +1660,6 @@ var prepareDocumentVectors;
|
|
|
1662
1660
|
var prepareStoredSynonyms;
|
|
1663
1661
|
var providedDocId;
|
|
1664
1662
|
var readHeapStatistics;
|
|
1665
|
-
var recordPendingAdmission;
|
|
1666
1663
|
var removeDocumentVectors;
|
|
1667
1664
|
var requireDocumentObject;
|
|
1668
1665
|
var requireDocumentWithinSizeLimit;
|
|
@@ -1703,19 +1700,17 @@ function bindOptionalWritesHost(host, segmentCodec) {
|
|
|
1703
1700
|
({
|
|
1704
1701
|
BATCH_CHUNK_SIZE,
|
|
1705
1702
|
ErrorCodes,
|
|
1703
|
+
MAX_CALLS_ACROSS_INDEX_CHANGES,
|
|
1706
1704
|
MIN_DOCUMENTS_FOR_SEGMENTS,
|
|
1707
1705
|
NarsilError,
|
|
1708
1706
|
admitBufferedUpsert,
|
|
1709
1707
|
admitInsert,
|
|
1710
1708
|
arrayPositionsAreStale,
|
|
1711
|
-
asBatchInsertError,
|
|
1712
1709
|
awaitWriteVisibility,
|
|
1710
|
+
capsEachPartition,
|
|
1713
1711
|
changeIndexOutsideExternalWaits,
|
|
1714
|
-
collectRequiredFieldFailures,
|
|
1715
1712
|
createFrozenSegment,
|
|
1716
|
-
createPendingAdmissions,
|
|
1717
1713
|
deleteNestedValue,
|
|
1718
|
-
embedChunkDocuments,
|
|
1719
1714
|
emitSyncChange,
|
|
1720
1715
|
extractVectorFromDoc,
|
|
1721
1716
|
fnv1a,
|
|
@@ -1737,7 +1732,6 @@ function bindOptionalWritesHost(host, segmentCodec) {
|
|
|
1737
1732
|
prepareStoredSynonyms,
|
|
1738
1733
|
providedDocId,
|
|
1739
1734
|
readHeapStatistics,
|
|
1740
|
-
recordPendingAdmission,
|
|
1741
1735
|
removeDocumentVectors,
|
|
1742
1736
|
requireDocumentObject,
|
|
1743
1737
|
requireDocumentWithinSizeLimit,
|
|
@@ -1995,6 +1989,326 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1995
1989
|
}
|
|
1996
1990
|
}
|
|
1997
1991
|
|
|
1992
|
+
// src/engine/mutations/batch-admissions.ts
|
|
1993
|
+
function createPendingAdmissions() {
|
|
1994
|
+
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
1995
|
+
}
|
|
1996
|
+
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
1997
|
+
pending.total += 1;
|
|
1998
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
1999
|
+
const partitionId = manager.routePartition(docId);
|
|
2000
|
+
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
2001
|
+
}
|
|
2002
|
+
function asBatchInsertError(err) {
|
|
2003
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
2004
|
+
}
|
|
2005
|
+
|
|
2006
|
+
// src/engine/mutations/outside-hold.ts
|
|
2007
|
+
function runOutside(hold, work) {
|
|
2008
|
+
return hold === void 0 ? work() : hold.outside(work);
|
|
2009
|
+
}
|
|
2010
|
+
|
|
2011
|
+
// src/engine/mutations/write-preparation.ts
|
|
2012
|
+
function settingsOf(entry) {
|
|
2013
|
+
return { required: entry.config.required, embedding: entry.config.embedding, adapter: entry.embeddingAdapter };
|
|
2014
|
+
}
|
|
2015
|
+
function embedsThroughAdapter(entry) {
|
|
2016
|
+
return entry.config.embedding !== void 0 && entry.embeddingAdapter !== null;
|
|
2017
|
+
}
|
|
2018
|
+
function sourcesOf(sources) {
|
|
2019
|
+
return typeof sources === "string" ? [sources] : sources;
|
|
2020
|
+
}
|
|
2021
|
+
function sameMapping(left, right) {
|
|
2022
|
+
if (left === right) return true;
|
|
2023
|
+
const targets = Object.keys(left);
|
|
2024
|
+
if (targets.length !== Object.keys(right).length) return false;
|
|
2025
|
+
for (const target of targets) {
|
|
2026
|
+
if (!Object.hasOwn(right, target)) return false;
|
|
2027
|
+
const leftSources = sourcesOf(left[target]);
|
|
2028
|
+
const rightSources = sourcesOf(right[target]);
|
|
2029
|
+
if (leftSources.length !== rightSources.length) return false;
|
|
2030
|
+
if (leftSources.some((source, position) => source !== rightSources[position])) return false;
|
|
2031
|
+
}
|
|
2032
|
+
return true;
|
|
2033
|
+
}
|
|
2034
|
+
function embeddingFits(entry, embedding) {
|
|
2035
|
+
const embeddingConfig = entry.config.embedding;
|
|
2036
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter === null) return embedding === null;
|
|
2037
|
+
return embedding !== null && embedding.adapter === entry.embeddingAdapter && sameMapping(embedding.mapping, embeddingConfig.fields);
|
|
2038
|
+
}
|
|
2039
|
+
function embeddingOfDocuments(embedding, documents) {
|
|
2040
|
+
if (embedding === null) return null;
|
|
2041
|
+
const fields = /* @__PURE__ */ new Map();
|
|
2042
|
+
for (const document of documents) {
|
|
2043
|
+
const embedded = embedding.fields.get(document);
|
|
2044
|
+
if (embedded !== void 0) fields.set(document, embedded);
|
|
2045
|
+
}
|
|
2046
|
+
return { adapter: embedding.adapter, mapping: embedding.mapping, fields };
|
|
2047
|
+
}
|
|
2048
|
+
function writable(preparations) {
|
|
2049
|
+
return preparations.filter((preparation) => preparation.failure === void 0);
|
|
2050
|
+
}
|
|
2051
|
+
function failAll(preparations, failure) {
|
|
2052
|
+
for (const preparation of writable(preparations)) preparation.failure = failure;
|
|
2053
|
+
}
|
|
2054
|
+
function requireFieldsOf(required, preparations) {
|
|
2055
|
+
if (required === void 0 || required.length === 0) return;
|
|
2056
|
+
for (const preparation of writable(preparations)) {
|
|
2057
|
+
try {
|
|
2058
|
+
validateRequiredFields(preparation.document, required);
|
|
2059
|
+
} catch (err) {
|
|
2060
|
+
preparation.failure = err;
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
async function requireMappedVectors(entry, preparations) {
|
|
2065
|
+
const embeddingConfig = entry.config.embedding;
|
|
2066
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter !== null) return;
|
|
2067
|
+
const { assertDocumentCarriesMappedVectors } = await loadEmbedCode();
|
|
2068
|
+
for (const preparation of writable(preparations)) {
|
|
2069
|
+
try {
|
|
2070
|
+
assertDocumentCarriesMappedVectors(
|
|
2071
|
+
preparation.document,
|
|
2072
|
+
embeddingConfig,
|
|
2073
|
+
entry.embeddingAdapterName
|
|
2074
|
+
);
|
|
2075
|
+
} catch (err) {
|
|
2076
|
+
preparation.failure = err;
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
}
|
|
2080
|
+
async function embedWith(settings, signal, preparations, earlier, together) {
|
|
2081
|
+
requireFieldsOf(settings.required, preparations);
|
|
2082
|
+
const { clearWriteEmbedding, embedDocumentsForWrite, lacksMappedVectors } = await loadEmbedCode();
|
|
2083
|
+
if (earlier !== null) clearWriteEmbedding(earlier);
|
|
2084
|
+
const { embedding: embeddingConfig, adapter } = settings;
|
|
2085
|
+
const embeddable = writable(preparations);
|
|
2086
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
2087
|
+
if (embeddingConfig === void 0 || adapter === null || embeddable.length === 0) {
|
|
2088
|
+
return { embedding: null, called: false, documentFailures };
|
|
2089
|
+
}
|
|
2090
|
+
const fields = /* @__PURE__ */ new Map();
|
|
2091
|
+
const embedding = { adapter, mapping: embeddingConfig.fields, fields };
|
|
2092
|
+
const documents = embeddable.map((preparation) => preparation.document);
|
|
2093
|
+
if (!lacksMappedVectors(documents, embeddingConfig)) return { embedding, called: false, documentFailures };
|
|
2094
|
+
for (let chunkStart = 0; chunkStart < embeddable.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2095
|
+
const chunk = embeddable.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
2096
|
+
const embedded = await embedDocumentsForWrite(
|
|
2097
|
+
chunk.map((preparation) => preparation.document),
|
|
2098
|
+
embeddingConfig,
|
|
2099
|
+
adapter,
|
|
2100
|
+
signal,
|
|
2101
|
+
together
|
|
2102
|
+
);
|
|
2103
|
+
for (const [document, embeddedFields] of embedded.embedding.fields) fields.set(document, embeddedFields);
|
|
2104
|
+
for (const [position, error] of embedded.failed) chunk[position].failure = error;
|
|
2105
|
+
for (const [position, error] of embedded.documentFailures) documentFailures.set(chunk[position], error);
|
|
2106
|
+
}
|
|
2107
|
+
return { embedding, called: true, documentFailures };
|
|
2108
|
+
}
|
|
2109
|
+
function settleDocumentFailures(entry, pass) {
|
|
2110
|
+
if (!embeddingFits(entry, pass.embedding)) return [...pass.documentFailures.keys()];
|
|
2111
|
+
for (const [preparation, failure] of pass.documentFailures) preparation.failure = failure;
|
|
2112
|
+
return [];
|
|
2113
|
+
}
|
|
2114
|
+
async function runBeforeWrites(preparations) {
|
|
2115
|
+
for (const preparation of preparations) {
|
|
2116
|
+
if (preparation.beforeWrite === null || preparation.failure !== void 0) continue;
|
|
2117
|
+
try {
|
|
2118
|
+
await preparation.beforeWrite();
|
|
2119
|
+
} catch (err) {
|
|
2120
|
+
preparation.failure = err;
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
async function embedThenRunHooks(settings, signal, preparations, earlier) {
|
|
2125
|
+
const pass = await embedWith(settings, signal, preparations, earlier, true);
|
|
2126
|
+
await runBeforeWrites(preparations.filter((preparation) => !pass.documentFailures.has(preparation)));
|
|
2127
|
+
return pass;
|
|
2128
|
+
}
|
|
2129
|
+
async function runHooksThenEmbedEach(ctx, indexName, preparations, earlier) {
|
|
2130
|
+
let settings = null;
|
|
2131
|
+
let embedding = null;
|
|
2132
|
+
let called = false;
|
|
2133
|
+
let uncleared = earlier;
|
|
2134
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
2135
|
+
for (const preparation of preparations) {
|
|
2136
|
+
await runBeforeWrites([preparation]);
|
|
2137
|
+
settings ??= settingsOf(ctx.requireIndex(indexName));
|
|
2138
|
+
const pass = await embedWith(settings, ctx.abortController.signal, [preparation], uncleared, false);
|
|
2139
|
+
uncleared = null;
|
|
2140
|
+
called ||= pass.called;
|
|
2141
|
+
for (const [failed, failure] of pass.documentFailures) documentFailures.set(failed, failure);
|
|
2142
|
+
if (pass.embedding === null) continue;
|
|
2143
|
+
if (embedding === null) embedding = pass.embedding;
|
|
2144
|
+
else for (const [document, fields] of pass.embedding.fields) embedding.fields.set(document, fields);
|
|
2145
|
+
}
|
|
2146
|
+
return { embedding, called, documentFailures };
|
|
2147
|
+
}
|
|
2148
|
+
function adapterChangesRefusal(indexName, embeddings) {
|
|
2149
|
+
return new NarsilError(
|
|
2150
|
+
ErrorCodes.WORKER_BUSY,
|
|
2151
|
+
`Changes to index "${indexName}" replaced its embedding adapter after each of ${embeddings} embeddings of this write`,
|
|
2152
|
+
{ indexName, embeddings }
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
async function prepareDocumentWrites(ctx, indexName, preparations, hold, together, earlier = null) {
|
|
2156
|
+
let entry = ctx.requireIndex(indexName);
|
|
2157
|
+
let embedding = earlier;
|
|
2158
|
+
if (writable(preparations).length === 0) return { entry, embedding };
|
|
2159
|
+
const signal = ctx.abortController.signal;
|
|
2160
|
+
let embeddings = 0;
|
|
2161
|
+
let unhooked = [];
|
|
2162
|
+
const hooked = preparations.some((preparation) => preparation.beforeWrite !== null);
|
|
2163
|
+
if (hooked || embedsThroughAdapter(entry) && !embeddingFits(entry, embedding)) {
|
|
2164
|
+
if (together && !embedsThroughAdapter(entry)) {
|
|
2165
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
2166
|
+
await requireMappedVectors(entry, preparations);
|
|
2167
|
+
}
|
|
2168
|
+
const settings = settingsOf(entry);
|
|
2169
|
+
const uncleared = embedding;
|
|
2170
|
+
const pass = await runOutside(
|
|
2171
|
+
hold,
|
|
2172
|
+
() => together ? embedThenRunHooks(settings, signal, preparations, uncleared) : runHooksThenEmbedEach(ctx, indexName, preparations, uncleared)
|
|
2173
|
+
);
|
|
2174
|
+
embedding = pass.embedding;
|
|
2175
|
+
if (pass.called) embeddings += 1;
|
|
2176
|
+
entry = ctx.requireIndex(indexName);
|
|
2177
|
+
unhooked = settleDocumentFailures(entry, pass);
|
|
2178
|
+
}
|
|
2179
|
+
while (writable(preparations).length > 0 && !embeddingFits(entry, embedding)) {
|
|
2180
|
+
if (embeddings >= MAX_CALLS_ACROSS_INDEX_CHANGES) {
|
|
2181
|
+
failAll(preparations, adapterChangesRefusal(indexName, embeddings));
|
|
2182
|
+
break;
|
|
2183
|
+
}
|
|
2184
|
+
const uncleared = embedding;
|
|
2185
|
+
const settings = settingsOf(entry);
|
|
2186
|
+
const hooking = unhooked;
|
|
2187
|
+
const pass = await runOutside(hold, async () => {
|
|
2188
|
+
const embedded = await embedWith(settings, signal, preparations, uncleared, together);
|
|
2189
|
+
await runBeforeWrites(hooking.filter((preparation) => !embedded.documentFailures.has(preparation)));
|
|
2190
|
+
return embedded;
|
|
2191
|
+
});
|
|
2192
|
+
embedding = pass.embedding;
|
|
2193
|
+
if (pass.called) embeddings += 1;
|
|
2194
|
+
entry = ctx.requireIndex(indexName);
|
|
2195
|
+
const stale = settleDocumentFailures(entry, pass);
|
|
2196
|
+
unhooked = hooking.filter((preparation) => stale.includes(preparation));
|
|
2197
|
+
}
|
|
2198
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
2199
|
+
await requireMappedVectors(entry, preparations);
|
|
2200
|
+
return { entry, embedding };
|
|
2201
|
+
}
|
|
2202
|
+
async function prepareDocumentWrite(ctx, indexName, document, hold, beforeWrite) {
|
|
2203
|
+
const preparation = { document, beforeWrite, failure: void 0 };
|
|
2204
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
2205
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
2206
|
+
return entry;
|
|
2207
|
+
}
|
|
2208
|
+
|
|
2209
|
+
// src/engine/mutations/insert-batch-admission.ts
|
|
2210
|
+
function pendingInsertOf(ctx, indexName, docId, document, hooked) {
|
|
2211
|
+
const pending = {
|
|
2212
|
+
docId,
|
|
2213
|
+
preparation: { document, beforeWrite: null, failure: void 0 },
|
|
2214
|
+
refusedByHook: false
|
|
2215
|
+
};
|
|
2216
|
+
if (hooked && ctx.pluginRegistry.hasHooks("beforeInsert")) {
|
|
2217
|
+
pending.preparation.beforeWrite = async () => {
|
|
2218
|
+
try {
|
|
2219
|
+
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document });
|
|
2220
|
+
} catch (err) {
|
|
2221
|
+
pending.refusedByHook = true;
|
|
2222
|
+
throw err;
|
|
2223
|
+
}
|
|
2224
|
+
};
|
|
2225
|
+
}
|
|
2226
|
+
return pending;
|
|
2227
|
+
}
|
|
2228
|
+
async function prepareInsertDocuments(ctx, indexName, documents, start, end, hold, hooked, failed, earlier = null) {
|
|
2229
|
+
const pending = [];
|
|
2230
|
+
for (let i = start; i < end; i++) {
|
|
2231
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2232
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2233
|
+
const document = documents[i];
|
|
2234
|
+
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
2235
|
+
try {
|
|
2236
|
+
const sizing = requireDocumentObject(document);
|
|
2237
|
+
if (sizing !== null) await sizing;
|
|
2238
|
+
validateDocId(docId);
|
|
2239
|
+
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
2240
|
+
} catch (err) {
|
|
2241
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
const { entry, embedding } = await prepareDocumentWrites(
|
|
2245
|
+
ctx,
|
|
2246
|
+
indexName,
|
|
2247
|
+
pending.map((item) => item.preparation),
|
|
2248
|
+
hold,
|
|
2249
|
+
true,
|
|
2250
|
+
earlier
|
|
2251
|
+
);
|
|
2252
|
+
const candidates = [];
|
|
2253
|
+
for (const item of pending) {
|
|
2254
|
+
const { document, failure } = item.preparation;
|
|
2255
|
+
if (failure === void 0) {
|
|
2256
|
+
candidates.push({ docId: item.docId, document });
|
|
2257
|
+
continue;
|
|
2258
|
+
}
|
|
2259
|
+
const docId = item.refusedByHook ? item.docId : providedDocId(document) ?? "";
|
|
2260
|
+
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
2261
|
+
}
|
|
2262
|
+
return { entry, candidates, embedding };
|
|
2263
|
+
}
|
|
2264
|
+
async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
|
|
2265
|
+
const { entry, candidates, embedding } = await prepareInsertDocuments(
|
|
2266
|
+
ctx,
|
|
2267
|
+
indexName,
|
|
2268
|
+
documents,
|
|
2269
|
+
0,
|
|
2270
|
+
documents.length,
|
|
2271
|
+
hold,
|
|
2272
|
+
true,
|
|
2273
|
+
failed
|
|
2274
|
+
);
|
|
2275
|
+
const manager = ctx.requireManager(indexName);
|
|
2276
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
2277
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2278
|
+
const strict = entry.config.strict === true;
|
|
2279
|
+
const admitted = [];
|
|
2280
|
+
const admittedIds = /* @__PURE__ */ new Set();
|
|
2281
|
+
const pending = createPendingAdmissions();
|
|
2282
|
+
for (let position = 0; position < candidates.length; position++) {
|
|
2283
|
+
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
2284
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2285
|
+
const { docId, document } = candidates[position];
|
|
2286
|
+
try {
|
|
2287
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2288
|
+
document,
|
|
2289
|
+
vectorFieldPaths
|
|
2290
|
+
);
|
|
2291
|
+
if (extractedVectors.size > 0) {
|
|
2292
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
2293
|
+
}
|
|
2294
|
+
admitInsert(ctx, indexName, manager, docId, pending);
|
|
2295
|
+
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
2296
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
2297
|
+
}
|
|
2298
|
+
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
2299
|
+
if (strict) {
|
|
2300
|
+
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
2301
|
+
}
|
|
2302
|
+
admittedIds.add(docId);
|
|
2303
|
+
recordPendingAdmission(ctx, pending, manager, docId);
|
|
2304
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2305
|
+
} catch (err) {
|
|
2306
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
return { admitted, embedding };
|
|
2310
|
+
}
|
|
2311
|
+
|
|
1998
2312
|
// src/engine/mutations/durable-rollback.ts
|
|
1999
2313
|
function holdVectorsForRollback(indexes, docId) {
|
|
2000
2314
|
const releases = [];
|
|
@@ -2230,84 +2544,29 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
2230
2544
|
return applications;
|
|
2231
2545
|
}
|
|
2232
2546
|
|
|
2233
|
-
// src/engine/mutations/insert-batch-
|
|
2234
|
-
async function
|
|
2235
|
-
|
|
2236
|
-
const manager = ctx.
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
2240
|
-
const strict = entry.config.strict === true;
|
|
2241
|
-
const required = entry.config.required;
|
|
2242
|
-
const admitted = [];
|
|
2243
|
-
const admittedIds = /* @__PURE__ */ new Set();
|
|
2244
|
-
const pending = createPendingAdmissions();
|
|
2245
|
-
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2246
|
-
if (ctx.abortController.signal.aborted) break;
|
|
2247
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2248
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
2249
|
-
if (required && required.length > 0) {
|
|
2250
|
-
collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
2251
|
-
}
|
|
2252
|
-
await embedChunkDocuments(
|
|
2253
|
-
entry,
|
|
2254
|
-
documents,
|
|
2255
|
-
chunkStart,
|
|
2256
|
-
chunkEnd,
|
|
2257
|
-
ctx.abortController.signal,
|
|
2258
|
-
chunkFailedIndexes,
|
|
2259
|
-
failed
|
|
2260
|
-
);
|
|
2261
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
2262
|
-
if (ctx.abortController.signal.aborted) break;
|
|
2263
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
2264
|
-
const docId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
2547
|
+
// src/engine/mutations/insert-batch-documents.ts
|
|
2548
|
+
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
2549
|
+
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
2550
|
+
const manager = ctx.executor.getManager(indexName);
|
|
2551
|
+
await runOutside(hold, async () => {
|
|
2552
|
+
for (const doc of inserted) {
|
|
2265
2553
|
try {
|
|
2266
|
-
|
|
2267
|
-
if (sizing !== null) await sizing;
|
|
2268
|
-
validateDocId(docId);
|
|
2269
|
-
if (hasBeforeHook) {
|
|
2270
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
|
|
2271
|
-
}
|
|
2272
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2273
|
-
documents[i],
|
|
2274
|
-
vectorFieldPaths
|
|
2275
|
-
);
|
|
2276
|
-
if (extractedVectors.size > 0) {
|
|
2277
|
-
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
2278
|
-
}
|
|
2279
|
-
admitInsert(ctx, indexName, manager, docId, pending);
|
|
2280
|
-
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
2281
|
-
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
2282
|
-
}
|
|
2283
|
-
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
2284
|
-
if (strict) {
|
|
2285
|
-
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
2286
|
-
}
|
|
2287
|
-
admittedIds.add(docId);
|
|
2288
|
-
recordPendingAdmission(ctx, pending, manager, docId);
|
|
2289
|
-
admitted.push({ docId, document: documents[i], partitionDoc, extractedVectors });
|
|
2554
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2290
2555
|
} catch (err) {
|
|
2291
|
-
|
|
2556
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2292
2557
|
}
|
|
2293
2558
|
}
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
}
|
|
2297
|
-
}
|
|
2298
|
-
return admitted;
|
|
2559
|
+
});
|
|
2560
|
+
return ctx.executor.getManager(indexName) !== manager;
|
|
2299
2561
|
}
|
|
2300
|
-
|
|
2301
|
-
// src/engine/mutations/insert-batch-documents.ts
|
|
2302
|
-
async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed) {
|
|
2303
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
2562
|
+
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
2304
2563
|
const succeeded = [];
|
|
2305
|
-
const buffered = /* @__PURE__ */ new Set();
|
|
2306
2564
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2307
2565
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2308
2566
|
if (ctx.abortController.signal.aborted) break;
|
|
2309
2567
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
2310
2568
|
const applications = await applyInsertChunk(ctx, indexName, chunk, options);
|
|
2569
|
+
const inserted = [];
|
|
2311
2570
|
for (let i = 0; i < chunk.length; i++) {
|
|
2312
2571
|
const doc = chunk[i];
|
|
2313
2572
|
const application = applications[i];
|
|
@@ -2316,30 +2575,103 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
|
|
|
2316
2575
|
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2317
2576
|
continue;
|
|
2318
2577
|
}
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
succeeded.push(doc.docId);
|
|
2322
|
-
continue;
|
|
2323
|
-
}
|
|
2578
|
+
succeeded.push(doc.docId);
|
|
2579
|
+
if (application.status === "buffered") continue;
|
|
2324
2580
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2325
2581
|
touchedVectorFields.add(fieldPath);
|
|
2326
2582
|
}
|
|
2327
|
-
|
|
2328
|
-
try {
|
|
2329
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2330
|
-
} catch (err) {
|
|
2331
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2332
|
-
}
|
|
2333
|
-
}
|
|
2334
|
-
succeeded.push(doc.docId);
|
|
2583
|
+
inserted.push(doc);
|
|
2335
2584
|
}
|
|
2336
2585
|
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
2586
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
2587
|
+
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
2588
|
+
}
|
|
2337
2589
|
if (applied < admitted.length) {
|
|
2338
2590
|
ctx.checkHeapPressure(indexName);
|
|
2339
2591
|
await new Promise((r) => setTimeout(r, 0));
|
|
2340
2592
|
}
|
|
2341
2593
|
}
|
|
2342
|
-
return { succeeded,
|
|
2594
|
+
return { succeeded, touchedVectorFields, unrecorded: [] };
|
|
2595
|
+
}
|
|
2596
|
+
|
|
2597
|
+
// src/engine/mutations/insert-batch-chunks.ts
|
|
2598
|
+
async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
2599
|
+
const { entry, candidates } = await prepareInsertDocuments(
|
|
2600
|
+
ctx,
|
|
2601
|
+
indexName,
|
|
2602
|
+
documents,
|
|
2603
|
+
chunkStart,
|
|
2604
|
+
chunkEnd,
|
|
2605
|
+
hold,
|
|
2606
|
+
hooked,
|
|
2607
|
+
failed,
|
|
2608
|
+
earlier
|
|
2609
|
+
);
|
|
2610
|
+
const manager = ctx.requireManager(indexName);
|
|
2611
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
2612
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2613
|
+
const admitted = [];
|
|
2614
|
+
for (const { docId, document } of candidates) {
|
|
2615
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2616
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2617
|
+
try {
|
|
2618
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2619
|
+
document,
|
|
2620
|
+
vectorFieldPaths
|
|
2621
|
+
);
|
|
2622
|
+
if (extractedVectors.size > 0) {
|
|
2623
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
2624
|
+
}
|
|
2625
|
+
admitInsert(ctx, indexName, manager, docId);
|
|
2626
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2627
|
+
} catch (err) {
|
|
2628
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2629
|
+
}
|
|
2630
|
+
}
|
|
2631
|
+
return admitted;
|
|
2632
|
+
}
|
|
2633
|
+
async function insertInChunks(ctx, indexName, documents, options, hold, hooked, failed, embedded = null) {
|
|
2634
|
+
const succeeded = [];
|
|
2635
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2636
|
+
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2637
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2638
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2639
|
+
const earlier = embeddingOfDocuments(embedded, documents.slice(chunkStart, chunkEnd));
|
|
2640
|
+
const prepared = await admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
2641
|
+
if (prepared.length > 0) {
|
|
2642
|
+
const applications = await applyInsertChunk(ctx, indexName, prepared, options);
|
|
2643
|
+
const inserted = [];
|
|
2644
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
2645
|
+
const doc = prepared[i];
|
|
2646
|
+
const application = applications[i];
|
|
2647
|
+
if (application.status === "skipped") continue;
|
|
2648
|
+
if (application.status === "failed") {
|
|
2649
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2650
|
+
continue;
|
|
2651
|
+
}
|
|
2652
|
+
succeeded.push(doc.docId);
|
|
2653
|
+
if (application.status === "buffered") continue;
|
|
2654
|
+
for (const fieldPath of doc.extractedVectors.keys()) touchedVectorFields.add(fieldPath);
|
|
2655
|
+
inserted.push(doc);
|
|
2656
|
+
}
|
|
2657
|
+
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
2658
|
+
}
|
|
2659
|
+
if (chunkEnd < documents.length) {
|
|
2660
|
+
ctx.checkHeapPressure(indexName);
|
|
2661
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2662
|
+
}
|
|
2663
|
+
}
|
|
2664
|
+
return { succeeded, touchedVectorFields };
|
|
2665
|
+
}
|
|
2666
|
+
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
2667
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
2668
|
+
for (const fieldPath of inserts.touchedVectorFields) {
|
|
2669
|
+
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
2670
|
+
}
|
|
2671
|
+
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
2672
|
+
ctx.checkHeapPressure(indexName);
|
|
2673
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
2674
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2343
2675
|
}
|
|
2344
2676
|
|
|
2345
2677
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -2444,10 +2776,6 @@ function mainStoreDocument(doc, options) {
|
|
|
2444
2776
|
if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
|
|
2445
2777
|
return structuredClone(doc.partitionDoc);
|
|
2446
2778
|
}
|
|
2447
|
-
async function applyIndividually(ctx, indexName, admitted, options, failed) {
|
|
2448
|
-
const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
|
|
2449
|
-
return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
|
|
2450
|
-
}
|
|
2451
2779
|
function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted) {
|
|
2452
2780
|
const chunks = [];
|
|
2453
2781
|
let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
@@ -2497,12 +2825,12 @@ function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberI
|
|
|
2497
2825
|
if (chunk.docs.length > 0) chunks.push(chunk);
|
|
2498
2826
|
return chunks;
|
|
2499
2827
|
}
|
|
2500
|
-
async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, failed) {
|
|
2501
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
2828
|
+
async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, hold, failed) {
|
|
2502
2829
|
const succeededIds = /* @__PURE__ */ new Set();
|
|
2503
2830
|
const errorOf = /* @__PURE__ */ new Map();
|
|
2504
2831
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2505
2832
|
const chunks = chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted);
|
|
2833
|
+
const unrecorded = [];
|
|
2506
2834
|
for (let index = 0; index < chunks.length; index++) {
|
|
2507
2835
|
const { docs, attachmentOf } = chunks[index];
|
|
2508
2836
|
const applies = docs.map((doc) => {
|
|
@@ -2512,6 +2840,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2512
2840
|
});
|
|
2513
2841
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2514
2842
|
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
2843
|
+
const inserted = [];
|
|
2515
2844
|
for (let i = 0; i < docs.length; i++) {
|
|
2516
2845
|
const doc = docs[i];
|
|
2517
2846
|
const failure = failures[i];
|
|
@@ -2543,15 +2872,13 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2543
2872
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2544
2873
|
touchedVectorFields.add(fieldPath);
|
|
2545
2874
|
}
|
|
2546
|
-
|
|
2547
|
-
try {
|
|
2548
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2549
|
-
} catch (err) {
|
|
2550
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2551
|
-
}
|
|
2552
|
-
}
|
|
2875
|
+
inserted.push(doc);
|
|
2553
2876
|
succeededIds.add(doc.docId);
|
|
2554
2877
|
}
|
|
2878
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
2879
|
+
for (const later of chunks.slice(index + 1)) unrecorded.push(...later.docs);
|
|
2880
|
+
break;
|
|
2881
|
+
}
|
|
2555
2882
|
if (index + 1 < chunks.length) {
|
|
2556
2883
|
ctx.checkHeapPressure(indexName);
|
|
2557
2884
|
await new Promise((r) => setTimeout(r, 0));
|
|
@@ -2566,7 +2893,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2566
2893
|
if (!errorOf.has(doc.docId)) continue;
|
|
2567
2894
|
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
2568
2895
|
}
|
|
2569
|
-
return { succeeded, touchedVectorFields };
|
|
2896
|
+
return { succeeded, touchedVectorFields, unrecorded };
|
|
2570
2897
|
}
|
|
2571
2898
|
function frozenSegmentOf(segment, members, options) {
|
|
2572
2899
|
if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
|
|
@@ -2577,14 +2904,15 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
2577
2904
|
segment.segmentId
|
|
2578
2905
|
);
|
|
2579
2906
|
}
|
|
2580
|
-
async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
2907
|
+
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
2581
2908
|
if (admitted.length === 0) {
|
|
2582
|
-
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set() };
|
|
2909
|
+
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] };
|
|
2583
2910
|
}
|
|
2911
|
+
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
2584
2912
|
const manager = ctx.requireManager(indexName);
|
|
2585
2913
|
const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
|
|
2586
2914
|
if (admitted.length < MIN_DOCUMENTS_FOR_SEGMENTS || workers <= 0) {
|
|
2587
|
-
return applyIndividually(
|
|
2915
|
+
return applyIndividually();
|
|
2588
2916
|
}
|
|
2589
2917
|
const docIds = admitted.map((doc) => doc.docId);
|
|
2590
2918
|
const partitionDocuments = admitted.map((doc) => doc.partitionDoc);
|
|
@@ -2597,7 +2925,7 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
2597
2925
|
built = null;
|
|
2598
2926
|
}
|
|
2599
2927
|
if (built === null) {
|
|
2600
|
-
return applyIndividually(
|
|
2928
|
+
return applyIndividually();
|
|
2601
2929
|
}
|
|
2602
2930
|
const positions = new Map(admitted.map((document, index) => [document.docId, index]));
|
|
2603
2931
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -2606,16 +2934,15 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
2606
2934
|
const members = [];
|
|
2607
2935
|
for (const docId of segment.docIds) {
|
|
2608
2936
|
const position = positions.get(docId);
|
|
2609
|
-
if (position === void 0 || used.has(position))
|
|
2610
|
-
return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
2937
|
+
if (position === void 0 || used.has(position)) return applyIndividually();
|
|
2611
2938
|
used.add(position);
|
|
2612
2939
|
members.push(position);
|
|
2613
2940
|
}
|
|
2614
2941
|
memberIndexes.push(members);
|
|
2615
2942
|
}
|
|
2616
|
-
if (used.size !== admitted.length) return applyIndividually(
|
|
2943
|
+
if (used.size !== admitted.length) return applyIndividually();
|
|
2617
2944
|
if (ctx.isRebalancing(indexName)) {
|
|
2618
|
-
return applyIndividually(
|
|
2945
|
+
return applyIndividually();
|
|
2619
2946
|
}
|
|
2620
2947
|
const frozen = [];
|
|
2621
2948
|
for (let i = 0; i < built.length; i++) {
|
|
@@ -2624,149 +2951,65 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
2624
2951
|
memberIndexes[i].map((m) => admitted[m]),
|
|
2625
2952
|
options
|
|
2626
2953
|
);
|
|
2627
|
-
if (segment === null) return applyIndividually(
|
|
2954
|
+
if (segment === null) return applyIndividually();
|
|
2628
2955
|
frozen.push(segment);
|
|
2629
2956
|
}
|
|
2630
2957
|
for (const doc of admitted) {
|
|
2631
|
-
if (manager.has(doc.docId)) return applyIndividually(
|
|
2958
|
+
if (manager.has(doc.docId)) return applyIndividually();
|
|
2632
2959
|
}
|
|
2633
|
-
return recordMergedDocuments(
|
|
2960
|
+
return recordMergedDocuments(
|
|
2961
|
+
ctx,
|
|
2962
|
+
indexName,
|
|
2963
|
+
partitionCount,
|
|
2964
|
+
built,
|
|
2965
|
+
frozen,
|
|
2966
|
+
memberIndexes,
|
|
2967
|
+
admitted,
|
|
2968
|
+
options,
|
|
2969
|
+
hold,
|
|
2970
|
+
failed
|
|
2971
|
+
);
|
|
2634
2972
|
}
|
|
2635
|
-
async function insertBatchViaSegments(ctx, indexName, documents, options) {
|
|
2973
|
+
async function insertBatchViaSegments(ctx, indexName, documents, options, hold) {
|
|
2636
2974
|
const failed = [];
|
|
2637
|
-
const admitted = await admitBatchDocuments(ctx, indexName, documents, failed);
|
|
2638
|
-
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, failed);
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
await ctx
|
|
2646
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2975
|
+
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents, failed, hold);
|
|
2976
|
+
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
2977
|
+
if (outcome.unrecorded.length > 0) {
|
|
2978
|
+
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
2979
|
+
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
2980
|
+
outcome.succeeded.push(...rest.succeeded);
|
|
2981
|
+
for (const fieldPath of rest.touchedVectorFields) outcome.touchedVectorFields.add(fieldPath);
|
|
2982
|
+
}
|
|
2983
|
+
await finishInsertBatch(ctx, indexName, outcome, options);
|
|
2647
2984
|
return { succeeded: outcome.succeeded, failed };
|
|
2648
2985
|
}
|
|
2649
2986
|
|
|
2650
2987
|
// src/engine/mutations/insert-batch.ts
|
|
2651
|
-
async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
2988
|
+
async function insertDocumentBatch(ctx, indexName, documents, options, hold) {
|
|
2652
2989
|
ctx.guardShutdown();
|
|
2653
|
-
|
|
2990
|
+
ctx.requireIndex(indexName);
|
|
2654
2991
|
validateBatchList(documents, "documents");
|
|
2655
2992
|
await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length);
|
|
2656
2993
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
2657
2994
|
if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
2658
|
-
return insertBatchViaSegments(ctx, indexName, documents, options);
|
|
2995
|
+
return insertBatchViaSegments(ctx, indexName, documents, options, hold);
|
|
2659
2996
|
}
|
|
2660
|
-
const succeeded = [];
|
|
2661
2997
|
const failed = [];
|
|
2662
|
-
const
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
const batchManager = ctx.requireManager(indexName);
|
|
2666
|
-
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
2667
|
-
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2668
|
-
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2669
|
-
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2670
|
-
if (ctx.abortController.signal.aborted) break;
|
|
2671
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2672
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
2673
|
-
if (required && required.length > 0) {
|
|
2674
|
-
collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
2675
|
-
}
|
|
2676
|
-
await embedChunkDocuments(
|
|
2677
|
-
entry,
|
|
2678
|
-
documents,
|
|
2679
|
-
chunkStart,
|
|
2680
|
-
chunkEnd,
|
|
2681
|
-
ctx.abortController.signal,
|
|
2682
|
-
chunkFailedIndexes,
|
|
2683
|
-
failed
|
|
2684
|
-
);
|
|
2685
|
-
async function applyPrepared(prepared2) {
|
|
2686
|
-
const applications = await applyInsertChunk(ctx, indexName, prepared2, options);
|
|
2687
|
-
for (let i = 0; i < prepared2.length; i++) {
|
|
2688
|
-
const doc = prepared2[i];
|
|
2689
|
-
const application = applications[i];
|
|
2690
|
-
if (application.status === "skipped") continue;
|
|
2691
|
-
if (application.status === "failed") {
|
|
2692
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2693
|
-
continue;
|
|
2694
|
-
}
|
|
2695
|
-
if (application.status === "buffered") {
|
|
2696
|
-
succeeded.push(doc.docId);
|
|
2697
|
-
continue;
|
|
2698
|
-
}
|
|
2699
|
-
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2700
|
-
touchedVectorFields.add(fieldPath);
|
|
2701
|
-
}
|
|
2702
|
-
if (hasAfterHook) {
|
|
2703
|
-
try {
|
|
2704
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2705
|
-
} catch (err) {
|
|
2706
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2707
|
-
}
|
|
2708
|
-
}
|
|
2709
|
-
succeeded.push(doc.docId);
|
|
2710
|
-
}
|
|
2711
|
-
}
|
|
2712
|
-
const prepared = [];
|
|
2713
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
2714
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2715
|
-
if (ctx.abortController.signal.aborted) break;
|
|
2716
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
2717
|
-
const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
2718
|
-
try {
|
|
2719
|
-
const sizing = requireDocumentObject(documents[i]);
|
|
2720
|
-
if (sizing !== null) await sizing;
|
|
2721
|
-
validateDocId(batchDocId);
|
|
2722
|
-
if (hasBeforeHook) {
|
|
2723
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
|
|
2724
|
-
}
|
|
2725
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2726
|
-
documents[i],
|
|
2727
|
-
batchVectorFieldPaths
|
|
2728
|
-
);
|
|
2729
|
-
if (extractedVectors.size > 0) {
|
|
2730
|
-
validateVectorDimensions(extractedVectors, batchVecIndexes);
|
|
2731
|
-
}
|
|
2732
|
-
admitInsert(ctx, indexName, batchManager, batchDocId);
|
|
2733
|
-
prepared.push({
|
|
2734
|
-
docId: batchDocId,
|
|
2735
|
-
document: documents[i],
|
|
2736
|
-
partitionDoc,
|
|
2737
|
-
extractedVectors
|
|
2738
|
-
});
|
|
2739
|
-
} catch (err) {
|
|
2740
|
-
failed.push({ docId: batchDocId, error: asBatchInsertError(err) });
|
|
2741
|
-
}
|
|
2742
|
-
}
|
|
2743
|
-
if (prepared.length > 0) await applyPrepared(prepared);
|
|
2744
|
-
if (chunkEnd < documents.length) {
|
|
2745
|
-
ctx.checkHeapPressure(indexName);
|
|
2746
|
-
await new Promise((r) => setTimeout(r, 0));
|
|
2747
|
-
}
|
|
2748
|
-
}
|
|
2749
|
-
for (const fieldPath of touchedVectorFields) {
|
|
2750
|
-
const vecIndex = batchVecIndexes.get(fieldPath);
|
|
2751
|
-
if (vecIndex) {
|
|
2752
|
-
vecIndex.scheduleBuild();
|
|
2753
|
-
}
|
|
2754
|
-
}
|
|
2755
|
-
ctx.checkWatermark(indexName, succeeded);
|
|
2756
|
-
ctx.checkHeapPressure(indexName);
|
|
2757
|
-
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
2758
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2759
|
-
return { succeeded, failed };
|
|
2998
|
+
const inserts = await insertInChunks(ctx, indexName, documents, options, hold, true, failed);
|
|
2999
|
+
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
3000
|
+
return { succeeded: inserts.succeeded, failed };
|
|
2760
3001
|
}
|
|
2761
3002
|
|
|
2762
3003
|
// src/engine/mutations/remove.ts
|
|
2763
3004
|
function asRemoveError(err) {
|
|
2764
3005
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
2765
3006
|
}
|
|
2766
|
-
|
|
3007
|
+
function validateRemoval(ctx, docId) {
|
|
2767
3008
|
ctx.guardShutdown();
|
|
2768
3009
|
validateDocId(docId);
|
|
2769
|
-
|
|
3010
|
+
}
|
|
3011
|
+
function prepareRemoval(ctx, indexName, docId) {
|
|
3012
|
+
ctx.guardShutdown();
|
|
2770
3013
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
2771
3014
|
if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
|
|
2772
3015
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
@@ -2816,20 +3059,31 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
2816
3059
|
};
|
|
2817
3060
|
return () => manager.publication.writeNow(indexChangeOutsideExternalWaits(removeFromIndex));
|
|
2818
3061
|
}
|
|
2819
|
-
async function
|
|
3062
|
+
async function settleRemovals(ctx, indexName, docIds, hold) {
|
|
3063
|
+
if (docIds.length === 0) return;
|
|
2820
3064
|
const settling = ctx.orchestrator.settlePublications(indexName, "none");
|
|
2821
3065
|
if (settling !== void 0) await settling;
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
3066
|
+
if (!ctx.pluginRegistry.hasHooks("afterRemove")) return;
|
|
3067
|
+
await runOutside(hold, async () => {
|
|
3068
|
+
for (const docId of docIds) {
|
|
3069
|
+
try {
|
|
3070
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
3071
|
+
} catch (err) {
|
|
3072
|
+
console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
3073
|
+
}
|
|
3074
|
+
}
|
|
3075
|
+
});
|
|
2828
3076
|
}
|
|
2829
|
-
async function removeDocument2(ctx, indexName, docId, options) {
|
|
3077
|
+
async function removeDocument2(ctx, indexName, docId, options, hold) {
|
|
2830
3078
|
ctx.guardShutdown();
|
|
2831
3079
|
ctx.requireIndex(indexName);
|
|
2832
|
-
|
|
3080
|
+
validateRemoval(ctx, docId);
|
|
3081
|
+
if (ctx.pluginRegistry.hasHooks("beforeRemove")) {
|
|
3082
|
+
await runOutside(hold, async () => {
|
|
3083
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
|
|
3084
|
+
});
|
|
3085
|
+
}
|
|
3086
|
+
const prepared = prepareRemoval(ctx, indexName, docId);
|
|
2833
3087
|
const apply = applyOfRemoval(ctx, indexName, prepared);
|
|
2834
3088
|
if (ctx.durability) {
|
|
2835
3089
|
try {
|
|
@@ -2843,25 +3097,52 @@ async function removeDocument2(ctx, indexName, docId, options) {
|
|
|
2843
3097
|
} else {
|
|
2844
3098
|
await apply();
|
|
2845
3099
|
}
|
|
2846
|
-
if (!prepared.buffered) await
|
|
3100
|
+
if (!prepared.buffered) await settleRemovals(ctx, indexName, [docId], hold);
|
|
2847
3101
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2848
3102
|
}
|
|
2849
|
-
async function
|
|
3103
|
+
async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold) {
|
|
3104
|
+
const refused = /* @__PURE__ */ new Map();
|
|
3105
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3106
|
+
try {
|
|
3107
|
+
validateRemoval(ctx, docIds[i]);
|
|
3108
|
+
} catch (err) {
|
|
3109
|
+
refused.set(i, err);
|
|
3110
|
+
}
|
|
3111
|
+
}
|
|
3112
|
+
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
3113
|
+
await runOutside(hold, async () => {
|
|
3114
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3115
|
+
if (refused.has(i)) continue;
|
|
3116
|
+
try {
|
|
3117
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
3118
|
+
} catch (err) {
|
|
3119
|
+
refused.set(i, err);
|
|
3120
|
+
}
|
|
3121
|
+
}
|
|
3122
|
+
});
|
|
3123
|
+
return refused;
|
|
3124
|
+
}
|
|
3125
|
+
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
2850
3126
|
ctx.guardShutdown();
|
|
2851
3127
|
ctx.requireIndex(indexName);
|
|
2852
3128
|
validateBatchList(docIds, "document IDs");
|
|
2853
3129
|
const succeeded = [];
|
|
2854
3130
|
const failed = [];
|
|
2855
|
-
const
|
|
2856
|
-
await
|
|
3131
|
+
const first = ctx.requireManager(indexName);
|
|
3132
|
+
await first.publication.write(() => first.beginBatchRemove());
|
|
3133
|
+
let manager = first;
|
|
2857
3134
|
try {
|
|
2858
3135
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2859
3136
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
3137
|
+
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
3138
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3139
|
+
const removed = [];
|
|
2860
3140
|
const prepared = [];
|
|
2861
3141
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
2862
3142
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2863
3143
|
try {
|
|
2864
|
-
|
|
3144
|
+
if (refused.has(i)) throw refused.get(i);
|
|
3145
|
+
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
2865
3146
|
} catch (err) {
|
|
2866
3147
|
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
2867
3148
|
}
|
|
@@ -2891,23 +3172,33 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
2891
3172
|
continue;
|
|
2892
3173
|
}
|
|
2893
3174
|
removal.releaseVectorHolds?.();
|
|
2894
|
-
if (!removal.buffered)
|
|
3175
|
+
if (!removal.buffered) removed.push(removal.docId);
|
|
2895
3176
|
succeeded.push(removal.docId);
|
|
2896
3177
|
}
|
|
2897
3178
|
} finally {
|
|
2898
3179
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
2899
3180
|
}
|
|
3181
|
+
await settleRemovals(ctx, indexName, removed, hold);
|
|
3182
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
2900
3183
|
if (chunkEnd < docIds.length) {
|
|
2901
3184
|
await new Promise((r) => setTimeout(r, 0));
|
|
2902
3185
|
}
|
|
2903
3186
|
}
|
|
2904
3187
|
} finally {
|
|
2905
|
-
|
|
3188
|
+
const ending = manager;
|
|
3189
|
+
await ending?.publication.write(() => ending.endBatchRemove());
|
|
2906
3190
|
}
|
|
2907
3191
|
ctx.checkHeapPressure(indexName);
|
|
2908
3192
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2909
3193
|
return { succeeded, failed };
|
|
2910
3194
|
}
|
|
3195
|
+
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
3196
|
+
const current = ctx.executor.getManager(indexName);
|
|
3197
|
+
if (current === batchManager) return batchManager;
|
|
3198
|
+
await batchManager?.publication.write(() => batchManager.endBatchRemove());
|
|
3199
|
+
await current?.publication.write(() => current.beginBatchRemove());
|
|
3200
|
+
return current;
|
|
3201
|
+
}
|
|
2911
3202
|
|
|
2912
3203
|
// src/engine/mutations/update.ts
|
|
2913
3204
|
function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
@@ -2941,59 +3232,47 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
2941
3232
|
function asUpdateError(err) {
|
|
2942
3233
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
2943
3234
|
}
|
|
2944
|
-
async function
|
|
3235
|
+
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
3236
|
+
for (const update of updates) {
|
|
3237
|
+
try {
|
|
3238
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
3239
|
+
indexName,
|
|
3240
|
+
docId: update.docId,
|
|
3241
|
+
oldDocument: update.oldDocument ?? {},
|
|
3242
|
+
newDocument: update.document
|
|
3243
|
+
});
|
|
3244
|
+
} catch (err) {
|
|
3245
|
+
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
3246
|
+
}
|
|
3247
|
+
}
|
|
3248
|
+
}
|
|
3249
|
+
async function settleUpdates(ctx, indexName, updates, hold) {
|
|
3250
|
+
if (updates.length === 0) return;
|
|
2945
3251
|
await ctx.orchestrator.settlePublications(indexName);
|
|
2946
|
-
|
|
2947
|
-
await
|
|
2948
|
-
indexName,
|
|
2949
|
-
docId: prepared.docId,
|
|
2950
|
-
oldDocument: prepared.oldDocument ?? {},
|
|
2951
|
-
newDocument: prepared.document
|
|
2952
|
-
});
|
|
2953
|
-
} catch (err) {
|
|
2954
|
-
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
3252
|
+
if (ctx.pluginRegistry.hasHooks("afterUpdate")) {
|
|
3253
|
+
await runOutside(hold, () => runAfterUpdateHooks(ctx, indexName, updates));
|
|
2955
3254
|
}
|
|
2956
|
-
const vecIndexes = ctx.
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
3255
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
3256
|
+
if (vecIndexes === void 0) return;
|
|
3257
|
+
for (const update of updates) {
|
|
3258
|
+
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
3259
|
+
if (vec !== null) vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
3260
|
+
}
|
|
2960
3261
|
}
|
|
2961
3262
|
}
|
|
2962
|
-
|
|
3263
|
+
function updatePreparationOf(ctx, indexName, docId, document) {
|
|
2963
3264
|
ctx.guardShutdown();
|
|
2964
3265
|
validateDocId(docId);
|
|
2965
3266
|
requireMatchingDocId(docId, document);
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
newDocument: document
|
|
2973
|
-
});
|
|
2974
|
-
}
|
|
2975
|
-
if (entry.config.required && entry.config.required.length > 0) {
|
|
2976
|
-
validateRequiredFields(document, entry.config.required);
|
|
2977
|
-
}
|
|
2978
|
-
if (entry.config.embedding) {
|
|
2979
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
2980
|
-
if (entry.embeddingAdapter) {
|
|
2981
|
-
await embedDocumentFields(
|
|
2982
|
-
document,
|
|
2983
|
-
entry.config.embedding,
|
|
2984
|
-
entry.embeddingAdapter,
|
|
2985
|
-
ctx.abortController.signal
|
|
2986
|
-
);
|
|
2987
|
-
} else {
|
|
2988
|
-
assertDocumentCarriesMappedVectors(
|
|
2989
|
-
document,
|
|
2990
|
-
entry.config.embedding,
|
|
2991
|
-
entry.embeddingAdapterName
|
|
2992
|
-
);
|
|
2993
|
-
}
|
|
2994
|
-
}
|
|
3267
|
+
if (!ctx.pluginRegistry.hasHooks("beforeUpdate")) return { document, beforeWrite: null, failure: void 0 };
|
|
3268
|
+
const oldDocument = ctx.requireManager(indexName).get(docId) ?? {};
|
|
3269
|
+
const beforeWrite = () => ctx.pluginRegistry.runHook("beforeUpdate", { indexName, docId, oldDocument, newDocument: document });
|
|
3270
|
+
return { document, beforeWrite, failure: void 0 };
|
|
3271
|
+
}
|
|
3272
|
+
async function finishUpdate(ctx, indexName, entry, docId, document) {
|
|
2995
3273
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
2996
3274
|
if (sizing !== null) await sizing;
|
|
3275
|
+
const manager = ctx.requireManager(indexName);
|
|
2997
3276
|
const vecIndexes = manager.getVectorIndexes();
|
|
2998
3277
|
const extractedVectors = /* @__PURE__ */ new Map();
|
|
2999
3278
|
if (vecIndexes.size > 0) {
|
|
@@ -3087,10 +3366,13 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
3087
3366
|
};
|
|
3088
3367
|
return () => manager.publication.write(indexChangeOutsideExternalWaits(updateInIndex));
|
|
3089
3368
|
}
|
|
3090
|
-
async function updateDocument(ctx, indexName, docId, document, options) {
|
|
3369
|
+
async function updateDocument(ctx, indexName, docId, document, options, hold) {
|
|
3091
3370
|
ctx.guardShutdown();
|
|
3092
|
-
|
|
3093
|
-
const
|
|
3371
|
+
ctx.requireIndex(indexName);
|
|
3372
|
+
const preparation = updatePreparationOf(ctx, indexName, docId, document);
|
|
3373
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
3374
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
3375
|
+
const prepared = await finishUpdate(ctx, indexName, entry, docId, document);
|
|
3094
3376
|
const apply = applyOfUpdate(ctx, indexName, prepared);
|
|
3095
3377
|
if (ctx.durability) {
|
|
3096
3378
|
try {
|
|
@@ -3113,10 +3395,12 @@ async function updateDocument(ctx, indexName, docId, document, options) {
|
|
|
3113
3395
|
} else {
|
|
3114
3396
|
await apply();
|
|
3115
3397
|
}
|
|
3116
|
-
if (!prepared.buffered) await
|
|
3398
|
+
if (!prepared.buffered) await settleUpdates(ctx, indexName, [prepared], hold);
|
|
3117
3399
|
ctx.checkHeapPressure(indexName);
|
|
3118
3400
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3119
3401
|
}
|
|
3402
|
+
|
|
3403
|
+
// src/engine/mutations/update-batch.ts
|
|
3120
3404
|
function reportedDocIdOf(update) {
|
|
3121
3405
|
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
3122
3406
|
return typeof docId === "string" ? docId : "";
|
|
@@ -3133,29 +3417,47 @@ function requireUpdateEntry(update) {
|
|
|
3133
3417
|
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
3134
3418
|
);
|
|
3135
3419
|
}
|
|
3136
|
-
async function
|
|
3420
|
+
async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed) {
|
|
3421
|
+
const preparations = [];
|
|
3422
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3423
|
+
const update = updates[i];
|
|
3424
|
+
const docId = reportedDocIdOf(update);
|
|
3425
|
+
try {
|
|
3426
|
+
const { document } = requireUpdateEntry(update);
|
|
3427
|
+
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
3428
|
+
} catch (err) {
|
|
3429
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
3430
|
+
}
|
|
3431
|
+
}
|
|
3432
|
+
const { entry } = await prepareDocumentWrites(
|
|
3433
|
+
ctx,
|
|
3434
|
+
indexName,
|
|
3435
|
+
preparations.map((pending) => pending.preparation),
|
|
3436
|
+
hold,
|
|
3437
|
+
false
|
|
3438
|
+
);
|
|
3439
|
+
const prepared = [];
|
|
3440
|
+
for (const { docId, preparation } of preparations) {
|
|
3441
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3442
|
+
try {
|
|
3443
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
3444
|
+
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
3445
|
+
} catch (err) {
|
|
3446
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
3447
|
+
}
|
|
3448
|
+
}
|
|
3449
|
+
return prepared;
|
|
3450
|
+
}
|
|
3451
|
+
async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
3137
3452
|
ctx.guardShutdown();
|
|
3138
|
-
|
|
3453
|
+
ctx.requireIndex(indexName);
|
|
3139
3454
|
validateBatchList(updates, "updates");
|
|
3140
3455
|
const succeeded = [];
|
|
3141
3456
|
const failed = [];
|
|
3142
|
-
const updateBatchManager = ctx.requireManager(indexName);
|
|
3143
|
-
const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
|
|
3144
3457
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
3145
3458
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
3146
3459
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
3147
|
-
const prepared =
|
|
3148
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3149
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3150
|
-
const update = updates[i];
|
|
3151
|
-
const docId = reportedDocIdOf(update);
|
|
3152
|
-
try {
|
|
3153
|
-
const { document } = requireUpdateEntry(update);
|
|
3154
|
-
prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document));
|
|
3155
|
-
} catch (err) {
|
|
3156
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
3157
|
-
}
|
|
3158
|
-
}
|
|
3460
|
+
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
3159
3461
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
3160
3462
|
let failures;
|
|
3161
3463
|
try {
|
|
@@ -3182,6 +3484,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
3182
3484
|
} finally {
|
|
3183
3485
|
for (const update of prepared) update.releaseVectorHolds?.();
|
|
3184
3486
|
}
|
|
3487
|
+
const settled = [];
|
|
3185
3488
|
for (let i = 0; i < prepared.length; i++) {
|
|
3186
3489
|
const update = prepared[i];
|
|
3187
3490
|
const failure = failures[i];
|
|
@@ -3189,19 +3492,21 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
3189
3492
|
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
3190
3493
|
continue;
|
|
3191
3494
|
}
|
|
3192
|
-
if (!update.buffered)
|
|
3495
|
+
if (!update.buffered) settled.push(update);
|
|
3193
3496
|
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
3194
3497
|
if (vec !== null) touchedVectorFields.add(fieldPath);
|
|
3195
3498
|
}
|
|
3196
3499
|
succeeded.push(update.docId);
|
|
3197
3500
|
}
|
|
3501
|
+
await settleUpdates(ctx, indexName, settled, hold);
|
|
3198
3502
|
if (chunkEnd < updates.length) {
|
|
3199
3503
|
ctx.checkHeapPressure(indexName);
|
|
3200
3504
|
await new Promise((r) => setTimeout(r, 0));
|
|
3201
3505
|
}
|
|
3202
3506
|
}
|
|
3507
|
+
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
3203
3508
|
for (const fieldPath of touchedVectorFields) {
|
|
3204
|
-
|
|
3509
|
+
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
3205
3510
|
}
|
|
3206
3511
|
ctx.checkHeapPressure(indexName);
|
|
3207
3512
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
@@ -3247,4 +3552,4 @@ async function updateEnginePartitionConfig(core, indexName, partitionConfig) {
|
|
|
3247
3552
|
core.watermarkNotifier.check(indexName);
|
|
3248
3553
|
}
|
|
3249
3554
|
|
|
3250
|
-
export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
|
|
3555
|
+
export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, prepareDocumentWrite, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
|