@delali/narsil 0.3.1-next.181 → 0.3.1-next.184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-CPSHS46N.mjs} +139 -177
- package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
- package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
- package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-ZTBHYQMN.mjs} +621 -314
- package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
- package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
- package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
- package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-4XMCITJ6.mjs} +886 -259
- package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
- package/dist/{chunk-KHJPMZE2.mjs → chunk-2KCCQK4R.mjs} +1 -1
- package/dist/{chunk-2KWTQ2P7.mjs → chunk-2YWWHMVC.mjs} +2 -2
- package/dist/{chunk-Y27E2MGC.mjs → chunk-3D4IMFSE.mjs} +2 -2
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/{chunk-XFOL77WH.mjs → chunk-4JBTALUC.mjs} +1 -1
- package/dist/{chunk-ISJ35523.mjs → chunk-5BUEBIXR.mjs} +2 -2
- package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
- package/dist/{chunk-UISV46WX.mjs → chunk-7WXGUTWK.mjs} +3 -3
- package/dist/{chunk-QFJMRB7D.mjs → chunk-A3OAFUZI.mjs} +1 -1
- package/dist/{chunk-GO6TRSJ6.mjs → chunk-BVZWWMOU.mjs} +1 -1
- package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
- package/dist/{chunk-XQQKETAF.mjs → chunk-CODGHJVW.mjs} +4 -4
- package/dist/{chunk-34IOOYXA.mjs → chunk-CYCVPTTR.mjs} +2 -2
- package/dist/{chunk-26E5MO3Y.mjs → chunk-DU23J5JG.mjs} +5 -5
- package/dist/{chunk-JCWC75NH.mjs → chunk-EDEE7ETQ.mjs} +1 -1
- package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
- package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
- package/dist/{chunk-BIIROQ5Q.mjs → chunk-FCQMZDRI.mjs} +2 -2
- package/dist/{chunk-YXZDTKQS.mjs → chunk-G3XRHAJA.mjs} +2 -2
- package/dist/{chunk-XTSFRWAS.mjs → chunk-GGIW6INA.mjs} +7 -6
- package/dist/{chunk-R7PLSVK3.mjs → chunk-H4QYQ5DE.mjs} +1 -1
- package/dist/{chunk-A4XYDUVN.mjs → chunk-HQPK73EW.mjs} +3 -3
- package/dist/{chunk-YPV2L7M5.mjs → chunk-II6XF6M4.mjs} +5 -5
- package/dist/{chunk-F3FDHXCJ.mjs → chunk-LR6JJVML.mjs} +251 -50
- package/dist/{chunk-ZB5X5WAK.mjs → chunk-M5I64AQW.mjs} +11 -6
- package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
- package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
- package/dist/{chunk-XITVDRRH.mjs → chunk-O337MDRJ.mjs} +6 -6
- package/dist/{chunk-XNFBYAXI.mjs → chunk-SEM3PBRH.mjs} +1 -1
- package/dist/{chunk-2C3BXZGA.mjs → chunk-SL53DHVE.mjs} +814 -539
- package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
- package/dist/{chunk-LOQHA6QK.mjs → chunk-URVH6BLO.mjs} +2 -2
- package/dist/{chunk-JDIATDPY.mjs → chunk-VYSWVQCE.mjs} +1 -1
- package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
- package/dist/{chunk-7FC2ZILY.mjs → chunk-WSIU6YXQ.mjs} +67 -78
- package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
- package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
- package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
- package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
- package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-37IH3ISS.mjs} +5 -4
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +398 -144
- package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
- package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
- package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +5 -4
- package/dist/index.react-native.mjs +4 -3
- package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-PU2XT527.mjs} +15 -14
- package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
- package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-YIVJMQ4D.mjs} +9 -6
- package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +5 -4
- package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -16
- package/dist/{query-actions-2JOLPELC.mjs → query-actions-7EOKT3WD.mjs} +5 -4
- package/dist/query-suggestions/merge-worker.mjs +8 -7
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +6 -5
- package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-7TPO4FLE.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-CBU4M3EU.mjs} +813 -538
- package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-IEY4DARA.mjs} +1 -1
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
- package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-T5HF3FW3.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-ZQPSXN6C.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-ZUFOYZ5P.mjs} +1 -1
- package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
- package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
- package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
- package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
- package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-QETIM6BI.mjs} +7 -4
- package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
- package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
- package/dist/{runtime-4GKUIAKV.mjs → react-native/chunks/runtime-UOHD2XUJ.mjs} +23 -3
- package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
- package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
- package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-LNOLKWZB.mjs} +895 -267
- package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
- package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
- package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
- package/dist/react-native/partial-words.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +4 -3
- package/dist/react-native/sync.mjs +4 -3
- package/dist/react-native/synonyms.mjs +4 -3
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-E2HFKKGJ.mjs → rerank-KTV5L7MZ.mjs} +5 -4
- package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
- package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
- package/dist/{react-native/chunks/runtime-BUCSAK4U.mjs → runtime-USQYKN3I.mjs} +23 -3
- package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
- package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +23 -22
- package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-ZI3H2WJY.mjs} +595 -164
- package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
- package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +11 -10
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +5 -4
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +5 -4
- package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
- package/dist/{thread-OWXVP2ZA.mjs → thread-BJXE77AI.mjs} +14 -13
- package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
- package/dist/workers/entry.mjs +11 -10
- package/openapi.json +32 -0
- package/package.json +7 -7
- package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
- package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
|
@@ -1628,19 +1628,17 @@ function removeDocument(input, generation, ordinal, fields) {
|
|
|
1628
1628
|
// src/engine/optional-writes-host.browser.ts
|
|
1629
1629
|
var BATCH_CHUNK_SIZE;
|
|
1630
1630
|
var ErrorCodes;
|
|
1631
|
+
var MAX_CALLS_ACROSS_INDEX_CHANGES;
|
|
1631
1632
|
var MIN_DOCUMENTS_FOR_SEGMENTS;
|
|
1632
1633
|
var NarsilError;
|
|
1633
1634
|
var admitBufferedUpsert;
|
|
1634
1635
|
var admitInsert;
|
|
1635
1636
|
var arrayPositionsAreStale;
|
|
1636
|
-
var asBatchInsertError;
|
|
1637
1637
|
var awaitWriteVisibility;
|
|
1638
|
+
var capsEachPartition;
|
|
1638
1639
|
var changeIndexOutsideExternalWaits;
|
|
1639
|
-
var collectRequiredFieldFailures;
|
|
1640
1640
|
var createFrozenSegment;
|
|
1641
|
-
var createPendingAdmissions;
|
|
1642
1641
|
var deleteNestedValue;
|
|
1643
|
-
var embedChunkDocuments;
|
|
1644
1642
|
var emitSyncChange;
|
|
1645
1643
|
var extractVectorFromDoc;
|
|
1646
1644
|
var fnv1a;
|
|
@@ -1662,7 +1660,6 @@ var prepareDocumentVectors;
|
|
|
1662
1660
|
var prepareStoredSynonyms;
|
|
1663
1661
|
var providedDocId;
|
|
1664
1662
|
var readHeapStatistics;
|
|
1665
|
-
var recordPendingAdmission;
|
|
1666
1663
|
var removeDocumentVectors;
|
|
1667
1664
|
var requireDocumentObject;
|
|
1668
1665
|
var requireDocumentWithinSizeLimit;
|
|
@@ -1703,19 +1700,17 @@ function bindOptionalWritesHost(host, segmentCodec) {
|
|
|
1703
1700
|
({
|
|
1704
1701
|
BATCH_CHUNK_SIZE,
|
|
1705
1702
|
ErrorCodes,
|
|
1703
|
+
MAX_CALLS_ACROSS_INDEX_CHANGES,
|
|
1706
1704
|
MIN_DOCUMENTS_FOR_SEGMENTS,
|
|
1707
1705
|
NarsilError,
|
|
1708
1706
|
admitBufferedUpsert,
|
|
1709
1707
|
admitInsert,
|
|
1710
1708
|
arrayPositionsAreStale,
|
|
1711
|
-
asBatchInsertError,
|
|
1712
1709
|
awaitWriteVisibility,
|
|
1710
|
+
capsEachPartition,
|
|
1713
1711
|
changeIndexOutsideExternalWaits,
|
|
1714
|
-
collectRequiredFieldFailures,
|
|
1715
1712
|
createFrozenSegment,
|
|
1716
|
-
createPendingAdmissions,
|
|
1717
1713
|
deleteNestedValue,
|
|
1718
|
-
embedChunkDocuments,
|
|
1719
1714
|
emitSyncChange,
|
|
1720
1715
|
extractVectorFromDoc,
|
|
1721
1716
|
fnv1a,
|
|
@@ -1737,7 +1732,6 @@ function bindOptionalWritesHost(host, segmentCodec) {
|
|
|
1737
1732
|
prepareStoredSynonyms,
|
|
1738
1733
|
providedDocId,
|
|
1739
1734
|
readHeapStatistics,
|
|
1740
|
-
recordPendingAdmission,
|
|
1741
1735
|
removeDocumentVectors,
|
|
1742
1736
|
requireDocumentObject,
|
|
1743
1737
|
requireDocumentWithinSizeLimit,
|
|
@@ -1963,6 +1957,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1963
1957
|
throw recoveryFailure(metadata, err);
|
|
1964
1958
|
}
|
|
1965
1959
|
deps.indexRegistry.set(metadata.indexName, {
|
|
1960
|
+
storedSizes: deps.indexRegistry.get(metadata.indexName)?.storedSizes,
|
|
1966
1961
|
config: indexConfig,
|
|
1967
1962
|
language,
|
|
1968
1963
|
embeddingAdapter: embedding.adapter,
|
|
@@ -1971,6 +1966,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1971
1966
|
indexUuid: metadata.indexUuid ?? null,
|
|
1972
1967
|
heldPartitions: metadata.heldPartitions ?? null,
|
|
1973
1968
|
documentCount: metadata.documentCount ?? 0,
|
|
1969
|
+
savedBytes: metadata.savedBytes,
|
|
1974
1970
|
partitionCount,
|
|
1975
1971
|
analysisRevision: metadata.analysisRevision ?? null,
|
|
1976
1972
|
arrayPositionGap: metadata.arrayPositionGap ?? null,
|
|
@@ -1993,6 +1989,326 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1993
1989
|
}
|
|
1994
1990
|
}
|
|
1995
1991
|
|
|
1992
|
+
// src/engine/mutations/batch-admissions.ts
|
|
1993
|
+
function createPendingAdmissions() {
|
|
1994
|
+
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
1995
|
+
}
|
|
1996
|
+
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
1997
|
+
pending.total += 1;
|
|
1998
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
1999
|
+
const partitionId = manager.routePartition(docId);
|
|
2000
|
+
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
2001
|
+
}
|
|
2002
|
+
function asBatchInsertError(err) {
|
|
2003
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
2004
|
+
}
|
|
2005
|
+
|
|
2006
|
+
// src/engine/mutations/outside-hold.ts
|
|
2007
|
+
function runOutside(hold, work) {
|
|
2008
|
+
return hold === void 0 ? work() : hold.outside(work);
|
|
2009
|
+
}
|
|
2010
|
+
|
|
2011
|
+
// src/engine/mutations/write-preparation.ts
|
|
2012
|
+
function settingsOf(entry) {
|
|
2013
|
+
return { required: entry.config.required, embedding: entry.config.embedding, adapter: entry.embeddingAdapter };
|
|
2014
|
+
}
|
|
2015
|
+
function embedsThroughAdapter(entry) {
|
|
2016
|
+
return entry.config.embedding !== void 0 && entry.embeddingAdapter !== null;
|
|
2017
|
+
}
|
|
2018
|
+
function sourcesOf(sources) {
|
|
2019
|
+
return typeof sources === "string" ? [sources] : sources;
|
|
2020
|
+
}
|
|
2021
|
+
function sameMapping(left, right) {
|
|
2022
|
+
if (left === right) return true;
|
|
2023
|
+
const targets = Object.keys(left);
|
|
2024
|
+
if (targets.length !== Object.keys(right).length) return false;
|
|
2025
|
+
for (const target of targets) {
|
|
2026
|
+
if (!Object.hasOwn(right, target)) return false;
|
|
2027
|
+
const leftSources = sourcesOf(left[target]);
|
|
2028
|
+
const rightSources = sourcesOf(right[target]);
|
|
2029
|
+
if (leftSources.length !== rightSources.length) return false;
|
|
2030
|
+
if (leftSources.some((source, position) => source !== rightSources[position])) return false;
|
|
2031
|
+
}
|
|
2032
|
+
return true;
|
|
2033
|
+
}
|
|
2034
|
+
function embeddingFits(entry, embedding) {
|
|
2035
|
+
const embeddingConfig = entry.config.embedding;
|
|
2036
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter === null) return embedding === null;
|
|
2037
|
+
return embedding !== null && embedding.adapter === entry.embeddingAdapter && sameMapping(embedding.mapping, embeddingConfig.fields);
|
|
2038
|
+
}
|
|
2039
|
+
function embeddingOfDocuments(embedding, documents) {
|
|
2040
|
+
if (embedding === null) return null;
|
|
2041
|
+
const fields = /* @__PURE__ */ new Map();
|
|
2042
|
+
for (const document of documents) {
|
|
2043
|
+
const embedded = embedding.fields.get(document);
|
|
2044
|
+
if (embedded !== void 0) fields.set(document, embedded);
|
|
2045
|
+
}
|
|
2046
|
+
return { adapter: embedding.adapter, mapping: embedding.mapping, fields };
|
|
2047
|
+
}
|
|
2048
|
+
function writable(preparations) {
|
|
2049
|
+
return preparations.filter((preparation) => preparation.failure === void 0);
|
|
2050
|
+
}
|
|
2051
|
+
function failAll(preparations, failure) {
|
|
2052
|
+
for (const preparation of writable(preparations)) preparation.failure = failure;
|
|
2053
|
+
}
|
|
2054
|
+
function requireFieldsOf(required, preparations) {
|
|
2055
|
+
if (required === void 0 || required.length === 0) return;
|
|
2056
|
+
for (const preparation of writable(preparations)) {
|
|
2057
|
+
try {
|
|
2058
|
+
validateRequiredFields(preparation.document, required);
|
|
2059
|
+
} catch (err) {
|
|
2060
|
+
preparation.failure = err;
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
async function requireMappedVectors(entry, preparations) {
|
|
2065
|
+
const embeddingConfig = entry.config.embedding;
|
|
2066
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter !== null) return;
|
|
2067
|
+
const { assertDocumentCarriesMappedVectors } = await loadEmbedCode();
|
|
2068
|
+
for (const preparation of writable(preparations)) {
|
|
2069
|
+
try {
|
|
2070
|
+
assertDocumentCarriesMappedVectors(
|
|
2071
|
+
preparation.document,
|
|
2072
|
+
embeddingConfig,
|
|
2073
|
+
entry.embeddingAdapterName
|
|
2074
|
+
);
|
|
2075
|
+
} catch (err) {
|
|
2076
|
+
preparation.failure = err;
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
}
|
|
2080
|
+
async function embedWith(settings, signal, preparations, earlier, together) {
|
|
2081
|
+
requireFieldsOf(settings.required, preparations);
|
|
2082
|
+
const { clearWriteEmbedding, embedDocumentsForWrite, lacksMappedVectors } = await loadEmbedCode();
|
|
2083
|
+
if (earlier !== null) clearWriteEmbedding(earlier);
|
|
2084
|
+
const { embedding: embeddingConfig, adapter } = settings;
|
|
2085
|
+
const embeddable = writable(preparations);
|
|
2086
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
2087
|
+
if (embeddingConfig === void 0 || adapter === null || embeddable.length === 0) {
|
|
2088
|
+
return { embedding: null, called: false, documentFailures };
|
|
2089
|
+
}
|
|
2090
|
+
const fields = /* @__PURE__ */ new Map();
|
|
2091
|
+
const embedding = { adapter, mapping: embeddingConfig.fields, fields };
|
|
2092
|
+
const documents = embeddable.map((preparation) => preparation.document);
|
|
2093
|
+
if (!lacksMappedVectors(documents, embeddingConfig)) return { embedding, called: false, documentFailures };
|
|
2094
|
+
for (let chunkStart = 0; chunkStart < embeddable.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2095
|
+
const chunk = embeddable.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
2096
|
+
const embedded = await embedDocumentsForWrite(
|
|
2097
|
+
chunk.map((preparation) => preparation.document),
|
|
2098
|
+
embeddingConfig,
|
|
2099
|
+
adapter,
|
|
2100
|
+
signal,
|
|
2101
|
+
together
|
|
2102
|
+
);
|
|
2103
|
+
for (const [document, embeddedFields] of embedded.embedding.fields) fields.set(document, embeddedFields);
|
|
2104
|
+
for (const [position, error] of embedded.failed) chunk[position].failure = error;
|
|
2105
|
+
for (const [position, error] of embedded.documentFailures) documentFailures.set(chunk[position], error);
|
|
2106
|
+
}
|
|
2107
|
+
return { embedding, called: true, documentFailures };
|
|
2108
|
+
}
|
|
2109
|
+
function settleDocumentFailures(entry, pass) {
|
|
2110
|
+
if (!embeddingFits(entry, pass.embedding)) return [...pass.documentFailures.keys()];
|
|
2111
|
+
for (const [preparation, failure] of pass.documentFailures) preparation.failure = failure;
|
|
2112
|
+
return [];
|
|
2113
|
+
}
|
|
2114
|
+
async function runBeforeWrites(preparations) {
|
|
2115
|
+
for (const preparation of preparations) {
|
|
2116
|
+
if (preparation.beforeWrite === null || preparation.failure !== void 0) continue;
|
|
2117
|
+
try {
|
|
2118
|
+
await preparation.beforeWrite();
|
|
2119
|
+
} catch (err) {
|
|
2120
|
+
preparation.failure = err;
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
async function embedThenRunHooks(settings, signal, preparations, earlier) {
|
|
2125
|
+
const pass = await embedWith(settings, signal, preparations, earlier, true);
|
|
2126
|
+
await runBeforeWrites(preparations.filter((preparation) => !pass.documentFailures.has(preparation)));
|
|
2127
|
+
return pass;
|
|
2128
|
+
}
|
|
2129
|
+
async function runHooksThenEmbedEach(ctx, indexName, preparations, earlier) {
|
|
2130
|
+
let settings = null;
|
|
2131
|
+
let embedding = null;
|
|
2132
|
+
let called = false;
|
|
2133
|
+
let uncleared = earlier;
|
|
2134
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
2135
|
+
for (const preparation of preparations) {
|
|
2136
|
+
await runBeforeWrites([preparation]);
|
|
2137
|
+
settings ??= settingsOf(ctx.requireIndex(indexName));
|
|
2138
|
+
const pass = await embedWith(settings, ctx.abortController.signal, [preparation], uncleared, false);
|
|
2139
|
+
uncleared = null;
|
|
2140
|
+
called ||= pass.called;
|
|
2141
|
+
for (const [failed, failure] of pass.documentFailures) documentFailures.set(failed, failure);
|
|
2142
|
+
if (pass.embedding === null) continue;
|
|
2143
|
+
if (embedding === null) embedding = pass.embedding;
|
|
2144
|
+
else for (const [document, fields] of pass.embedding.fields) embedding.fields.set(document, fields);
|
|
2145
|
+
}
|
|
2146
|
+
return { embedding, called, documentFailures };
|
|
2147
|
+
}
|
|
2148
|
+
function adapterChangesRefusal(indexName, embeddings) {
|
|
2149
|
+
return new NarsilError(
|
|
2150
|
+
ErrorCodes.WORKER_BUSY,
|
|
2151
|
+
`Changes to index "${indexName}" replaced its embedding adapter after each of ${embeddings} embeddings of this write`,
|
|
2152
|
+
{ indexName, embeddings }
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
async function prepareDocumentWrites(ctx, indexName, preparations, hold, together, earlier = null) {
|
|
2156
|
+
let entry = ctx.requireIndex(indexName);
|
|
2157
|
+
let embedding = earlier;
|
|
2158
|
+
if (writable(preparations).length === 0) return { entry, embedding };
|
|
2159
|
+
const signal = ctx.abortController.signal;
|
|
2160
|
+
let embeddings = 0;
|
|
2161
|
+
let unhooked = [];
|
|
2162
|
+
const hooked = preparations.some((preparation) => preparation.beforeWrite !== null);
|
|
2163
|
+
if (hooked || embedsThroughAdapter(entry) && !embeddingFits(entry, embedding)) {
|
|
2164
|
+
if (together && !embedsThroughAdapter(entry)) {
|
|
2165
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
2166
|
+
await requireMappedVectors(entry, preparations);
|
|
2167
|
+
}
|
|
2168
|
+
const settings = settingsOf(entry);
|
|
2169
|
+
const uncleared = embedding;
|
|
2170
|
+
const pass = await runOutside(
|
|
2171
|
+
hold,
|
|
2172
|
+
() => together ? embedThenRunHooks(settings, signal, preparations, uncleared) : runHooksThenEmbedEach(ctx, indexName, preparations, uncleared)
|
|
2173
|
+
);
|
|
2174
|
+
embedding = pass.embedding;
|
|
2175
|
+
if (pass.called) embeddings += 1;
|
|
2176
|
+
entry = ctx.requireIndex(indexName);
|
|
2177
|
+
unhooked = settleDocumentFailures(entry, pass);
|
|
2178
|
+
}
|
|
2179
|
+
while (writable(preparations).length > 0 && !embeddingFits(entry, embedding)) {
|
|
2180
|
+
if (embeddings >= MAX_CALLS_ACROSS_INDEX_CHANGES) {
|
|
2181
|
+
failAll(preparations, adapterChangesRefusal(indexName, embeddings));
|
|
2182
|
+
break;
|
|
2183
|
+
}
|
|
2184
|
+
const uncleared = embedding;
|
|
2185
|
+
const settings = settingsOf(entry);
|
|
2186
|
+
const hooking = unhooked;
|
|
2187
|
+
const pass = await runOutside(hold, async () => {
|
|
2188
|
+
const embedded = await embedWith(settings, signal, preparations, uncleared, together);
|
|
2189
|
+
await runBeforeWrites(hooking.filter((preparation) => !embedded.documentFailures.has(preparation)));
|
|
2190
|
+
return embedded;
|
|
2191
|
+
});
|
|
2192
|
+
embedding = pass.embedding;
|
|
2193
|
+
if (pass.called) embeddings += 1;
|
|
2194
|
+
entry = ctx.requireIndex(indexName);
|
|
2195
|
+
const stale = settleDocumentFailures(entry, pass);
|
|
2196
|
+
unhooked = hooking.filter((preparation) => stale.includes(preparation));
|
|
2197
|
+
}
|
|
2198
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
2199
|
+
await requireMappedVectors(entry, preparations);
|
|
2200
|
+
return { entry, embedding };
|
|
2201
|
+
}
|
|
2202
|
+
async function prepareDocumentWrite(ctx, indexName, document, hold, beforeWrite) {
|
|
2203
|
+
const preparation = { document, beforeWrite, failure: void 0 };
|
|
2204
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
2205
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
2206
|
+
return entry;
|
|
2207
|
+
}
|
|
2208
|
+
|
|
2209
|
+
// src/engine/mutations/insert-batch-admission.ts
|
|
2210
|
+
function pendingInsertOf(ctx, indexName, docId, document, hooked) {
|
|
2211
|
+
const pending = {
|
|
2212
|
+
docId,
|
|
2213
|
+
preparation: { document, beforeWrite: null, failure: void 0 },
|
|
2214
|
+
refusedByHook: false
|
|
2215
|
+
};
|
|
2216
|
+
if (hooked && ctx.pluginRegistry.hasHooks("beforeInsert")) {
|
|
2217
|
+
pending.preparation.beforeWrite = async () => {
|
|
2218
|
+
try {
|
|
2219
|
+
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document });
|
|
2220
|
+
} catch (err) {
|
|
2221
|
+
pending.refusedByHook = true;
|
|
2222
|
+
throw err;
|
|
2223
|
+
}
|
|
2224
|
+
};
|
|
2225
|
+
}
|
|
2226
|
+
return pending;
|
|
2227
|
+
}
|
|
2228
|
+
async function prepareInsertDocuments(ctx, indexName, documents, start, end, hold, hooked, failed, earlier = null) {
|
|
2229
|
+
const pending = [];
|
|
2230
|
+
for (let i = start; i < end; i++) {
|
|
2231
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2232
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2233
|
+
const document = documents[i];
|
|
2234
|
+
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
2235
|
+
try {
|
|
2236
|
+
const sizing = requireDocumentObject(document);
|
|
2237
|
+
if (sizing !== null) await sizing;
|
|
2238
|
+
validateDocId(docId);
|
|
2239
|
+
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
2240
|
+
} catch (err) {
|
|
2241
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
const { entry, embedding } = await prepareDocumentWrites(
|
|
2245
|
+
ctx,
|
|
2246
|
+
indexName,
|
|
2247
|
+
pending.map((item) => item.preparation),
|
|
2248
|
+
hold,
|
|
2249
|
+
true,
|
|
2250
|
+
earlier
|
|
2251
|
+
);
|
|
2252
|
+
const candidates = [];
|
|
2253
|
+
for (const item of pending) {
|
|
2254
|
+
const { document, failure } = item.preparation;
|
|
2255
|
+
if (failure === void 0) {
|
|
2256
|
+
candidates.push({ docId: item.docId, document });
|
|
2257
|
+
continue;
|
|
2258
|
+
}
|
|
2259
|
+
const docId = item.refusedByHook ? item.docId : providedDocId(document) ?? "";
|
|
2260
|
+
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
2261
|
+
}
|
|
2262
|
+
return { entry, candidates, embedding };
|
|
2263
|
+
}
|
|
2264
|
+
async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
|
|
2265
|
+
const { entry, candidates, embedding } = await prepareInsertDocuments(
|
|
2266
|
+
ctx,
|
|
2267
|
+
indexName,
|
|
2268
|
+
documents,
|
|
2269
|
+
0,
|
|
2270
|
+
documents.length,
|
|
2271
|
+
hold,
|
|
2272
|
+
true,
|
|
2273
|
+
failed
|
|
2274
|
+
);
|
|
2275
|
+
const manager = ctx.requireManager(indexName);
|
|
2276
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
2277
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2278
|
+
const strict = entry.config.strict === true;
|
|
2279
|
+
const admitted = [];
|
|
2280
|
+
const admittedIds = /* @__PURE__ */ new Set();
|
|
2281
|
+
const pending = createPendingAdmissions();
|
|
2282
|
+
for (let position = 0; position < candidates.length; position++) {
|
|
2283
|
+
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
2284
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2285
|
+
const { docId, document } = candidates[position];
|
|
2286
|
+
try {
|
|
2287
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2288
|
+
document,
|
|
2289
|
+
vectorFieldPaths
|
|
2290
|
+
);
|
|
2291
|
+
if (extractedVectors.size > 0) {
|
|
2292
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
2293
|
+
}
|
|
2294
|
+
admitInsert(ctx, indexName, manager, docId, pending);
|
|
2295
|
+
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
2296
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
2297
|
+
}
|
|
2298
|
+
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
2299
|
+
if (strict) {
|
|
2300
|
+
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
2301
|
+
}
|
|
2302
|
+
admittedIds.add(docId);
|
|
2303
|
+
recordPendingAdmission(ctx, pending, manager, docId);
|
|
2304
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2305
|
+
} catch (err) {
|
|
2306
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
return { admitted, embedding };
|
|
2310
|
+
}
|
|
2311
|
+
|
|
1996
2312
|
// src/engine/mutations/durable-rollback.ts
|
|
1997
2313
|
function holdVectorsForRollback(indexes, docId) {
|
|
1998
2314
|
const releases = [];
|
|
@@ -2228,84 +2544,29 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
|
2228
2544
|
return applications;
|
|
2229
2545
|
}
|
|
2230
2546
|
|
|
2231
|
-
// src/engine/mutations/insert-batch-
|
|
2232
|
-
async function
|
|
2233
|
-
|
|
2234
|
-
const manager = ctx.
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
2238
|
-
const strict = entry.config.strict === true;
|
|
2239
|
-
const required = entry.config.required;
|
|
2240
|
-
const admitted = [];
|
|
2241
|
-
const admittedIds = /* @__PURE__ */ new Set();
|
|
2242
|
-
const pending = createPendingAdmissions();
|
|
2243
|
-
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2244
|
-
if (ctx.abortController.signal.aborted) break;
|
|
2245
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2246
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
2247
|
-
if (required && required.length > 0) {
|
|
2248
|
-
collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
2249
|
-
}
|
|
2250
|
-
await embedChunkDocuments(
|
|
2251
|
-
entry,
|
|
2252
|
-
documents,
|
|
2253
|
-
chunkStart,
|
|
2254
|
-
chunkEnd,
|
|
2255
|
-
ctx.abortController.signal,
|
|
2256
|
-
chunkFailedIndexes,
|
|
2257
|
-
failed
|
|
2258
|
-
);
|
|
2259
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
2260
|
-
if (ctx.abortController.signal.aborted) break;
|
|
2261
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
2262
|
-
const docId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
2547
|
+
// src/engine/mutations/insert-batch-documents.ts
|
|
2548
|
+
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
2549
|
+
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
2550
|
+
const manager = ctx.executor.getManager(indexName);
|
|
2551
|
+
await runOutside(hold, async () => {
|
|
2552
|
+
for (const doc of inserted) {
|
|
2263
2553
|
try {
|
|
2264
|
-
|
|
2265
|
-
if (sizing !== null) await sizing;
|
|
2266
|
-
validateDocId(docId);
|
|
2267
|
-
if (hasBeforeHook) {
|
|
2268
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
|
|
2269
|
-
}
|
|
2270
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2271
|
-
documents[i],
|
|
2272
|
-
vectorFieldPaths
|
|
2273
|
-
);
|
|
2274
|
-
if (extractedVectors.size > 0) {
|
|
2275
|
-
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
2276
|
-
}
|
|
2277
|
-
admitInsert(ctx, indexName, manager, docId, pending);
|
|
2278
|
-
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
2279
|
-
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
2280
|
-
}
|
|
2281
|
-
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
2282
|
-
if (strict) {
|
|
2283
|
-
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
2284
|
-
}
|
|
2285
|
-
admittedIds.add(docId);
|
|
2286
|
-
recordPendingAdmission(ctx, pending, manager, docId);
|
|
2287
|
-
admitted.push({ docId, document: documents[i], partitionDoc, extractedVectors });
|
|
2554
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2288
2555
|
} catch (err) {
|
|
2289
|
-
|
|
2556
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2290
2557
|
}
|
|
2291
2558
|
}
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
}
|
|
2295
|
-
}
|
|
2296
|
-
return admitted;
|
|
2559
|
+
});
|
|
2560
|
+
return ctx.executor.getManager(indexName) !== manager;
|
|
2297
2561
|
}
|
|
2298
|
-
|
|
2299
|
-
// src/engine/mutations/insert-batch-documents.ts
|
|
2300
|
-
async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed) {
|
|
2301
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
2562
|
+
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
2302
2563
|
const succeeded = [];
|
|
2303
|
-
const buffered = /* @__PURE__ */ new Set();
|
|
2304
2564
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2305
2565
|
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2306
2566
|
if (ctx.abortController.signal.aborted) break;
|
|
2307
2567
|
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
2308
2568
|
const applications = await applyInsertChunk(ctx, indexName, chunk, options);
|
|
2569
|
+
const inserted = [];
|
|
2309
2570
|
for (let i = 0; i < chunk.length; i++) {
|
|
2310
2571
|
const doc = chunk[i];
|
|
2311
2572
|
const application = applications[i];
|
|
@@ -2314,30 +2575,103 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
|
|
|
2314
2575
|
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2315
2576
|
continue;
|
|
2316
2577
|
}
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
succeeded.push(doc.docId);
|
|
2320
|
-
continue;
|
|
2321
|
-
}
|
|
2578
|
+
succeeded.push(doc.docId);
|
|
2579
|
+
if (application.status === "buffered") continue;
|
|
2322
2580
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2323
2581
|
touchedVectorFields.add(fieldPath);
|
|
2324
2582
|
}
|
|
2325
|
-
|
|
2326
|
-
try {
|
|
2327
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2328
|
-
} catch (err) {
|
|
2329
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2330
|
-
}
|
|
2331
|
-
}
|
|
2332
|
-
succeeded.push(doc.docId);
|
|
2583
|
+
inserted.push(doc);
|
|
2333
2584
|
}
|
|
2334
2585
|
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
2586
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
2587
|
+
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
2588
|
+
}
|
|
2335
2589
|
if (applied < admitted.length) {
|
|
2336
2590
|
ctx.checkHeapPressure(indexName);
|
|
2337
2591
|
await new Promise((r) => setTimeout(r, 0));
|
|
2338
2592
|
}
|
|
2339
2593
|
}
|
|
2340
|
-
return { succeeded,
|
|
2594
|
+
return { succeeded, touchedVectorFields, unrecorded: [] };
|
|
2595
|
+
}
|
|
2596
|
+
|
|
2597
|
+
// src/engine/mutations/insert-batch-chunks.ts
|
|
2598
|
+
async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
2599
|
+
const { entry, candidates } = await prepareInsertDocuments(
|
|
2600
|
+
ctx,
|
|
2601
|
+
indexName,
|
|
2602
|
+
documents,
|
|
2603
|
+
chunkStart,
|
|
2604
|
+
chunkEnd,
|
|
2605
|
+
hold,
|
|
2606
|
+
hooked,
|
|
2607
|
+
failed,
|
|
2608
|
+
earlier
|
|
2609
|
+
);
|
|
2610
|
+
const manager = ctx.requireManager(indexName);
|
|
2611
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
2612
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2613
|
+
const admitted = [];
|
|
2614
|
+
for (const { docId, document } of candidates) {
|
|
2615
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2616
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2617
|
+
try {
|
|
2618
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2619
|
+
document,
|
|
2620
|
+
vectorFieldPaths
|
|
2621
|
+
);
|
|
2622
|
+
if (extractedVectors.size > 0) {
|
|
2623
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
2624
|
+
}
|
|
2625
|
+
admitInsert(ctx, indexName, manager, docId);
|
|
2626
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2627
|
+
} catch (err) {
|
|
2628
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2629
|
+
}
|
|
2630
|
+
}
|
|
2631
|
+
return admitted;
|
|
2632
|
+
}
|
|
2633
|
+
async function insertInChunks(ctx, indexName, documents, options, hold, hooked, failed, embedded = null) {
|
|
2634
|
+
const succeeded = [];
|
|
2635
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2636
|
+
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2637
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2638
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2639
|
+
const earlier = embeddingOfDocuments(embedded, documents.slice(chunkStart, chunkEnd));
|
|
2640
|
+
const prepared = await admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
2641
|
+
if (prepared.length > 0) {
|
|
2642
|
+
const applications = await applyInsertChunk(ctx, indexName, prepared, options);
|
|
2643
|
+
const inserted = [];
|
|
2644
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
2645
|
+
const doc = prepared[i];
|
|
2646
|
+
const application = applications[i];
|
|
2647
|
+
if (application.status === "skipped") continue;
|
|
2648
|
+
if (application.status === "failed") {
|
|
2649
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2650
|
+
continue;
|
|
2651
|
+
}
|
|
2652
|
+
succeeded.push(doc.docId);
|
|
2653
|
+
if (application.status === "buffered") continue;
|
|
2654
|
+
for (const fieldPath of doc.extractedVectors.keys()) touchedVectorFields.add(fieldPath);
|
|
2655
|
+
inserted.push(doc);
|
|
2656
|
+
}
|
|
2657
|
+
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
2658
|
+
}
|
|
2659
|
+
if (chunkEnd < documents.length) {
|
|
2660
|
+
ctx.checkHeapPressure(indexName);
|
|
2661
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2662
|
+
}
|
|
2663
|
+
}
|
|
2664
|
+
return { succeeded, touchedVectorFields };
|
|
2665
|
+
}
|
|
2666
|
+
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
2667
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
2668
|
+
for (const fieldPath of inserts.touchedVectorFields) {
|
|
2669
|
+
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
2670
|
+
}
|
|
2671
|
+
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
2672
|
+
ctx.checkHeapPressure(indexName);
|
|
2673
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
2674
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2341
2675
|
}
|
|
2342
2676
|
|
|
2343
2677
|
// src/engine/mutations/segment-publication.ts
|
|
@@ -2442,10 +2776,6 @@ function mainStoreDocument(doc, options) {
|
|
|
2442
2776
|
if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
|
|
2443
2777
|
return structuredClone(doc.partitionDoc);
|
|
2444
2778
|
}
|
|
2445
|
-
async function applyIndividually(ctx, indexName, admitted, options, failed) {
|
|
2446
|
-
const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
|
|
2447
|
-
return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
|
|
2448
|
-
}
|
|
2449
2779
|
function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted) {
|
|
2450
2780
|
const chunks = [];
|
|
2451
2781
|
let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
@@ -2495,12 +2825,12 @@ function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberI
|
|
|
2495
2825
|
if (chunk.docs.length > 0) chunks.push(chunk);
|
|
2496
2826
|
return chunks;
|
|
2497
2827
|
}
|
|
2498
|
-
async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, failed) {
|
|
2499
|
-
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
2828
|
+
async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, hold, failed) {
|
|
2500
2829
|
const succeededIds = /* @__PURE__ */ new Set();
|
|
2501
2830
|
const errorOf = /* @__PURE__ */ new Map();
|
|
2502
2831
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2503
2832
|
const chunks = chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted);
|
|
2833
|
+
const unrecorded = [];
|
|
2504
2834
|
for (let index = 0; index < chunks.length; index++) {
|
|
2505
2835
|
const { docs, attachmentOf } = chunks[index];
|
|
2506
2836
|
const applies = docs.map((doc) => {
|
|
@@ -2510,6 +2840,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2510
2840
|
});
|
|
2511
2841
|
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2512
2842
|
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
2843
|
+
const inserted = [];
|
|
2513
2844
|
for (let i = 0; i < docs.length; i++) {
|
|
2514
2845
|
const doc = docs[i];
|
|
2515
2846
|
const failure = failures[i];
|
|
@@ -2541,15 +2872,13 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2541
2872
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2542
2873
|
touchedVectorFields.add(fieldPath);
|
|
2543
2874
|
}
|
|
2544
|
-
|
|
2545
|
-
try {
|
|
2546
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2547
|
-
} catch (err) {
|
|
2548
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2549
|
-
}
|
|
2550
|
-
}
|
|
2875
|
+
inserted.push(doc);
|
|
2551
2876
|
succeededIds.add(doc.docId);
|
|
2552
2877
|
}
|
|
2878
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
2879
|
+
for (const later of chunks.slice(index + 1)) unrecorded.push(...later.docs);
|
|
2880
|
+
break;
|
|
2881
|
+
}
|
|
2553
2882
|
if (index + 1 < chunks.length) {
|
|
2554
2883
|
ctx.checkHeapPressure(indexName);
|
|
2555
2884
|
await new Promise((r) => setTimeout(r, 0));
|
|
@@ -2564,7 +2893,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
|
|
|
2564
2893
|
if (!errorOf.has(doc.docId)) continue;
|
|
2565
2894
|
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
2566
2895
|
}
|
|
2567
|
-
return { succeeded, touchedVectorFields };
|
|
2896
|
+
return { succeeded, touchedVectorFields, unrecorded };
|
|
2568
2897
|
}
|
|
2569
2898
|
function frozenSegmentOf(segment, members, options) {
|
|
2570
2899
|
if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
|
|
@@ -2575,14 +2904,15 @@ function frozenSegmentOf(segment, members, options) {
|
|
|
2575
2904
|
segment.segmentId
|
|
2576
2905
|
);
|
|
2577
2906
|
}
|
|
2578
|
-
async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
2907
|
+
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
2579
2908
|
if (admitted.length === 0) {
|
|
2580
|
-
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set() };
|
|
2909
|
+
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] };
|
|
2581
2910
|
}
|
|
2911
|
+
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
2582
2912
|
const manager = ctx.requireManager(indexName);
|
|
2583
2913
|
const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
|
|
2584
2914
|
if (admitted.length < MIN_DOCUMENTS_FOR_SEGMENTS || workers <= 0) {
|
|
2585
|
-
return applyIndividually(
|
|
2915
|
+
return applyIndividually();
|
|
2586
2916
|
}
|
|
2587
2917
|
const docIds = admitted.map((doc) => doc.docId);
|
|
2588
2918
|
const partitionDocuments = admitted.map((doc) => doc.partitionDoc);
|
|
@@ -2595,7 +2925,7 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
2595
2925
|
built = null;
|
|
2596
2926
|
}
|
|
2597
2927
|
if (built === null) {
|
|
2598
|
-
return applyIndividually(
|
|
2928
|
+
return applyIndividually();
|
|
2599
2929
|
}
|
|
2600
2930
|
const positions = new Map(admitted.map((document, index) => [document.docId, index]));
|
|
2601
2931
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -2604,16 +2934,15 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
2604
2934
|
const members = [];
|
|
2605
2935
|
for (const docId of segment.docIds) {
|
|
2606
2936
|
const position = positions.get(docId);
|
|
2607
|
-
if (position === void 0 || used.has(position))
|
|
2608
|
-
return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
2937
|
+
if (position === void 0 || used.has(position)) return applyIndividually();
|
|
2609
2938
|
used.add(position);
|
|
2610
2939
|
members.push(position);
|
|
2611
2940
|
}
|
|
2612
2941
|
memberIndexes.push(members);
|
|
2613
2942
|
}
|
|
2614
|
-
if (used.size !== admitted.length) return applyIndividually(
|
|
2943
|
+
if (used.size !== admitted.length) return applyIndividually();
|
|
2615
2944
|
if (ctx.isRebalancing(indexName)) {
|
|
2616
|
-
return applyIndividually(
|
|
2945
|
+
return applyIndividually();
|
|
2617
2946
|
}
|
|
2618
2947
|
const frozen = [];
|
|
2619
2948
|
for (let i = 0; i < built.length; i++) {
|
|
@@ -2622,149 +2951,65 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
2622
2951
|
memberIndexes[i].map((m) => admitted[m]),
|
|
2623
2952
|
options
|
|
2624
2953
|
);
|
|
2625
|
-
if (segment === null) return applyIndividually(
|
|
2954
|
+
if (segment === null) return applyIndividually();
|
|
2626
2955
|
frozen.push(segment);
|
|
2627
2956
|
}
|
|
2628
2957
|
for (const doc of admitted) {
|
|
2629
|
-
if (manager.has(doc.docId)) return applyIndividually(
|
|
2958
|
+
if (manager.has(doc.docId)) return applyIndividually();
|
|
2630
2959
|
}
|
|
2631
|
-
return recordMergedDocuments(
|
|
2960
|
+
return recordMergedDocuments(
|
|
2961
|
+
ctx,
|
|
2962
|
+
indexName,
|
|
2963
|
+
partitionCount,
|
|
2964
|
+
built,
|
|
2965
|
+
frozen,
|
|
2966
|
+
memberIndexes,
|
|
2967
|
+
admitted,
|
|
2968
|
+
options,
|
|
2969
|
+
hold,
|
|
2970
|
+
failed
|
|
2971
|
+
);
|
|
2632
2972
|
}
|
|
2633
|
-
async function insertBatchViaSegments(ctx, indexName, documents, options) {
|
|
2973
|
+
async function insertBatchViaSegments(ctx, indexName, documents, options, hold) {
|
|
2634
2974
|
const failed = [];
|
|
2635
|
-
const admitted = await admitBatchDocuments(ctx, indexName, documents, failed);
|
|
2636
|
-
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, failed);
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
await ctx
|
|
2644
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2975
|
+
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents, failed, hold);
|
|
2976
|
+
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
2977
|
+
if (outcome.unrecorded.length > 0) {
|
|
2978
|
+
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
2979
|
+
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
2980
|
+
outcome.succeeded.push(...rest.succeeded);
|
|
2981
|
+
for (const fieldPath of rest.touchedVectorFields) outcome.touchedVectorFields.add(fieldPath);
|
|
2982
|
+
}
|
|
2983
|
+
await finishInsertBatch(ctx, indexName, outcome, options);
|
|
2645
2984
|
return { succeeded: outcome.succeeded, failed };
|
|
2646
2985
|
}
|
|
2647
2986
|
|
|
2648
2987
|
// src/engine/mutations/insert-batch.ts
|
|
2649
|
-
async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
2988
|
+
async function insertDocumentBatch(ctx, indexName, documents, options, hold) {
|
|
2650
2989
|
ctx.guardShutdown();
|
|
2651
|
-
|
|
2990
|
+
ctx.requireIndex(indexName);
|
|
2652
2991
|
validateBatchList(documents, "documents");
|
|
2653
2992
|
await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length);
|
|
2654
2993
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
2655
2994
|
if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
2656
|
-
return insertBatchViaSegments(ctx, indexName, documents, options);
|
|
2995
|
+
return insertBatchViaSegments(ctx, indexName, documents, options, hold);
|
|
2657
2996
|
}
|
|
2658
|
-
const succeeded = [];
|
|
2659
2997
|
const failed = [];
|
|
2660
|
-
const
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
const batchManager = ctx.requireManager(indexName);
|
|
2664
|
-
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
2665
|
-
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2666
|
-
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2667
|
-
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2668
|
-
if (ctx.abortController.signal.aborted) break;
|
|
2669
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2670
|
-
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
2671
|
-
if (required && required.length > 0) {
|
|
2672
|
-
collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
2673
|
-
}
|
|
2674
|
-
await embedChunkDocuments(
|
|
2675
|
-
entry,
|
|
2676
|
-
documents,
|
|
2677
|
-
chunkStart,
|
|
2678
|
-
chunkEnd,
|
|
2679
|
-
ctx.abortController.signal,
|
|
2680
|
-
chunkFailedIndexes,
|
|
2681
|
-
failed
|
|
2682
|
-
);
|
|
2683
|
-
async function applyPrepared(prepared2) {
|
|
2684
|
-
const applications = await applyInsertChunk(ctx, indexName, prepared2, options);
|
|
2685
|
-
for (let i = 0; i < prepared2.length; i++) {
|
|
2686
|
-
const doc = prepared2[i];
|
|
2687
|
-
const application = applications[i];
|
|
2688
|
-
if (application.status === "skipped") continue;
|
|
2689
|
-
if (application.status === "failed") {
|
|
2690
|
-
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2691
|
-
continue;
|
|
2692
|
-
}
|
|
2693
|
-
if (application.status === "buffered") {
|
|
2694
|
-
succeeded.push(doc.docId);
|
|
2695
|
-
continue;
|
|
2696
|
-
}
|
|
2697
|
-
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2698
|
-
touchedVectorFields.add(fieldPath);
|
|
2699
|
-
}
|
|
2700
|
-
if (hasAfterHook) {
|
|
2701
|
-
try {
|
|
2702
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2703
|
-
} catch (err) {
|
|
2704
|
-
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2705
|
-
}
|
|
2706
|
-
}
|
|
2707
|
-
succeeded.push(doc.docId);
|
|
2708
|
-
}
|
|
2709
|
-
}
|
|
2710
|
-
const prepared = [];
|
|
2711
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
2712
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2713
|
-
if (ctx.abortController.signal.aborted) break;
|
|
2714
|
-
if (chunkFailedIndexes.has(i)) continue;
|
|
2715
|
-
const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
2716
|
-
try {
|
|
2717
|
-
const sizing = requireDocumentObject(documents[i]);
|
|
2718
|
-
if (sizing !== null) await sizing;
|
|
2719
|
-
validateDocId(batchDocId);
|
|
2720
|
-
if (hasBeforeHook) {
|
|
2721
|
-
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
|
|
2722
|
-
}
|
|
2723
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2724
|
-
documents[i],
|
|
2725
|
-
batchVectorFieldPaths
|
|
2726
|
-
);
|
|
2727
|
-
if (extractedVectors.size > 0) {
|
|
2728
|
-
validateVectorDimensions(extractedVectors, batchVecIndexes);
|
|
2729
|
-
}
|
|
2730
|
-
admitInsert(ctx, indexName, batchManager, batchDocId);
|
|
2731
|
-
prepared.push({
|
|
2732
|
-
docId: batchDocId,
|
|
2733
|
-
document: documents[i],
|
|
2734
|
-
partitionDoc,
|
|
2735
|
-
extractedVectors
|
|
2736
|
-
});
|
|
2737
|
-
} catch (err) {
|
|
2738
|
-
failed.push({ docId: batchDocId, error: asBatchInsertError(err) });
|
|
2739
|
-
}
|
|
2740
|
-
}
|
|
2741
|
-
if (prepared.length > 0) await applyPrepared(prepared);
|
|
2742
|
-
if (chunkEnd < documents.length) {
|
|
2743
|
-
ctx.checkHeapPressure(indexName);
|
|
2744
|
-
await new Promise((r) => setTimeout(r, 0));
|
|
2745
|
-
}
|
|
2746
|
-
}
|
|
2747
|
-
for (const fieldPath of touchedVectorFields) {
|
|
2748
|
-
const vecIndex = batchVecIndexes.get(fieldPath);
|
|
2749
|
-
if (vecIndex) {
|
|
2750
|
-
vecIndex.scheduleBuild();
|
|
2751
|
-
}
|
|
2752
|
-
}
|
|
2753
|
-
ctx.checkWatermark(indexName, succeeded);
|
|
2754
|
-
ctx.checkHeapPressure(indexName);
|
|
2755
|
-
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
2756
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2757
|
-
return { succeeded, failed };
|
|
2998
|
+
const inserts = await insertInChunks(ctx, indexName, documents, options, hold, true, failed);
|
|
2999
|
+
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
3000
|
+
return { succeeded: inserts.succeeded, failed };
|
|
2758
3001
|
}
|
|
2759
3002
|
|
|
2760
3003
|
// src/engine/mutations/remove.ts
|
|
2761
3004
|
function asRemoveError(err) {
|
|
2762
3005
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
2763
3006
|
}
|
|
2764
|
-
|
|
3007
|
+
function validateRemoval(ctx, docId) {
|
|
2765
3008
|
ctx.guardShutdown();
|
|
2766
3009
|
validateDocId(docId);
|
|
2767
|
-
|
|
3010
|
+
}
|
|
3011
|
+
function prepareRemoval(ctx, indexName, docId) {
|
|
3012
|
+
ctx.guardShutdown();
|
|
2768
3013
|
const rebalancing = ctx.isRebalancing(indexName);
|
|
2769
3014
|
if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
|
|
2770
3015
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
@@ -2814,20 +3059,31 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
2814
3059
|
};
|
|
2815
3060
|
return () => manager.publication.writeNow(indexChangeOutsideExternalWaits(removeFromIndex));
|
|
2816
3061
|
}
|
|
2817
|
-
async function
|
|
3062
|
+
async function settleRemovals(ctx, indexName, docIds, hold) {
|
|
3063
|
+
if (docIds.length === 0) return;
|
|
2818
3064
|
const settling = ctx.orchestrator.settlePublications(indexName, "none");
|
|
2819
3065
|
if (settling !== void 0) await settling;
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
3066
|
+
if (!ctx.pluginRegistry.hasHooks("afterRemove")) return;
|
|
3067
|
+
await runOutside(hold, async () => {
|
|
3068
|
+
for (const docId of docIds) {
|
|
3069
|
+
try {
|
|
3070
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
3071
|
+
} catch (err) {
|
|
3072
|
+
console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
3073
|
+
}
|
|
3074
|
+
}
|
|
3075
|
+
});
|
|
2826
3076
|
}
|
|
2827
|
-
async function removeDocument2(ctx, indexName, docId, options) {
|
|
3077
|
+
async function removeDocument2(ctx, indexName, docId, options, hold) {
|
|
2828
3078
|
ctx.guardShutdown();
|
|
2829
3079
|
ctx.requireIndex(indexName);
|
|
2830
|
-
|
|
3080
|
+
validateRemoval(ctx, docId);
|
|
3081
|
+
if (ctx.pluginRegistry.hasHooks("beforeRemove")) {
|
|
3082
|
+
await runOutside(hold, async () => {
|
|
3083
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
|
|
3084
|
+
});
|
|
3085
|
+
}
|
|
3086
|
+
const prepared = prepareRemoval(ctx, indexName, docId);
|
|
2831
3087
|
const apply = applyOfRemoval(ctx, indexName, prepared);
|
|
2832
3088
|
if (ctx.durability) {
|
|
2833
3089
|
try {
|
|
@@ -2841,25 +3097,52 @@ async function removeDocument2(ctx, indexName, docId, options) {
|
|
|
2841
3097
|
} else {
|
|
2842
3098
|
await apply();
|
|
2843
3099
|
}
|
|
2844
|
-
if (!prepared.buffered) await
|
|
3100
|
+
if (!prepared.buffered) await settleRemovals(ctx, indexName, [docId], hold);
|
|
2845
3101
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2846
3102
|
}
|
|
2847
|
-
async function
|
|
3103
|
+
async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold) {
|
|
3104
|
+
const refused = /* @__PURE__ */ new Map();
|
|
3105
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3106
|
+
try {
|
|
3107
|
+
validateRemoval(ctx, docIds[i]);
|
|
3108
|
+
} catch (err) {
|
|
3109
|
+
refused.set(i, err);
|
|
3110
|
+
}
|
|
3111
|
+
}
|
|
3112
|
+
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
3113
|
+
await runOutside(hold, async () => {
|
|
3114
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3115
|
+
if (refused.has(i)) continue;
|
|
3116
|
+
try {
|
|
3117
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
3118
|
+
} catch (err) {
|
|
3119
|
+
refused.set(i, err);
|
|
3120
|
+
}
|
|
3121
|
+
}
|
|
3122
|
+
});
|
|
3123
|
+
return refused;
|
|
3124
|
+
}
|
|
3125
|
+
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
2848
3126
|
ctx.guardShutdown();
|
|
2849
3127
|
ctx.requireIndex(indexName);
|
|
2850
3128
|
validateBatchList(docIds, "document IDs");
|
|
2851
3129
|
const succeeded = [];
|
|
2852
3130
|
const failed = [];
|
|
2853
|
-
const
|
|
2854
|
-
await
|
|
3131
|
+
const first = ctx.requireManager(indexName);
|
|
3132
|
+
await first.publication.write(() => first.beginBatchRemove());
|
|
3133
|
+
let manager = first;
|
|
2855
3134
|
try {
|
|
2856
3135
|
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2857
3136
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
3137
|
+
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
3138
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3139
|
+
const removed = [];
|
|
2858
3140
|
const prepared = [];
|
|
2859
3141
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
2860
3142
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2861
3143
|
try {
|
|
2862
|
-
|
|
3144
|
+
if (refused.has(i)) throw refused.get(i);
|
|
3145
|
+
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
2863
3146
|
} catch (err) {
|
|
2864
3147
|
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
2865
3148
|
}
|
|
@@ -2889,23 +3172,33 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
2889
3172
|
continue;
|
|
2890
3173
|
}
|
|
2891
3174
|
removal.releaseVectorHolds?.();
|
|
2892
|
-
if (!removal.buffered)
|
|
3175
|
+
if (!removal.buffered) removed.push(removal.docId);
|
|
2893
3176
|
succeeded.push(removal.docId);
|
|
2894
3177
|
}
|
|
2895
3178
|
} finally {
|
|
2896
3179
|
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
2897
3180
|
}
|
|
3181
|
+
await settleRemovals(ctx, indexName, removed, hold);
|
|
3182
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
2898
3183
|
if (chunkEnd < docIds.length) {
|
|
2899
3184
|
await new Promise((r) => setTimeout(r, 0));
|
|
2900
3185
|
}
|
|
2901
3186
|
}
|
|
2902
3187
|
} finally {
|
|
2903
|
-
|
|
3188
|
+
const ending = manager;
|
|
3189
|
+
await ending?.publication.write(() => ending.endBatchRemove());
|
|
2904
3190
|
}
|
|
2905
3191
|
ctx.checkHeapPressure(indexName);
|
|
2906
3192
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2907
3193
|
return { succeeded, failed };
|
|
2908
3194
|
}
|
|
3195
|
+
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
3196
|
+
const current = ctx.executor.getManager(indexName);
|
|
3197
|
+
if (current === batchManager) return batchManager;
|
|
3198
|
+
await batchManager?.publication.write(() => batchManager.endBatchRemove());
|
|
3199
|
+
await current?.publication.write(() => current.beginBatchRemove());
|
|
3200
|
+
return current;
|
|
3201
|
+
}
|
|
2909
3202
|
|
|
2910
3203
|
// src/engine/mutations/update.ts
|
|
2911
3204
|
function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
@@ -2939,59 +3232,47 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
2939
3232
|
function asUpdateError(err) {
|
|
2940
3233
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
2941
3234
|
}
|
|
2942
|
-
async function
|
|
3235
|
+
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
3236
|
+
for (const update of updates) {
|
|
3237
|
+
try {
|
|
3238
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
3239
|
+
indexName,
|
|
3240
|
+
docId: update.docId,
|
|
3241
|
+
oldDocument: update.oldDocument ?? {},
|
|
3242
|
+
newDocument: update.document
|
|
3243
|
+
});
|
|
3244
|
+
} catch (err) {
|
|
3245
|
+
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
3246
|
+
}
|
|
3247
|
+
}
|
|
3248
|
+
}
|
|
3249
|
+
async function settleUpdates(ctx, indexName, updates, hold) {
|
|
3250
|
+
if (updates.length === 0) return;
|
|
2943
3251
|
await ctx.orchestrator.settlePublications(indexName);
|
|
2944
|
-
|
|
2945
|
-
await
|
|
2946
|
-
indexName,
|
|
2947
|
-
docId: prepared.docId,
|
|
2948
|
-
oldDocument: prepared.oldDocument ?? {},
|
|
2949
|
-
newDocument: prepared.document
|
|
2950
|
-
});
|
|
2951
|
-
} catch (err) {
|
|
2952
|
-
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
3252
|
+
if (ctx.pluginRegistry.hasHooks("afterUpdate")) {
|
|
3253
|
+
await runOutside(hold, () => runAfterUpdateHooks(ctx, indexName, updates));
|
|
2953
3254
|
}
|
|
2954
|
-
const vecIndexes = ctx.
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
3255
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
3256
|
+
if (vecIndexes === void 0) return;
|
|
3257
|
+
for (const update of updates) {
|
|
3258
|
+
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
3259
|
+
if (vec !== null) vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
3260
|
+
}
|
|
2958
3261
|
}
|
|
2959
3262
|
}
|
|
2960
|
-
|
|
3263
|
+
function updatePreparationOf(ctx, indexName, docId, document) {
|
|
2961
3264
|
ctx.guardShutdown();
|
|
2962
3265
|
validateDocId(docId);
|
|
2963
3266
|
requireMatchingDocId(docId, document);
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
newDocument: document
|
|
2971
|
-
});
|
|
2972
|
-
}
|
|
2973
|
-
if (entry.config.required && entry.config.required.length > 0) {
|
|
2974
|
-
validateRequiredFields(document, entry.config.required);
|
|
2975
|
-
}
|
|
2976
|
-
if (entry.config.embedding) {
|
|
2977
|
-
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
2978
|
-
if (entry.embeddingAdapter) {
|
|
2979
|
-
await embedDocumentFields(
|
|
2980
|
-
document,
|
|
2981
|
-
entry.config.embedding,
|
|
2982
|
-
entry.embeddingAdapter,
|
|
2983
|
-
ctx.abortController.signal
|
|
2984
|
-
);
|
|
2985
|
-
} else {
|
|
2986
|
-
assertDocumentCarriesMappedVectors(
|
|
2987
|
-
document,
|
|
2988
|
-
entry.config.embedding,
|
|
2989
|
-
entry.embeddingAdapterName
|
|
2990
|
-
);
|
|
2991
|
-
}
|
|
2992
|
-
}
|
|
3267
|
+
if (!ctx.pluginRegistry.hasHooks("beforeUpdate")) return { document, beforeWrite: null, failure: void 0 };
|
|
3268
|
+
const oldDocument = ctx.requireManager(indexName).get(docId) ?? {};
|
|
3269
|
+
const beforeWrite = () => ctx.pluginRegistry.runHook("beforeUpdate", { indexName, docId, oldDocument, newDocument: document });
|
|
3270
|
+
return { document, beforeWrite, failure: void 0 };
|
|
3271
|
+
}
|
|
3272
|
+
async function finishUpdate(ctx, indexName, entry, docId, document) {
|
|
2993
3273
|
const sizing = requireDocumentWithinSizeLimit(document);
|
|
2994
3274
|
if (sizing !== null) await sizing;
|
|
3275
|
+
const manager = ctx.requireManager(indexName);
|
|
2995
3276
|
const vecIndexes = manager.getVectorIndexes();
|
|
2996
3277
|
const extractedVectors = /* @__PURE__ */ new Map();
|
|
2997
3278
|
if (vecIndexes.size > 0) {
|
|
@@ -3085,10 +3366,13 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
3085
3366
|
};
|
|
3086
3367
|
return () => manager.publication.write(indexChangeOutsideExternalWaits(updateInIndex));
|
|
3087
3368
|
}
|
|
3088
|
-
async function updateDocument(ctx, indexName, docId, document, options) {
|
|
3369
|
+
async function updateDocument(ctx, indexName, docId, document, options, hold) {
|
|
3089
3370
|
ctx.guardShutdown();
|
|
3090
|
-
|
|
3091
|
-
const
|
|
3371
|
+
ctx.requireIndex(indexName);
|
|
3372
|
+
const preparation = updatePreparationOf(ctx, indexName, docId, document);
|
|
3373
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
3374
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
3375
|
+
const prepared = await finishUpdate(ctx, indexName, entry, docId, document);
|
|
3092
3376
|
const apply = applyOfUpdate(ctx, indexName, prepared);
|
|
3093
3377
|
if (ctx.durability) {
|
|
3094
3378
|
try {
|
|
@@ -3111,10 +3395,12 @@ async function updateDocument(ctx, indexName, docId, document, options) {
|
|
|
3111
3395
|
} else {
|
|
3112
3396
|
await apply();
|
|
3113
3397
|
}
|
|
3114
|
-
if (!prepared.buffered) await
|
|
3398
|
+
if (!prepared.buffered) await settleUpdates(ctx, indexName, [prepared], hold);
|
|
3115
3399
|
ctx.checkHeapPressure(indexName);
|
|
3116
3400
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3117
3401
|
}
|
|
3402
|
+
|
|
3403
|
+
// src/engine/mutations/update-batch.ts
|
|
3118
3404
|
function reportedDocIdOf(update) {
|
|
3119
3405
|
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
3120
3406
|
return typeof docId === "string" ? docId : "";
|
|
@@ -3131,29 +3417,47 @@ function requireUpdateEntry(update) {
|
|
|
3131
3417
|
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
3132
3418
|
);
|
|
3133
3419
|
}
|
|
3134
|
-
async function
|
|
3420
|
+
async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed) {
|
|
3421
|
+
const preparations = [];
|
|
3422
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3423
|
+
const update = updates[i];
|
|
3424
|
+
const docId = reportedDocIdOf(update);
|
|
3425
|
+
try {
|
|
3426
|
+
const { document } = requireUpdateEntry(update);
|
|
3427
|
+
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
3428
|
+
} catch (err) {
|
|
3429
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
3430
|
+
}
|
|
3431
|
+
}
|
|
3432
|
+
const { entry } = await prepareDocumentWrites(
|
|
3433
|
+
ctx,
|
|
3434
|
+
indexName,
|
|
3435
|
+
preparations.map((pending) => pending.preparation),
|
|
3436
|
+
hold,
|
|
3437
|
+
false
|
|
3438
|
+
);
|
|
3439
|
+
const prepared = [];
|
|
3440
|
+
for (const { docId, preparation } of preparations) {
|
|
3441
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3442
|
+
try {
|
|
3443
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
3444
|
+
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
3445
|
+
} catch (err) {
|
|
3446
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
3447
|
+
}
|
|
3448
|
+
}
|
|
3449
|
+
return prepared;
|
|
3450
|
+
}
|
|
3451
|
+
async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
3135
3452
|
ctx.guardShutdown();
|
|
3136
|
-
|
|
3453
|
+
ctx.requireIndex(indexName);
|
|
3137
3454
|
validateBatchList(updates, "updates");
|
|
3138
3455
|
const succeeded = [];
|
|
3139
3456
|
const failed = [];
|
|
3140
|
-
const updateBatchManager = ctx.requireManager(indexName);
|
|
3141
|
-
const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
|
|
3142
3457
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
3143
3458
|
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
3144
3459
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
3145
|
-
const prepared =
|
|
3146
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3147
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3148
|
-
const update = updates[i];
|
|
3149
|
-
const docId = reportedDocIdOf(update);
|
|
3150
|
-
try {
|
|
3151
|
-
const { document } = requireUpdateEntry(update);
|
|
3152
|
-
prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document));
|
|
3153
|
-
} catch (err) {
|
|
3154
|
-
failed.push({ docId, error: asUpdateError(err) });
|
|
3155
|
-
}
|
|
3156
|
-
}
|
|
3460
|
+
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
3157
3461
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
3158
3462
|
let failures;
|
|
3159
3463
|
try {
|
|
@@ -3180,6 +3484,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
3180
3484
|
} finally {
|
|
3181
3485
|
for (const update of prepared) update.releaseVectorHolds?.();
|
|
3182
3486
|
}
|
|
3487
|
+
const settled = [];
|
|
3183
3488
|
for (let i = 0; i < prepared.length; i++) {
|
|
3184
3489
|
const update = prepared[i];
|
|
3185
3490
|
const failure = failures[i];
|
|
@@ -3187,19 +3492,21 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
3187
3492
|
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
3188
3493
|
continue;
|
|
3189
3494
|
}
|
|
3190
|
-
if (!update.buffered)
|
|
3495
|
+
if (!update.buffered) settled.push(update);
|
|
3191
3496
|
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
3192
3497
|
if (vec !== null) touchedVectorFields.add(fieldPath);
|
|
3193
3498
|
}
|
|
3194
3499
|
succeeded.push(update.docId);
|
|
3195
3500
|
}
|
|
3501
|
+
await settleUpdates(ctx, indexName, settled, hold);
|
|
3196
3502
|
if (chunkEnd < updates.length) {
|
|
3197
3503
|
ctx.checkHeapPressure(indexName);
|
|
3198
3504
|
await new Promise((r) => setTimeout(r, 0));
|
|
3199
3505
|
}
|
|
3200
3506
|
}
|
|
3507
|
+
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
3201
3508
|
for (const fieldPath of touchedVectorFields) {
|
|
3202
|
-
|
|
3509
|
+
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
3203
3510
|
}
|
|
3204
3511
|
ctx.checkHeapPressure(indexName);
|
|
3205
3512
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
@@ -3245,4 +3552,4 @@ async function updateEnginePartitionConfig(core, indexName, partitionConfig) {
|
|
|
3245
3552
|
core.watermarkNotifier.check(indexName);
|
|
3246
3553
|
}
|
|
3247
3554
|
|
|
3248
|
-
export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
|
|
3555
|
+
export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, prepareDocumentWrite, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
|