@delali/narsil 0.3.1-next.183 → 0.3.1-next.185

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/dist/browser/{chunk-ZZUXMJIK.mjs → chunk-CPSHS46N.mjs} +129 -165
  2. package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
  3. package/dist/browser/{optional-writes-IQFTXANH.mjs → optional-writes-ZTBHYQMN.mjs} +619 -314
  4. package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
  5. package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
  6. package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
  7. package/dist/browser/{storage-leaf-2CO6SQ7R.mjs → storage-leaf-4XMCITJ6.mjs} +27 -9
  8. package/dist/{chunk-GGG5R4CU.mjs → chunk-2GONI5HL.mjs} +5 -3
  9. package/dist/{chunk-PK3BGGHO.mjs → chunk-2KCCQK4R.mjs} +1 -1
  10. package/dist/{chunk-JVR5KMO4.mjs → chunk-2YWWHMVC.mjs} +2 -2
  11. package/dist/chunk-3ECIRFS5.mjs +61 -0
  12. package/dist/{react-native/chunks/chunk-G4YYO5TC.mjs → chunk-4JBTALUC.mjs} +1 -1
  13. package/dist/{chunk-PXEGKQHS.mjs → chunk-5BUEBIXR.mjs} +2 -2
  14. package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
  15. package/dist/{chunk-NPXORGMK.mjs → chunk-7WXGUTWK.mjs} +3 -3
  16. package/dist/{chunk-2DK74OQL.mjs → chunk-A3OAFUZI.mjs} +1 -1
  17. package/dist/{chunk-632ABEIR.mjs → chunk-BVZWWMOU.mjs} +1 -1
  18. package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
  19. package/dist/{chunk-F2UOTRY7.mjs → chunk-CODGHJVW.mjs} +4 -4
  20. package/dist/{chunk-6ICOWB6I.mjs → chunk-CYCVPTTR.mjs} +1 -1
  21. package/dist/{chunk-WUWSUJ2X.mjs → chunk-DU23J5JG.mjs} +5 -5
  22. package/dist/{chunk-C77OBNPP.mjs → chunk-EDEE7ETQ.mjs} +1 -1
  23. package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
  24. package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
  25. package/dist/{chunk-V7F5DOSC.mjs → chunk-FBQYC5OX.mjs} +3 -3
  26. package/dist/{chunk-EGWBZSHK.mjs → chunk-FCQMZDRI.mjs} +2 -2
  27. package/dist/{chunk-MCAJOLVK.mjs → chunk-G3XRHAJA.mjs} +2 -2
  28. package/dist/{chunk-OQANB7SJ.mjs → chunk-H4QYQ5DE.mjs} +1 -1
  29. package/dist/{chunk-CKS4QQEC.mjs → chunk-II6XF6M4.mjs} +5 -5
  30. package/dist/{chunk-C3EINJNJ.mjs → chunk-JDG6CCXM.mjs} +1 -1
  31. package/dist/{chunk-3OEXOSWI.mjs → chunk-LR6JJVML.mjs} +4 -4
  32. package/dist/{chunk-TTTBE4TB.mjs → chunk-M5I64AQW.mjs} +3 -3
  33. package/dist/{chunk-FIXUVFUC.mjs → chunk-MHBG33R5.mjs} +7 -6
  34. package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
  35. package/dist/{chunk-M3H7CTIC.mjs → chunk-O337MDRJ.mjs} +5 -5
  36. package/dist/{chunk-CWXR7OP4.mjs → chunk-SL53DHVE.mjs} +804 -527
  37. package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
  38. package/dist/{chunk-SA7U67XF.mjs → chunk-URVH6BLO.mjs} +2 -2
  39. package/dist/{chunk-T3YI4KN6.mjs → chunk-VYSWVQCE.mjs} +1 -1
  40. package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
  41. package/dist/{chunk-ZVZ2V2PZ.mjs → chunk-WSIU6YXQ.mjs} +39 -20
  42. package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
  43. package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
  44. package/dist/{direct-executor-JXYIL4OQ.mjs → direct-executor-37IH3ISS.mjs} +5 -4
  45. package/dist/distribution.mjs +254 -76
  46. package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
  47. package/dist/index.browser.mjs +1 -1
  48. package/dist/index.mjs +5 -4
  49. package/dist/index.react-native.mjs +4 -3
  50. package/dist/{long-query-thread-HPKGRYWM.mjs → long-query-thread-DIPJBMV7.mjs} +15 -14
  51. package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
  52. package/dist/{optional-writes-QE7QUI45.mjs → optional-writes-YIVJMQ4D.mjs} +7 -6
  53. package/dist/{paged-partition-archive-IK2UOY2G.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
  54. package/dist/partial-words.browser.mjs +1 -1
  55. package/dist/partial-words.mjs +5 -4
  56. package/dist/persistence/durability/checkpoint-worker.mjs +13 -12
  57. package/dist/{query-actions-COEFVUH3.mjs → query-actions-7EOKT3WD.mjs} +5 -4
  58. package/dist/query-suggestions/merge-worker.mjs +8 -7
  59. package/dist/query-suggestions.browser.mjs +1 -1
  60. package/dist/query-suggestions.mjs +6 -5
  61. package/dist/react-native/chunks/{chunk-DZ4FQW5K.mjs → chunk-7TPO4FLE.mjs} +2 -2
  62. package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
  63. package/dist/react-native/chunks/{chunk-6A5XE2MA.mjs → chunk-CBU4M3EU.mjs} +803 -526
  64. package/dist/react-native/chunks/{chunk-DN2GVVR4.mjs → chunk-IEY4DARA.mjs} +1 -1
  65. package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
  66. package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
  67. package/dist/react-native/chunks/{chunk-UPN4NUXU.mjs → chunk-T5HF3FW3.mjs} +1 -1
  68. package/dist/react-native/chunks/{chunk-XNIH3WIB.mjs → chunk-ZQPSXN6C.mjs} +1 -1
  69. package/dist/{chunk-YWSIRKKQ.mjs → react-native/chunks/chunk-ZUFOYZ5P.mjs} +1 -1
  70. package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
  71. package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
  72. package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
  73. package/dist/react-native/chunks/{optional-writes-YZ7WTR7V.mjs → optional-writes-QETIM6BI.mjs} +5 -4
  74. package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
  75. package/dist/react-native/chunks/{restore-queue-IGKD6Q2Y.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
  76. package/dist/react-native/chunks/{runtime-BUCSAK4U.mjs → runtime-UOHD2XUJ.mjs} +23 -3
  77. package/dist/react-native/chunks/{segment-codec-leaf-JTTGDT3G.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
  78. package/dist/react-native/chunks/{segment-merge-leaf-D5ZFJRUB.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
  79. package/dist/react-native/chunks/{storage-leaf-ZKIM6IF6.mjs → storage-leaf-LNOLKWZB.mjs} +36 -17
  80. package/dist/react-native/chunks/{store-leaf-PHBOTT5J.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
  81. package/dist/react-native/chunks/{suggestions-leaf-R7MIH6Q3.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
  82. package/dist/react-native/chunks/{worker-copies-U35KPVJ7.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
  83. package/dist/react-native/partial-words.mjs +4 -3
  84. package/dist/react-native/query-suggestions.mjs +4 -3
  85. package/dist/react-native/sync.mjs +4 -3
  86. package/dist/react-native/synonyms.mjs +4 -3
  87. package/dist/{rerank-LYL3D7LU.mjs → rerank-KTV5L7MZ.mjs} +5 -4
  88. package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
  89. package/dist/{restore-queue-LF4DTVJC.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
  90. package/dist/{runtime-4GKUIAKV.mjs → runtime-USQYKN3I.mjs} +23 -3
  91. package/dist/{segment-codec-leaf-ACALKS2C.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
  92. package/dist/{segment-merge-leaf-SKMPASFK.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
  93. package/dist/server.mjs +22 -21
  94. package/dist/{storage-leaf-OJ55R4C5.mjs → storage-leaf-ZI3H2WJY.mjs} +26 -25
  95. package/dist/{store-leaf-CWHHEDRK.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
  96. package/dist/{suggestions-leaf-ZTU3ZTWJ.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
  97. package/dist/sync/filesystem.mjs +10 -9
  98. package/dist/sync.browser.mjs +1 -1
  99. package/dist/sync.mjs +5 -4
  100. package/dist/synonyms.browser.mjs +1 -1
  101. package/dist/synonyms.mjs +5 -4
  102. package/dist/{thread-BEGFRULV.mjs → thread-NQRNW3IH.mjs} +14 -13
  103. package/dist/vector/search-worker.mjs +5 -4
  104. package/dist/{worker-copies-373P6IYZ.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
  105. package/dist/workers/entry.mjs +11 -10
  106. package/package.json +7 -7
  107. package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
  108. package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
@@ -1628,19 +1628,17 @@ function removeDocument(input, generation, ordinal, fields) {
1628
1628
  // src/engine/optional-writes-host.browser.ts
1629
1629
  var BATCH_CHUNK_SIZE;
1630
1630
  var ErrorCodes;
1631
+ var MAX_CALLS_ACROSS_INDEX_CHANGES;
1631
1632
  var MIN_DOCUMENTS_FOR_SEGMENTS;
1632
1633
  var NarsilError;
1633
1634
  var admitBufferedUpsert;
1634
1635
  var admitInsert;
1635
1636
  var arrayPositionsAreStale;
1636
- var asBatchInsertError;
1637
1637
  var awaitWriteVisibility;
1638
+ var capsEachPartition;
1638
1639
  var changeIndexOutsideExternalWaits;
1639
- var collectRequiredFieldFailures;
1640
1640
  var createFrozenSegment;
1641
- var createPendingAdmissions;
1642
1641
  var deleteNestedValue;
1643
- var embedChunkDocuments;
1644
1642
  var emitSyncChange;
1645
1643
  var extractVectorFromDoc;
1646
1644
  var fnv1a;
@@ -1662,7 +1660,6 @@ var prepareDocumentVectors;
1662
1660
  var prepareStoredSynonyms;
1663
1661
  var providedDocId;
1664
1662
  var readHeapStatistics;
1665
- var recordPendingAdmission;
1666
1663
  var removeDocumentVectors;
1667
1664
  var requireDocumentObject;
1668
1665
  var requireDocumentWithinSizeLimit;
@@ -1703,19 +1700,17 @@ function bindOptionalWritesHost(host, segmentCodec) {
1703
1700
  ({
1704
1701
  BATCH_CHUNK_SIZE,
1705
1702
  ErrorCodes,
1703
+ MAX_CALLS_ACROSS_INDEX_CHANGES,
1706
1704
  MIN_DOCUMENTS_FOR_SEGMENTS,
1707
1705
  NarsilError,
1708
1706
  admitBufferedUpsert,
1709
1707
  admitInsert,
1710
1708
  arrayPositionsAreStale,
1711
- asBatchInsertError,
1712
1709
  awaitWriteVisibility,
1710
+ capsEachPartition,
1713
1711
  changeIndexOutsideExternalWaits,
1714
- collectRequiredFieldFailures,
1715
1712
  createFrozenSegment,
1716
- createPendingAdmissions,
1717
1713
  deleteNestedValue,
1718
- embedChunkDocuments,
1719
1714
  emitSyncChange,
1720
1715
  extractVectorFromDoc,
1721
1716
  fnv1a,
@@ -1737,7 +1732,6 @@ function bindOptionalWritesHost(host, segmentCodec) {
1737
1732
  prepareStoredSynonyms,
1738
1733
  providedDocId,
1739
1734
  readHeapStatistics,
1740
- recordPendingAdmission,
1741
1735
  removeDocumentVectors,
1742
1736
  requireDocumentObject,
1743
1737
  requireDocumentWithinSizeLimit,
@@ -1995,6 +1989,326 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1995
1989
  }
1996
1990
  }
1997
1991
 
1992
+ // src/engine/mutations/batch-admissions.ts
1993
+ function createPendingAdmissions() {
1994
+ return { total: 0, byPartition: /* @__PURE__ */ new Map() };
1995
+ }
1996
+ function recordPendingAdmission(ctx, pending, manager, docId) {
1997
+ pending.total += 1;
1998
+ if (!capsEachPartition(ctx, manager)) return;
1999
+ const partitionId = manager.routePartition(docId);
2000
+ pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
2001
+ }
2002
+ function asBatchInsertError(err) {
2003
+ return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
2004
+ }
2005
+
2006
+ // src/engine/mutations/outside-hold.ts
2007
+ function runOutside(hold, work) {
2008
+ return hold === void 0 ? work() : hold.outside(work);
2009
+ }
2010
+
2011
+ // src/engine/mutations/write-preparation.ts
2012
+ function settingsOf(entry) {
2013
+ return { required: entry.config.required, embedding: entry.config.embedding, adapter: entry.embeddingAdapter };
2014
+ }
2015
+ function embedsThroughAdapter(entry) {
2016
+ return entry.config.embedding !== void 0 && entry.embeddingAdapter !== null;
2017
+ }
2018
+ function sourcesOf(sources) {
2019
+ return typeof sources === "string" ? [sources] : sources;
2020
+ }
2021
+ function sameMapping(left, right) {
2022
+ if (left === right) return true;
2023
+ const targets = Object.keys(left);
2024
+ if (targets.length !== Object.keys(right).length) return false;
2025
+ for (const target of targets) {
2026
+ if (!Object.hasOwn(right, target)) return false;
2027
+ const leftSources = sourcesOf(left[target]);
2028
+ const rightSources = sourcesOf(right[target]);
2029
+ if (leftSources.length !== rightSources.length) return false;
2030
+ if (leftSources.some((source, position) => source !== rightSources[position])) return false;
2031
+ }
2032
+ return true;
2033
+ }
2034
+ function embeddingFits(entry, embedding) {
2035
+ const embeddingConfig = entry.config.embedding;
2036
+ if (embeddingConfig === void 0 || entry.embeddingAdapter === null) return embedding === null;
2037
+ return embedding !== null && embedding.adapter === entry.embeddingAdapter && sameMapping(embedding.mapping, embeddingConfig.fields);
2038
+ }
2039
+ function embeddingOfDocuments(embedding, documents) {
2040
+ if (embedding === null) return null;
2041
+ const fields = /* @__PURE__ */ new Map();
2042
+ for (const document of documents) {
2043
+ const embedded = embedding.fields.get(document);
2044
+ if (embedded !== void 0) fields.set(document, embedded);
2045
+ }
2046
+ return { adapter: embedding.adapter, mapping: embedding.mapping, fields };
2047
+ }
2048
+ function writable(preparations) {
2049
+ return preparations.filter((preparation) => preparation.failure === void 0);
2050
+ }
2051
+ function failAll(preparations, failure) {
2052
+ for (const preparation of writable(preparations)) preparation.failure = failure;
2053
+ }
2054
+ function requireFieldsOf(required, preparations) {
2055
+ if (required === void 0 || required.length === 0) return;
2056
+ for (const preparation of writable(preparations)) {
2057
+ try {
2058
+ validateRequiredFields(preparation.document, required);
2059
+ } catch (err) {
2060
+ preparation.failure = err;
2061
+ }
2062
+ }
2063
+ }
2064
+ async function requireMappedVectors(entry, preparations) {
2065
+ const embeddingConfig = entry.config.embedding;
2066
+ if (embeddingConfig === void 0 || entry.embeddingAdapter !== null) return;
2067
+ const { assertDocumentCarriesMappedVectors } = await loadEmbedCode();
2068
+ for (const preparation of writable(preparations)) {
2069
+ try {
2070
+ assertDocumentCarriesMappedVectors(
2071
+ preparation.document,
2072
+ embeddingConfig,
2073
+ entry.embeddingAdapterName
2074
+ );
2075
+ } catch (err) {
2076
+ preparation.failure = err;
2077
+ }
2078
+ }
2079
+ }
2080
+ async function embedWith(settings, signal, preparations, earlier, together) {
2081
+ requireFieldsOf(settings.required, preparations);
2082
+ const { clearWriteEmbedding, embedDocumentsForWrite, lacksMappedVectors } = await loadEmbedCode();
2083
+ if (earlier !== null) clearWriteEmbedding(earlier);
2084
+ const { embedding: embeddingConfig, adapter } = settings;
2085
+ const embeddable = writable(preparations);
2086
+ const documentFailures = /* @__PURE__ */ new Map();
2087
+ if (embeddingConfig === void 0 || adapter === null || embeddable.length === 0) {
2088
+ return { embedding: null, called: false, documentFailures };
2089
+ }
2090
+ const fields = /* @__PURE__ */ new Map();
2091
+ const embedding = { adapter, mapping: embeddingConfig.fields, fields };
2092
+ const documents = embeddable.map((preparation) => preparation.document);
2093
+ if (!lacksMappedVectors(documents, embeddingConfig)) return { embedding, called: false, documentFailures };
2094
+ for (let chunkStart = 0; chunkStart < embeddable.length; chunkStart += BATCH_CHUNK_SIZE) {
2095
+ const chunk = embeddable.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
2096
+ const embedded = await embedDocumentsForWrite(
2097
+ chunk.map((preparation) => preparation.document),
2098
+ embeddingConfig,
2099
+ adapter,
2100
+ signal,
2101
+ together
2102
+ );
2103
+ for (const [document, embeddedFields] of embedded.embedding.fields) fields.set(document, embeddedFields);
2104
+ for (const [position, error] of embedded.failed) chunk[position].failure = error;
2105
+ for (const [position, error] of embedded.documentFailures) documentFailures.set(chunk[position], error);
2106
+ }
2107
+ return { embedding, called: true, documentFailures };
2108
+ }
2109
+ function settleDocumentFailures(entry, pass) {
2110
+ if (!embeddingFits(entry, pass.embedding)) return [...pass.documentFailures.keys()];
2111
+ for (const [preparation, failure] of pass.documentFailures) preparation.failure = failure;
2112
+ return [];
2113
+ }
2114
+ async function runBeforeWrites(preparations) {
2115
+ for (const preparation of preparations) {
2116
+ if (preparation.beforeWrite === null || preparation.failure !== void 0) continue;
2117
+ try {
2118
+ await preparation.beforeWrite();
2119
+ } catch (err) {
2120
+ preparation.failure = err;
2121
+ }
2122
+ }
2123
+ }
2124
+ async function embedThenRunHooks(settings, signal, preparations, earlier) {
2125
+ const pass = await embedWith(settings, signal, preparations, earlier, true);
2126
+ await runBeforeWrites(preparations.filter((preparation) => !pass.documentFailures.has(preparation)));
2127
+ return pass;
2128
+ }
2129
+ async function runHooksThenEmbedEach(ctx, indexName, preparations, earlier) {
2130
+ let settings = null;
2131
+ let embedding = null;
2132
+ let called = false;
2133
+ let uncleared = earlier;
2134
+ const documentFailures = /* @__PURE__ */ new Map();
2135
+ for (const preparation of preparations) {
2136
+ await runBeforeWrites([preparation]);
2137
+ settings ??= settingsOf(ctx.requireIndex(indexName));
2138
+ const pass = await embedWith(settings, ctx.abortController.signal, [preparation], uncleared, false);
2139
+ uncleared = null;
2140
+ called ||= pass.called;
2141
+ for (const [failed, failure] of pass.documentFailures) documentFailures.set(failed, failure);
2142
+ if (pass.embedding === null) continue;
2143
+ if (embedding === null) embedding = pass.embedding;
2144
+ else for (const [document, fields] of pass.embedding.fields) embedding.fields.set(document, fields);
2145
+ }
2146
+ return { embedding, called, documentFailures };
2147
+ }
2148
+ function adapterChangesRefusal(indexName, embeddings) {
2149
+ return new NarsilError(
2150
+ ErrorCodes.WORKER_BUSY,
2151
+ `Changes to index "${indexName}" replaced its embedding adapter after each of ${embeddings} embeddings of this write`,
2152
+ { indexName, embeddings }
2153
+ );
2154
+ }
2155
+ async function prepareDocumentWrites(ctx, indexName, preparations, hold, together, earlier = null) {
2156
+ let entry = ctx.requireIndex(indexName);
2157
+ let embedding = earlier;
2158
+ if (writable(preparations).length === 0) return { entry, embedding };
2159
+ const signal = ctx.abortController.signal;
2160
+ let embeddings = 0;
2161
+ let unhooked = [];
2162
+ const hooked = preparations.some((preparation) => preparation.beforeWrite !== null);
2163
+ if (hooked || embedsThroughAdapter(entry) && !embeddingFits(entry, embedding)) {
2164
+ if (together && !embedsThroughAdapter(entry)) {
2165
+ requireFieldsOf(entry.config.required, preparations);
2166
+ await requireMappedVectors(entry, preparations);
2167
+ }
2168
+ const settings = settingsOf(entry);
2169
+ const uncleared = embedding;
2170
+ const pass = await runOutside(
2171
+ hold,
2172
+ () => together ? embedThenRunHooks(settings, signal, preparations, uncleared) : runHooksThenEmbedEach(ctx, indexName, preparations, uncleared)
2173
+ );
2174
+ embedding = pass.embedding;
2175
+ if (pass.called) embeddings += 1;
2176
+ entry = ctx.requireIndex(indexName);
2177
+ unhooked = settleDocumentFailures(entry, pass);
2178
+ }
2179
+ while (writable(preparations).length > 0 && !embeddingFits(entry, embedding)) {
2180
+ if (embeddings >= MAX_CALLS_ACROSS_INDEX_CHANGES) {
2181
+ failAll(preparations, adapterChangesRefusal(indexName, embeddings));
2182
+ break;
2183
+ }
2184
+ const uncleared = embedding;
2185
+ const settings = settingsOf(entry);
2186
+ const hooking = unhooked;
2187
+ const pass = await runOutside(hold, async () => {
2188
+ const embedded = await embedWith(settings, signal, preparations, uncleared, together);
2189
+ await runBeforeWrites(hooking.filter((preparation) => !embedded.documentFailures.has(preparation)));
2190
+ return embedded;
2191
+ });
2192
+ embedding = pass.embedding;
2193
+ if (pass.called) embeddings += 1;
2194
+ entry = ctx.requireIndex(indexName);
2195
+ const stale = settleDocumentFailures(entry, pass);
2196
+ unhooked = hooking.filter((preparation) => stale.includes(preparation));
2197
+ }
2198
+ requireFieldsOf(entry.config.required, preparations);
2199
+ await requireMappedVectors(entry, preparations);
2200
+ return { entry, embedding };
2201
+ }
2202
+ async function prepareDocumentWrite(ctx, indexName, document, hold, beforeWrite) {
2203
+ const preparation = { document, beforeWrite, failure: void 0 };
2204
+ const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
2205
+ if (preparation.failure !== void 0) throw preparation.failure;
2206
+ return entry;
2207
+ }
2208
+
2209
+ // src/engine/mutations/insert-batch-admission.ts
2210
+ function pendingInsertOf(ctx, indexName, docId, document, hooked) {
2211
+ const pending = {
2212
+ docId,
2213
+ preparation: { document, beforeWrite: null, failure: void 0 },
2214
+ refusedByHook: false
2215
+ };
2216
+ if (hooked && ctx.pluginRegistry.hasHooks("beforeInsert")) {
2217
+ pending.preparation.beforeWrite = async () => {
2218
+ try {
2219
+ await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document });
2220
+ } catch (err) {
2221
+ pending.refusedByHook = true;
2222
+ throw err;
2223
+ }
2224
+ };
2225
+ }
2226
+ return pending;
2227
+ }
2228
+ async function prepareInsertDocuments(ctx, indexName, documents, start, end, hold, hooked, failed, earlier = null) {
2229
+ const pending = [];
2230
+ for (let i = start; i < end; i++) {
2231
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2232
+ if (ctx.abortController.signal.aborted) break;
2233
+ const document = documents[i];
2234
+ const docId = providedDocId(document) ?? ctx.idGenerator();
2235
+ try {
2236
+ const sizing = requireDocumentObject(document);
2237
+ if (sizing !== null) await sizing;
2238
+ validateDocId(docId);
2239
+ pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
2240
+ } catch (err) {
2241
+ failed.push({ docId, error: asBatchInsertError(err) });
2242
+ }
2243
+ }
2244
+ const { entry, embedding } = await prepareDocumentWrites(
2245
+ ctx,
2246
+ indexName,
2247
+ pending.map((item) => item.preparation),
2248
+ hold,
2249
+ true,
2250
+ earlier
2251
+ );
2252
+ const candidates = [];
2253
+ for (const item of pending) {
2254
+ const { document, failure } = item.preparation;
2255
+ if (failure === void 0) {
2256
+ candidates.push({ docId: item.docId, document });
2257
+ continue;
2258
+ }
2259
+ const docId = item.refusedByHook ? item.docId : providedDocId(document) ?? "";
2260
+ failed.push({ docId, error: asBatchInsertError(failure) });
2261
+ }
2262
+ return { entry, candidates, embedding };
2263
+ }
2264
+ async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
2265
+ const { entry, candidates, embedding } = await prepareInsertDocuments(
2266
+ ctx,
2267
+ indexName,
2268
+ documents,
2269
+ 0,
2270
+ documents.length,
2271
+ hold,
2272
+ true,
2273
+ failed
2274
+ );
2275
+ const manager = ctx.requireManager(indexName);
2276
+ const vecIndexes = manager.getVectorIndexes();
2277
+ const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
2278
+ const strict = entry.config.strict === true;
2279
+ const admitted = [];
2280
+ const admittedIds = /* @__PURE__ */ new Set();
2281
+ const pending = createPendingAdmissions();
2282
+ for (let position = 0; position < candidates.length; position++) {
2283
+ if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
2284
+ if (ctx.abortController.signal.aborted) break;
2285
+ const { docId, document } = candidates[position];
2286
+ try {
2287
+ const { partitionDoc, extractedVectors } = prepareDocumentVectors(
2288
+ document,
2289
+ vectorFieldPaths
2290
+ );
2291
+ if (extractedVectors.size > 0) {
2292
+ validateVectorDimensions(extractedVectors, vecIndexes);
2293
+ }
2294
+ admitInsert(ctx, indexName, manager, docId, pending);
2295
+ if (admittedIds.has(docId) || manager.has(docId)) {
2296
+ throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
2297
+ }
2298
+ validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
2299
+ if (strict) {
2300
+ validateDocumentStrict(partitionDoc, entry.config.schema);
2301
+ }
2302
+ admittedIds.add(docId);
2303
+ recordPendingAdmission(ctx, pending, manager, docId);
2304
+ admitted.push({ docId, document, partitionDoc, extractedVectors });
2305
+ } catch (err) {
2306
+ failed.push({ docId, error: asBatchInsertError(err) });
2307
+ }
2308
+ }
2309
+ return { admitted, embedding };
2310
+ }
2311
+
1998
2312
  // src/engine/mutations/durable-rollback.ts
1999
2313
  function holdVectorsForRollback(indexes, docId) {
2000
2314
  const releases = [];
@@ -2230,84 +2544,29 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
2230
2544
  return applications;
2231
2545
  }
2232
2546
 
2233
- // src/engine/mutations/insert-batch-admission.ts
2234
- async function admitBatchDocuments(ctx, indexName, documents, failed) {
2235
- const entry = ctx.requireIndex(indexName);
2236
- const manager = ctx.requireManager(indexName);
2237
- const vecIndexes = manager.getVectorIndexes();
2238
- const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
2239
- const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
2240
- const strict = entry.config.strict === true;
2241
- const required = entry.config.required;
2242
- const admitted = [];
2243
- const admittedIds = /* @__PURE__ */ new Set();
2244
- const pending = createPendingAdmissions();
2245
- for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
2246
- if (ctx.abortController.signal.aborted) break;
2247
- const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
2248
- const chunkFailedIndexes = /* @__PURE__ */ new Set();
2249
- if (required && required.length > 0) {
2250
- collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
2251
- }
2252
- await embedChunkDocuments(
2253
- entry,
2254
- documents,
2255
- chunkStart,
2256
- chunkEnd,
2257
- ctx.abortController.signal,
2258
- chunkFailedIndexes,
2259
- failed
2260
- );
2261
- for (let i = chunkStart; i < chunkEnd; i++) {
2262
- if (ctx.abortController.signal.aborted) break;
2263
- if (chunkFailedIndexes.has(i)) continue;
2264
- const docId = providedDocId(documents[i]) ?? ctx.idGenerator();
2547
+ // src/engine/mutations/insert-batch-documents.ts
2548
+ async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
2549
+ if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
2550
+ const manager = ctx.executor.getManager(indexName);
2551
+ await runOutside(hold, async () => {
2552
+ for (const doc of inserted) {
2265
2553
  try {
2266
- const sizing = requireDocumentObject(documents[i]);
2267
- if (sizing !== null) await sizing;
2268
- validateDocId(docId);
2269
- if (hasBeforeHook) {
2270
- await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
2271
- }
2272
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
2273
- documents[i],
2274
- vectorFieldPaths
2275
- );
2276
- if (extractedVectors.size > 0) {
2277
- validateVectorDimensions(extractedVectors, vecIndexes);
2278
- }
2279
- admitInsert(ctx, indexName, manager, docId, pending);
2280
- if (admittedIds.has(docId) || manager.has(docId)) {
2281
- throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
2282
- }
2283
- validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
2284
- if (strict) {
2285
- validateDocumentStrict(partitionDoc, entry.config.schema);
2286
- }
2287
- admittedIds.add(docId);
2288
- recordPendingAdmission(ctx, pending, manager, docId);
2289
- admitted.push({ docId, document: documents[i], partitionDoc, extractedVectors });
2554
+ await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
2290
2555
  } catch (err) {
2291
- failed.push({ docId, error: asBatchInsertError(err) });
2556
+ console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
2292
2557
  }
2293
2558
  }
2294
- if (chunkEnd < documents.length) {
2295
- await new Promise((r) => setTimeout(r, 0));
2296
- }
2297
- }
2298
- return admitted;
2559
+ });
2560
+ return ctx.executor.getManager(indexName) !== manager;
2299
2561
  }
2300
-
2301
- // src/engine/mutations/insert-batch-documents.ts
2302
- async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed) {
2303
- const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
2562
+ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
2304
2563
  const succeeded = [];
2305
- const buffered = /* @__PURE__ */ new Set();
2306
2564
  const touchedVectorFields = /* @__PURE__ */ new Set();
2307
2565
  for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
2308
2566
  if (ctx.abortController.signal.aborted) break;
2309
2567
  const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
2310
2568
  const applications = await applyInsertChunk(ctx, indexName, chunk, options);
2569
+ const inserted = [];
2311
2570
  for (let i = 0; i < chunk.length; i++) {
2312
2571
  const doc = chunk[i];
2313
2572
  const application = applications[i];
@@ -2316,30 +2575,103 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
2316
2575
  failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
2317
2576
  continue;
2318
2577
  }
2319
- if (application.status === "buffered") {
2320
- buffered.add(doc.docId);
2321
- succeeded.push(doc.docId);
2322
- continue;
2323
- }
2578
+ succeeded.push(doc.docId);
2579
+ if (application.status === "buffered") continue;
2324
2580
  for (const fieldPath of doc.extractedVectors.keys()) {
2325
2581
  touchedVectorFields.add(fieldPath);
2326
2582
  }
2327
- if (hasAfterHook) {
2328
- try {
2329
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
2330
- } catch (err) {
2331
- console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
2332
- }
2333
- }
2334
- succeeded.push(doc.docId);
2583
+ inserted.push(doc);
2335
2584
  }
2336
2585
  const applied = chunkStart + BATCH_CHUNK_SIZE;
2586
+ if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
2587
+ return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
2588
+ }
2337
2589
  if (applied < admitted.length) {
2338
2590
  ctx.checkHeapPressure(indexName);
2339
2591
  await new Promise((r) => setTimeout(r, 0));
2340
2592
  }
2341
2593
  }
2342
- return { succeeded, buffered, touchedVectorFields };
2594
+ return { succeeded, touchedVectorFields, unrecorded: [] };
2595
+ }
2596
+
2597
+ // src/engine/mutations/insert-batch-chunks.ts
2598
+ async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
2599
+ const { entry, candidates } = await prepareInsertDocuments(
2600
+ ctx,
2601
+ indexName,
2602
+ documents,
2603
+ chunkStart,
2604
+ chunkEnd,
2605
+ hold,
2606
+ hooked,
2607
+ failed,
2608
+ earlier
2609
+ );
2610
+ const manager = ctx.requireManager(indexName);
2611
+ const vecIndexes = manager.getVectorIndexes();
2612
+ const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
2613
+ const admitted = [];
2614
+ for (const { docId, document } of candidates) {
2615
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2616
+ if (ctx.abortController.signal.aborted) break;
2617
+ try {
2618
+ const { partitionDoc, extractedVectors } = prepareDocumentVectors(
2619
+ document,
2620
+ vectorFieldPaths
2621
+ );
2622
+ if (extractedVectors.size > 0) {
2623
+ validateVectorDimensions(extractedVectors, vecIndexes);
2624
+ }
2625
+ admitInsert(ctx, indexName, manager, docId);
2626
+ admitted.push({ docId, document, partitionDoc, extractedVectors });
2627
+ } catch (err) {
2628
+ failed.push({ docId, error: asBatchInsertError(err) });
2629
+ }
2630
+ }
2631
+ return admitted;
2632
+ }
2633
+ async function insertInChunks(ctx, indexName, documents, options, hold, hooked, failed, embedded = null) {
2634
+ const succeeded = [];
2635
+ const touchedVectorFields = /* @__PURE__ */ new Set();
2636
+ for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
2637
+ if (ctx.abortController.signal.aborted) break;
2638
+ const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
2639
+ const earlier = embeddingOfDocuments(embedded, documents.slice(chunkStart, chunkEnd));
2640
+ const prepared = await admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier);
2641
+ if (prepared.length > 0) {
2642
+ const applications = await applyInsertChunk(ctx, indexName, prepared, options);
2643
+ const inserted = [];
2644
+ for (let i = 0; i < prepared.length; i++) {
2645
+ const doc = prepared[i];
2646
+ const application = applications[i];
2647
+ if (application.status === "skipped") continue;
2648
+ if (application.status === "failed") {
2649
+ failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
2650
+ continue;
2651
+ }
2652
+ succeeded.push(doc.docId);
2653
+ if (application.status === "buffered") continue;
2654
+ for (const fieldPath of doc.extractedVectors.keys()) touchedVectorFields.add(fieldPath);
2655
+ inserted.push(doc);
2656
+ }
2657
+ await runAfterInsertHooks(ctx, indexName, inserted, hold);
2658
+ }
2659
+ if (chunkEnd < documents.length) {
2660
+ ctx.checkHeapPressure(indexName);
2661
+ await new Promise((r) => setTimeout(r, 0));
2662
+ }
2663
+ }
2664
+ return { succeeded, touchedVectorFields };
2665
+ }
2666
+ async function finishInsertBatch(ctx, indexName, inserts, options) {
2667
+ const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
2668
+ for (const fieldPath of inserts.touchedVectorFields) {
2669
+ vecIndexes?.get(fieldPath)?.scheduleBuild();
2670
+ }
2671
+ ctx.checkWatermark(indexName, inserts.succeeded);
2672
+ ctx.checkHeapPressure(indexName);
2673
+ await ctx.orchestrator.scaleOutReadyIndexes();
2674
+ if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2343
2675
  }
2344
2676
 
2345
2677
  // src/engine/mutations/segment-publication.ts
@@ -2444,10 +2776,6 @@ function mainStoreDocument(doc, options) {
2444
2776
  if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
2445
2777
  return structuredClone(doc.partitionDoc);
2446
2778
  }
2447
- async function applyIndividually(ctx, indexName, admitted, options, failed) {
2448
- const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
2449
- return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
2450
- }
2451
2779
  function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted) {
2452
2780
  const chunks = [];
2453
2781
  let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
@@ -2497,12 +2825,12 @@ function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberI
2497
2825
  if (chunk.docs.length > 0) chunks.push(chunk);
2498
2826
  return chunks;
2499
2827
  }
2500
- async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, failed) {
2501
- const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
2828
+ async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, hold, failed) {
2502
2829
  const succeededIds = /* @__PURE__ */ new Set();
2503
2830
  const errorOf = /* @__PURE__ */ new Map();
2504
2831
  const touchedVectorFields = /* @__PURE__ */ new Set();
2505
2832
  const chunks = chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted);
2833
+ const unrecorded = [];
2506
2834
  for (let index = 0; index < chunks.length; index++) {
2507
2835
  const { docs, attachmentOf } = chunks[index];
2508
2836
  const applies = docs.map((doc) => {
@@ -2512,6 +2840,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
2512
2840
  });
2513
2841
  const failures = await recordChunk(ctx, indexName, docs, applies);
2514
2842
  await ctx.orchestrator.settlePublications(indexName, "segments");
2843
+ const inserted = [];
2515
2844
  for (let i = 0; i < docs.length; i++) {
2516
2845
  const doc = docs[i];
2517
2846
  const failure = failures[i];
@@ -2543,15 +2872,13 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
2543
2872
  for (const fieldPath of doc.extractedVectors.keys()) {
2544
2873
  touchedVectorFields.add(fieldPath);
2545
2874
  }
2546
- if (hasAfterHook) {
2547
- try {
2548
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
2549
- } catch (err) {
2550
- console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
2551
- }
2552
- }
2875
+ inserted.push(doc);
2553
2876
  succeededIds.add(doc.docId);
2554
2877
  }
2878
+ if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
2879
+ for (const later of chunks.slice(index + 1)) unrecorded.push(...later.docs);
2880
+ break;
2881
+ }
2555
2882
  if (index + 1 < chunks.length) {
2556
2883
  ctx.checkHeapPressure(indexName);
2557
2884
  await new Promise((r) => setTimeout(r, 0));
@@ -2566,7 +2893,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
2566
2893
  if (!errorOf.has(doc.docId)) continue;
2567
2894
  failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
2568
2895
  }
2569
- return { succeeded, touchedVectorFields };
2896
+ return { succeeded, touchedVectorFields, unrecorded };
2570
2897
  }
2571
2898
  function frozenSegmentOf(segment, members, options) {
2572
2899
  if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
@@ -2577,14 +2904,15 @@ function frozenSegmentOf(segment, members, options) {
2577
2904
  segment.segmentId
2578
2905
  );
2579
2906
  }
2580
- async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
2907
+ async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
2581
2908
  if (admitted.length === 0) {
2582
- return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set() };
2909
+ return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] };
2583
2910
  }
2911
+ const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
2584
2912
  const manager = ctx.requireManager(indexName);
2585
2913
  const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
2586
2914
  if (admitted.length < MIN_DOCUMENTS_FOR_SEGMENTS || workers <= 0) {
2587
- return applyIndividually(ctx, indexName, admitted, options, failed);
2915
+ return applyIndividually();
2588
2916
  }
2589
2917
  const docIds = admitted.map((doc) => doc.docId);
2590
2918
  const partitionDocuments = admitted.map((doc) => doc.partitionDoc);
@@ -2597,7 +2925,7 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
2597
2925
  built = null;
2598
2926
  }
2599
2927
  if (built === null) {
2600
- return applyIndividually(ctx, indexName, admitted, options, failed);
2928
+ return applyIndividually();
2601
2929
  }
2602
2930
  const positions = new Map(admitted.map((document, index) => [document.docId, index]));
2603
2931
  const used = /* @__PURE__ */ new Set();
@@ -2606,16 +2934,15 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
2606
2934
  const members = [];
2607
2935
  for (const docId of segment.docIds) {
2608
2936
  const position = positions.get(docId);
2609
- if (position === void 0 || used.has(position))
2610
- return applyIndividually(ctx, indexName, admitted, options, failed);
2937
+ if (position === void 0 || used.has(position)) return applyIndividually();
2611
2938
  used.add(position);
2612
2939
  members.push(position);
2613
2940
  }
2614
2941
  memberIndexes.push(members);
2615
2942
  }
2616
- if (used.size !== admitted.length) return applyIndividually(ctx, indexName, admitted, options, failed);
2943
+ if (used.size !== admitted.length) return applyIndividually();
2617
2944
  if (ctx.isRebalancing(indexName)) {
2618
- return applyIndividually(ctx, indexName, admitted, options, failed);
2945
+ return applyIndividually();
2619
2946
  }
2620
2947
  const frozen = [];
2621
2948
  for (let i = 0; i < built.length; i++) {
@@ -2624,149 +2951,65 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
2624
2951
  memberIndexes[i].map((m) => admitted[m]),
2625
2952
  options
2626
2953
  );
2627
- if (segment === null) return applyIndividually(ctx, indexName, admitted, options, failed);
2954
+ if (segment === null) return applyIndividually();
2628
2955
  frozen.push(segment);
2629
2956
  }
2630
2957
  for (const doc of admitted) {
2631
- if (manager.has(doc.docId)) return applyIndividually(ctx, indexName, admitted, options, failed);
2958
+ if (manager.has(doc.docId)) return applyIndividually();
2632
2959
  }
2633
- return recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, failed);
2960
+ return recordMergedDocuments(
2961
+ ctx,
2962
+ indexName,
2963
+ partitionCount,
2964
+ built,
2965
+ frozen,
2966
+ memberIndexes,
2967
+ admitted,
2968
+ options,
2969
+ hold,
2970
+ failed
2971
+ );
2634
2972
  }
2635
- async function insertBatchViaSegments(ctx, indexName, documents, options) {
2973
+ async function insertBatchViaSegments(ctx, indexName, documents, options, hold) {
2636
2974
  const failed = [];
2637
- const admitted = await admitBatchDocuments(ctx, indexName, documents, failed);
2638
- const outcome = await ingestAdmitted(ctx, indexName, admitted, options, failed);
2639
- const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
2640
- for (const fieldPath of outcome.touchedVectorFields) {
2641
- vecIndexes.get(fieldPath)?.scheduleBuild();
2642
- }
2643
- ctx.checkWatermark(indexName, outcome.succeeded);
2644
- ctx.checkHeapPressure(indexName);
2645
- await ctx.orchestrator.scaleOutReadyIndexes();
2646
- if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2975
+ const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents, failed, hold);
2976
+ const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
2977
+ if (outcome.unrecorded.length > 0) {
2978
+ const unrecorded = outcome.unrecorded.map((doc) => doc.document);
2979
+ const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
2980
+ outcome.succeeded.push(...rest.succeeded);
2981
+ for (const fieldPath of rest.touchedVectorFields) outcome.touchedVectorFields.add(fieldPath);
2982
+ }
2983
+ await finishInsertBatch(ctx, indexName, outcome, options);
2647
2984
  return { succeeded: outcome.succeeded, failed };
2648
2985
  }
2649
2986
 
2650
2987
  // src/engine/mutations/insert-batch.ts
2651
- async function insertDocumentBatch(ctx, indexName, documents, options) {
2988
+ async function insertDocumentBatch(ctx, indexName, documents, options, hold) {
2652
2989
  ctx.guardShutdown();
2653
- const entry = ctx.requireIndex(indexName);
2990
+ ctx.requireIndex(indexName);
2654
2991
  validateBatchList(documents, "documents");
2655
2992
  await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length);
2656
2993
  const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
2657
2994
  if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
2658
- return insertBatchViaSegments(ctx, indexName, documents, options);
2995
+ return insertBatchViaSegments(ctx, indexName, documents, options, hold);
2659
2996
  }
2660
- const succeeded = [];
2661
2997
  const failed = [];
2662
- const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
2663
- const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
2664
- const required = entry.config.required;
2665
- const batchManager = ctx.requireManager(indexName);
2666
- const batchVecIndexes = batchManager.getVectorIndexes();
2667
- const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
2668
- const touchedVectorFields = /* @__PURE__ */ new Set();
2669
- for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
2670
- if (ctx.abortController.signal.aborted) break;
2671
- const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
2672
- const chunkFailedIndexes = /* @__PURE__ */ new Set();
2673
- if (required && required.length > 0) {
2674
- collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
2675
- }
2676
- await embedChunkDocuments(
2677
- entry,
2678
- documents,
2679
- chunkStart,
2680
- chunkEnd,
2681
- ctx.abortController.signal,
2682
- chunkFailedIndexes,
2683
- failed
2684
- );
2685
- async function applyPrepared(prepared2) {
2686
- const applications = await applyInsertChunk(ctx, indexName, prepared2, options);
2687
- for (let i = 0; i < prepared2.length; i++) {
2688
- const doc = prepared2[i];
2689
- const application = applications[i];
2690
- if (application.status === "skipped") continue;
2691
- if (application.status === "failed") {
2692
- failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
2693
- continue;
2694
- }
2695
- if (application.status === "buffered") {
2696
- succeeded.push(doc.docId);
2697
- continue;
2698
- }
2699
- for (const fieldPath of doc.extractedVectors.keys()) {
2700
- touchedVectorFields.add(fieldPath);
2701
- }
2702
- if (hasAfterHook) {
2703
- try {
2704
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
2705
- } catch (err) {
2706
- console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
2707
- }
2708
- }
2709
- succeeded.push(doc.docId);
2710
- }
2711
- }
2712
- const prepared = [];
2713
- for (let i = chunkStart; i < chunkEnd; i++) {
2714
- if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2715
- if (ctx.abortController.signal.aborted) break;
2716
- if (chunkFailedIndexes.has(i)) continue;
2717
- const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
2718
- try {
2719
- const sizing = requireDocumentObject(documents[i]);
2720
- if (sizing !== null) await sizing;
2721
- validateDocId(batchDocId);
2722
- if (hasBeforeHook) {
2723
- await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
2724
- }
2725
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
2726
- documents[i],
2727
- batchVectorFieldPaths
2728
- );
2729
- if (extractedVectors.size > 0) {
2730
- validateVectorDimensions(extractedVectors, batchVecIndexes);
2731
- }
2732
- admitInsert(ctx, indexName, batchManager, batchDocId);
2733
- prepared.push({
2734
- docId: batchDocId,
2735
- document: documents[i],
2736
- partitionDoc,
2737
- extractedVectors
2738
- });
2739
- } catch (err) {
2740
- failed.push({ docId: batchDocId, error: asBatchInsertError(err) });
2741
- }
2742
- }
2743
- if (prepared.length > 0) await applyPrepared(prepared);
2744
- if (chunkEnd < documents.length) {
2745
- ctx.checkHeapPressure(indexName);
2746
- await new Promise((r) => setTimeout(r, 0));
2747
- }
2748
- }
2749
- for (const fieldPath of touchedVectorFields) {
2750
- const vecIndex = batchVecIndexes.get(fieldPath);
2751
- if (vecIndex) {
2752
- vecIndex.scheduleBuild();
2753
- }
2754
- }
2755
- ctx.checkWatermark(indexName, succeeded);
2756
- ctx.checkHeapPressure(indexName);
2757
- await ctx.orchestrator.scaleOutReadyIndexes();
2758
- if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2759
- return { succeeded, failed };
2998
+ const inserts = await insertInChunks(ctx, indexName, documents, options, hold, true, failed);
2999
+ await finishInsertBatch(ctx, indexName, inserts, options);
3000
+ return { succeeded: inserts.succeeded, failed };
2760
3001
  }
2761
3002
 
2762
3003
  // src/engine/mutations/remove.ts
2763
3004
  function asRemoveError(err) {
2764
3005
  return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
2765
3006
  }
2766
- async function prepareRemoval(ctx, indexName, docId) {
3007
+ function validateRemoval(ctx, docId) {
2767
3008
  ctx.guardShutdown();
2768
3009
  validateDocId(docId);
2769
- await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
3010
+ }
3011
+ function prepareRemoval(ctx, indexName, docId) {
3012
+ ctx.guardShutdown();
2770
3013
  const rebalancing = ctx.isRebalancing(indexName);
2771
3014
  if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
2772
3015
  throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
@@ -2816,20 +3059,31 @@ function applyOfRemoval(ctx, indexName, prepared) {
2816
3059
  };
2817
3060
  return () => manager.publication.writeNow(indexChangeOutsideExternalWaits(removeFromIndex));
2818
3061
  }
2819
- async function settleRemoval(ctx, indexName, prepared) {
3062
+ async function settleRemovals(ctx, indexName, docIds, hold) {
3063
+ if (docIds.length === 0) return;
2820
3064
  const settling = ctx.orchestrator.settlePublications(indexName, "none");
2821
3065
  if (settling !== void 0) await settling;
2822
- const { docId } = prepared;
2823
- try {
2824
- await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
2825
- } catch (err) {
2826
- console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
2827
- }
3066
+ if (!ctx.pluginRegistry.hasHooks("afterRemove")) return;
3067
+ await runOutside(hold, async () => {
3068
+ for (const docId of docIds) {
3069
+ try {
3070
+ await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
3071
+ } catch (err) {
3072
+ console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
3073
+ }
3074
+ }
3075
+ });
2828
3076
  }
2829
- async function removeDocument2(ctx, indexName, docId, options) {
3077
+ async function removeDocument2(ctx, indexName, docId, options, hold) {
2830
3078
  ctx.guardShutdown();
2831
3079
  ctx.requireIndex(indexName);
2832
- const prepared = await prepareRemoval(ctx, indexName, docId);
3080
+ validateRemoval(ctx, docId);
3081
+ if (ctx.pluginRegistry.hasHooks("beforeRemove")) {
3082
+ await runOutside(hold, async () => {
3083
+ await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
3084
+ });
3085
+ }
3086
+ const prepared = prepareRemoval(ctx, indexName, docId);
2833
3087
  const apply = applyOfRemoval(ctx, indexName, prepared);
2834
3088
  if (ctx.durability) {
2835
3089
  try {
@@ -2843,25 +3097,52 @@ async function removeDocument2(ctx, indexName, docId, options) {
2843
3097
  } else {
2844
3098
  await apply();
2845
3099
  }
2846
- if (!prepared.buffered) await settleRemoval(ctx, indexName, prepared);
3100
+ if (!prepared.buffered) await settleRemovals(ctx, indexName, [docId], hold);
2847
3101
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2848
3102
  }
2849
- async function removeDocumentBatch(ctx, indexName, docIds, options) {
3103
+ async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold) {
3104
+ const refused = /* @__PURE__ */ new Map();
3105
+ for (let i = chunkStart; i < chunkEnd; i++) {
3106
+ try {
3107
+ validateRemoval(ctx, docIds[i]);
3108
+ } catch (err) {
3109
+ refused.set(i, err);
3110
+ }
3111
+ }
3112
+ if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
3113
+ await runOutside(hold, async () => {
3114
+ for (let i = chunkStart; i < chunkEnd; i++) {
3115
+ if (refused.has(i)) continue;
3116
+ try {
3117
+ await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
3118
+ } catch (err) {
3119
+ refused.set(i, err);
3120
+ }
3121
+ }
3122
+ });
3123
+ return refused;
3124
+ }
3125
+ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
2850
3126
  ctx.guardShutdown();
2851
3127
  ctx.requireIndex(indexName);
2852
3128
  validateBatchList(docIds, "document IDs");
2853
3129
  const succeeded = [];
2854
3130
  const failed = [];
2855
- const manager = ctx.requireManager(indexName);
2856
- await manager.publication.write(() => manager.beginBatchRemove());
3131
+ const first = ctx.requireManager(indexName);
3132
+ await first.publication.write(() => first.beginBatchRemove());
3133
+ let manager = first;
2857
3134
  try {
2858
3135
  for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
2859
3136
  const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
3137
+ const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
3138
+ manager = await batchManagerAfterHooks(ctx, indexName, manager);
3139
+ const removed = [];
2860
3140
  const prepared = [];
2861
3141
  for (let i = chunkStart; i < chunkEnd; i++) {
2862
3142
  if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2863
3143
  try {
2864
- prepared.push(await prepareRemoval(ctx, indexName, docIds[i]));
3144
+ if (refused.has(i)) throw refused.get(i);
3145
+ prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
2865
3146
  } catch (err) {
2866
3147
  failed.push({ docId: docIds[i], error: asRemoveError(err) });
2867
3148
  }
@@ -2891,23 +3172,33 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
2891
3172
  continue;
2892
3173
  }
2893
3174
  removal.releaseVectorHolds?.();
2894
- if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
3175
+ if (!removal.buffered) removed.push(removal.docId);
2895
3176
  succeeded.push(removal.docId);
2896
3177
  }
2897
3178
  } finally {
2898
3179
  for (const removal of prepared) removal.releaseVectorHolds?.();
2899
3180
  }
3181
+ await settleRemovals(ctx, indexName, removed, hold);
3182
+ manager = await batchManagerAfterHooks(ctx, indexName, manager);
2900
3183
  if (chunkEnd < docIds.length) {
2901
3184
  await new Promise((r) => setTimeout(r, 0));
2902
3185
  }
2903
3186
  }
2904
3187
  } finally {
2905
- await manager.publication.write(() => manager.endBatchRemove());
3188
+ const ending = manager;
3189
+ await ending?.publication.write(() => ending.endBatchRemove());
2906
3190
  }
2907
3191
  ctx.checkHeapPressure(indexName);
2908
3192
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2909
3193
  return { succeeded, failed };
2910
3194
  }
3195
+ async function batchManagerAfterHooks(ctx, indexName, batchManager) {
3196
+ const current = ctx.executor.getManager(indexName);
3197
+ if (current === batchManager) return batchManager;
3198
+ await batchManager?.publication.write(() => batchManager.endBatchRemove());
3199
+ await current?.publication.write(() => current.beginBatchRemove());
3200
+ return current;
3201
+ }
2911
3202
 
2912
3203
  // src/engine/mutations/update.ts
2913
3204
  function prepareUpdatePartitionDoc(document, extractedVectors) {
@@ -2941,59 +3232,47 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
2941
3232
  function asUpdateError(err) {
2942
3233
  return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
2943
3234
  }
2944
- async function settleUpdate(ctx, indexName, prepared) {
3235
+ async function runAfterUpdateHooks(ctx, indexName, updates) {
3236
+ for (const update of updates) {
3237
+ try {
3238
+ await ctx.pluginRegistry.runHook("afterUpdate", {
3239
+ indexName,
3240
+ docId: update.docId,
3241
+ oldDocument: update.oldDocument ?? {},
3242
+ newDocument: update.document
3243
+ });
3244
+ } catch (err) {
3245
+ console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
3246
+ }
3247
+ }
3248
+ }
3249
+ async function settleUpdates(ctx, indexName, updates, hold) {
3250
+ if (updates.length === 0) return;
2945
3251
  await ctx.orchestrator.settlePublications(indexName);
2946
- try {
2947
- await ctx.pluginRegistry.runHook("afterUpdate", {
2948
- indexName,
2949
- docId: prepared.docId,
2950
- oldDocument: prepared.oldDocument ?? {},
2951
- newDocument: prepared.document
2952
- });
2953
- } catch (err) {
2954
- console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
3252
+ if (ctx.pluginRegistry.hasHooks("afterUpdate")) {
3253
+ await runOutside(hold, () => runAfterUpdateHooks(ctx, indexName, updates));
2955
3254
  }
2956
- const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
2957
- for (const [fieldPath, vec] of prepared.extractedVectors) {
2958
- if (vec === null) continue;
2959
- vecIndexes.get(fieldPath)?.scheduleBuild();
3255
+ const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
3256
+ if (vecIndexes === void 0) return;
3257
+ for (const update of updates) {
3258
+ for (const [fieldPath, vec] of update.extractedVectors) {
3259
+ if (vec !== null) vecIndexes.get(fieldPath)?.scheduleBuild();
3260
+ }
2960
3261
  }
2961
3262
  }
2962
- async function prepareUpdate(ctx, indexName, entry, docId, document) {
3263
+ function updatePreparationOf(ctx, indexName, docId, document) {
2963
3264
  ctx.guardShutdown();
2964
3265
  validateDocId(docId);
2965
3266
  requireMatchingDocId(docId, document);
2966
- const manager = ctx.requireManager(indexName);
2967
- if (ctx.pluginRegistry.hasHooks("beforeUpdate")) {
2968
- await ctx.pluginRegistry.runHook("beforeUpdate", {
2969
- indexName,
2970
- docId,
2971
- oldDocument: manager.get(docId) ?? {},
2972
- newDocument: document
2973
- });
2974
- }
2975
- if (entry.config.required && entry.config.required.length > 0) {
2976
- validateRequiredFields(document, entry.config.required);
2977
- }
2978
- if (entry.config.embedding) {
2979
- const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
2980
- if (entry.embeddingAdapter) {
2981
- await embedDocumentFields(
2982
- document,
2983
- entry.config.embedding,
2984
- entry.embeddingAdapter,
2985
- ctx.abortController.signal
2986
- );
2987
- } else {
2988
- assertDocumentCarriesMappedVectors(
2989
- document,
2990
- entry.config.embedding,
2991
- entry.embeddingAdapterName
2992
- );
2993
- }
2994
- }
3267
+ if (!ctx.pluginRegistry.hasHooks("beforeUpdate")) return { document, beforeWrite: null, failure: void 0 };
3268
+ const oldDocument = ctx.requireManager(indexName).get(docId) ?? {};
3269
+ const beforeWrite = () => ctx.pluginRegistry.runHook("beforeUpdate", { indexName, docId, oldDocument, newDocument: document });
3270
+ return { document, beforeWrite, failure: void 0 };
3271
+ }
3272
+ async function finishUpdate(ctx, indexName, entry, docId, document) {
2995
3273
  const sizing = requireDocumentWithinSizeLimit(document);
2996
3274
  if (sizing !== null) await sizing;
3275
+ const manager = ctx.requireManager(indexName);
2997
3276
  const vecIndexes = manager.getVectorIndexes();
2998
3277
  const extractedVectors = /* @__PURE__ */ new Map();
2999
3278
  if (vecIndexes.size > 0) {
@@ -3087,10 +3366,13 @@ function applyOfUpdate(ctx, indexName, prepared) {
3087
3366
  };
3088
3367
  return () => manager.publication.write(indexChangeOutsideExternalWaits(updateInIndex));
3089
3368
  }
3090
- async function updateDocument(ctx, indexName, docId, document, options) {
3369
+ async function updateDocument(ctx, indexName, docId, document, options, hold) {
3091
3370
  ctx.guardShutdown();
3092
- const entry = ctx.requireIndex(indexName);
3093
- const prepared = await prepareUpdate(ctx, indexName, entry, docId, document);
3371
+ ctx.requireIndex(indexName);
3372
+ const preparation = updatePreparationOf(ctx, indexName, docId, document);
3373
+ const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
3374
+ if (preparation.failure !== void 0) throw preparation.failure;
3375
+ const prepared = await finishUpdate(ctx, indexName, entry, docId, document);
3094
3376
  const apply = applyOfUpdate(ctx, indexName, prepared);
3095
3377
  if (ctx.durability) {
3096
3378
  try {
@@ -3113,10 +3395,12 @@ async function updateDocument(ctx, indexName, docId, document, options) {
3113
3395
  } else {
3114
3396
  await apply();
3115
3397
  }
3116
- if (!prepared.buffered) await settleUpdate(ctx, indexName, prepared);
3398
+ if (!prepared.buffered) await settleUpdates(ctx, indexName, [prepared], hold);
3117
3399
  ctx.checkHeapPressure(indexName);
3118
3400
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
3119
3401
  }
3402
+
3403
+ // src/engine/mutations/update-batch.ts
3120
3404
  function reportedDocIdOf(update) {
3121
3405
  const docId = update !== null && typeof update === "object" ? update.docId : void 0;
3122
3406
  return typeof docId === "string" ? docId : "";
@@ -3133,29 +3417,47 @@ function requireUpdateEntry(update) {
3133
3417
  "Each entry of a batch update must be an object whose docId is a string and whose document is an object"
3134
3418
  );
3135
3419
  }
3136
- async function updateDocumentBatch(ctx, indexName, updates, options) {
3420
+ async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed) {
3421
+ const preparations = [];
3422
+ for (let i = chunkStart; i < chunkEnd; i++) {
3423
+ const update = updates[i];
3424
+ const docId = reportedDocIdOf(update);
3425
+ try {
3426
+ const { document } = requireUpdateEntry(update);
3427
+ preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
3428
+ } catch (err) {
3429
+ failed.push({ docId, error: asUpdateError(err) });
3430
+ }
3431
+ }
3432
+ const { entry } = await prepareDocumentWrites(
3433
+ ctx,
3434
+ indexName,
3435
+ preparations.map((pending) => pending.preparation),
3436
+ hold,
3437
+ false
3438
+ );
3439
+ const prepared = [];
3440
+ for (const { docId, preparation } of preparations) {
3441
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
3442
+ try {
3443
+ if (preparation.failure !== void 0) throw preparation.failure;
3444
+ prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
3445
+ } catch (err) {
3446
+ failed.push({ docId, error: asUpdateError(err) });
3447
+ }
3448
+ }
3449
+ return prepared;
3450
+ }
3451
+ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
3137
3452
  ctx.guardShutdown();
3138
- const entry = ctx.requireIndex(indexName);
3453
+ ctx.requireIndex(indexName);
3139
3454
  validateBatchList(updates, "updates");
3140
3455
  const succeeded = [];
3141
3456
  const failed = [];
3142
- const updateBatchManager = ctx.requireManager(indexName);
3143
- const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
3144
3457
  const touchedVectorFields = /* @__PURE__ */ new Set();
3145
3458
  for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
3146
3459
  const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
3147
- const prepared = [];
3148
- for (let i = chunkStart; i < chunkEnd; i++) {
3149
- if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
3150
- const update = updates[i];
3151
- const docId = reportedDocIdOf(update);
3152
- try {
3153
- const { document } = requireUpdateEntry(update);
3154
- prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document));
3155
- } catch (err) {
3156
- failed.push({ docId, error: asUpdateError(err) });
3157
- }
3158
- }
3460
+ const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
3159
3461
  const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
3160
3462
  let failures;
3161
3463
  try {
@@ -3182,6 +3484,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
3182
3484
  } finally {
3183
3485
  for (const update of prepared) update.releaseVectorHolds?.();
3184
3486
  }
3487
+ const settled = [];
3185
3488
  for (let i = 0; i < prepared.length; i++) {
3186
3489
  const update = prepared[i];
3187
3490
  const failure = failures[i];
@@ -3189,19 +3492,21 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
3189
3492
  failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
3190
3493
  continue;
3191
3494
  }
3192
- if (!update.buffered) await settleUpdate(ctx, indexName, update);
3495
+ if (!update.buffered) settled.push(update);
3193
3496
  for (const [fieldPath, vec] of update.extractedVectors) {
3194
3497
  if (vec !== null) touchedVectorFields.add(fieldPath);
3195
3498
  }
3196
3499
  succeeded.push(update.docId);
3197
3500
  }
3501
+ await settleUpdates(ctx, indexName, settled, hold);
3198
3502
  if (chunkEnd < updates.length) {
3199
3503
  ctx.checkHeapPressure(indexName);
3200
3504
  await new Promise((r) => setTimeout(r, 0));
3201
3505
  }
3202
3506
  }
3507
+ const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
3203
3508
  for (const fieldPath of touchedVectorFields) {
3204
- updateBatchVecIndexes.get(fieldPath)?.scheduleBuild();
3509
+ vectorIndexes?.get(fieldPath)?.scheduleBuild();
3205
3510
  }
3206
3511
  ctx.checkHeapPressure(indexName);
3207
3512
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
@@ -3247,4 +3552,4 @@ async function updateEnginePartitionConfig(core, indexName, partitionConfig) {
3247
3552
  core.watermarkNotifier.check(indexName);
3248
3553
  }
3249
3554
 
3250
- export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
3555
+ export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, prepareDocumentWrite, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };