@delali/narsil 0.3.1-next.181 → 0.3.1-next.184

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-CPSHS46N.mjs} +139 -177
  2. package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
  3. package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
  4. package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-ZTBHYQMN.mjs} +621 -314
  5. package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
  6. package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
  7. package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
  8. package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-4XMCITJ6.mjs} +886 -259
  9. package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
  10. package/dist/{chunk-KHJPMZE2.mjs → chunk-2KCCQK4R.mjs} +1 -1
  11. package/dist/{chunk-2KWTQ2P7.mjs → chunk-2YWWHMVC.mjs} +2 -2
  12. package/dist/{chunk-Y27E2MGC.mjs → chunk-3D4IMFSE.mjs} +2 -2
  13. package/dist/chunk-3ECIRFS5.mjs +61 -0
  14. package/dist/{chunk-XFOL77WH.mjs → chunk-4JBTALUC.mjs} +1 -1
  15. package/dist/{chunk-ISJ35523.mjs → chunk-5BUEBIXR.mjs} +2 -2
  16. package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
  17. package/dist/{chunk-UISV46WX.mjs → chunk-7WXGUTWK.mjs} +3 -3
  18. package/dist/{chunk-QFJMRB7D.mjs → chunk-A3OAFUZI.mjs} +1 -1
  19. package/dist/{chunk-GO6TRSJ6.mjs → chunk-BVZWWMOU.mjs} +1 -1
  20. package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
  21. package/dist/{chunk-XQQKETAF.mjs → chunk-CODGHJVW.mjs} +4 -4
  22. package/dist/{chunk-34IOOYXA.mjs → chunk-CYCVPTTR.mjs} +2 -2
  23. package/dist/{chunk-26E5MO3Y.mjs → chunk-DU23J5JG.mjs} +5 -5
  24. package/dist/{chunk-JCWC75NH.mjs → chunk-EDEE7ETQ.mjs} +1 -1
  25. package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
  26. package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
  27. package/dist/{chunk-BIIROQ5Q.mjs → chunk-FCQMZDRI.mjs} +2 -2
  28. package/dist/{chunk-YXZDTKQS.mjs → chunk-G3XRHAJA.mjs} +2 -2
  29. package/dist/{chunk-XTSFRWAS.mjs → chunk-GGIW6INA.mjs} +7 -6
  30. package/dist/{chunk-R7PLSVK3.mjs → chunk-H4QYQ5DE.mjs} +1 -1
  31. package/dist/{chunk-A4XYDUVN.mjs → chunk-HQPK73EW.mjs} +3 -3
  32. package/dist/{chunk-YPV2L7M5.mjs → chunk-II6XF6M4.mjs} +5 -5
  33. package/dist/{chunk-F3FDHXCJ.mjs → chunk-LR6JJVML.mjs} +251 -50
  34. package/dist/{chunk-ZB5X5WAK.mjs → chunk-M5I64AQW.mjs} +11 -6
  35. package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
  36. package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
  37. package/dist/{chunk-XITVDRRH.mjs → chunk-O337MDRJ.mjs} +6 -6
  38. package/dist/{chunk-XNFBYAXI.mjs → chunk-SEM3PBRH.mjs} +1 -1
  39. package/dist/{chunk-2C3BXZGA.mjs → chunk-SL53DHVE.mjs} +814 -539
  40. package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
  41. package/dist/{chunk-LOQHA6QK.mjs → chunk-URVH6BLO.mjs} +2 -2
  42. package/dist/{chunk-JDIATDPY.mjs → chunk-VYSWVQCE.mjs} +1 -1
  43. package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
  44. package/dist/{chunk-7FC2ZILY.mjs → chunk-WSIU6YXQ.mjs} +67 -78
  45. package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
  46. package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
  47. package/dist/client.d.ts +6 -6
  48. package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
  49. package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
  50. package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
  51. package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-37IH3ISS.mjs} +5 -4
  52. package/dist/distribution.d.ts +4 -4
  53. package/dist/distribution.mjs +398 -144
  54. package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
  55. package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
  56. package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
  57. package/dist/index.browser.mjs +1 -1
  58. package/dist/index.d.ts +4 -4
  59. package/dist/index.mjs +5 -4
  60. package/dist/index.react-native.mjs +4 -3
  61. package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-PU2XT527.mjs} +15 -14
  62. package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
  63. package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-YIVJMQ4D.mjs} +9 -6
  64. package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
  65. package/dist/partial-words.browser.mjs +1 -1
  66. package/dist/partial-words.mjs +5 -4
  67. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
  68. package/dist/persistence/durability/checkpoint-worker.mjs +28 -16
  69. package/dist/{query-actions-2JOLPELC.mjs → query-actions-7EOKT3WD.mjs} +5 -4
  70. package/dist/query-suggestions/merge-worker.mjs +8 -7
  71. package/dist/query-suggestions.browser.mjs +1 -1
  72. package/dist/query-suggestions.d.ts +1 -1
  73. package/dist/query-suggestions.mjs +6 -5
  74. package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-7TPO4FLE.mjs} +2 -2
  75. package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
  76. package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-CBU4M3EU.mjs} +813 -538
  77. package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-IEY4DARA.mjs} +1 -1
  78. package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
  79. package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
  80. package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-T5HF3FW3.mjs} +1 -1
  81. package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-ZQPSXN6C.mjs} +1 -1
  82. package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-ZUFOYZ5P.mjs} +1 -1
  83. package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
  84. package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
  85. package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
  86. package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
  87. package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-QETIM6BI.mjs} +7 -4
  88. package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
  89. package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
  90. package/dist/{runtime-4GKUIAKV.mjs → react-native/chunks/runtime-UOHD2XUJ.mjs} +23 -3
  91. package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
  92. package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
  93. package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-LNOLKWZB.mjs} +895 -267
  94. package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
  95. package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
  96. package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
  97. package/dist/react-native/partial-words.mjs +4 -3
  98. package/dist/react-native/query-suggestions.mjs +4 -3
  99. package/dist/react-native/sync.mjs +4 -3
  100. package/dist/react-native/synonyms.mjs +4 -3
  101. package/dist/react.d.ts +7 -7
  102. package/dist/{rerank-E2HFKKGJ.mjs → rerank-KTV5L7MZ.mjs} +5 -4
  103. package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
  104. package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
  105. package/dist/{react-native/chunks/runtime-BUCSAK4U.mjs → runtime-USQYKN3I.mjs} +23 -3
  106. package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
  107. package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
  108. package/dist/server.d.ts +8 -8
  109. package/dist/server.mjs +23 -22
  110. package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-ZI3H2WJY.mjs} +595 -164
  111. package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
  112. package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
  113. package/dist/sync/filesystem.d.ts +2 -2
  114. package/dist/sync/filesystem.mjs +11 -10
  115. package/dist/sync/indexeddb.d.ts +2 -2
  116. package/dist/sync.browser.mjs +1 -1
  117. package/dist/sync.d.ts +3 -3
  118. package/dist/sync.mjs +5 -4
  119. package/dist/synonyms.browser.mjs +1 -1
  120. package/dist/synonyms.mjs +5 -4
  121. package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
  122. package/dist/{thread-OWXVP2ZA.mjs → thread-BJXE77AI.mjs} +14 -13
  123. package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
  124. package/dist/vector/search-worker.mjs +5 -4
  125. package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
  126. package/dist/workers/entry.mjs +11 -10
  127. package/openapi.json +32 -0
  128. package/package.json +7 -7
  129. package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
  130. package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
@@ -1628,19 +1628,17 @@ function removeDocument(input, generation, ordinal, fields) {
1628
1628
  // src/engine/optional-writes-host.browser.ts
1629
1629
  var BATCH_CHUNK_SIZE;
1630
1630
  var ErrorCodes;
1631
+ var MAX_CALLS_ACROSS_INDEX_CHANGES;
1631
1632
  var MIN_DOCUMENTS_FOR_SEGMENTS;
1632
1633
  var NarsilError;
1633
1634
  var admitBufferedUpsert;
1634
1635
  var admitInsert;
1635
1636
  var arrayPositionsAreStale;
1636
- var asBatchInsertError;
1637
1637
  var awaitWriteVisibility;
1638
+ var capsEachPartition;
1638
1639
  var changeIndexOutsideExternalWaits;
1639
- var collectRequiredFieldFailures;
1640
1640
  var createFrozenSegment;
1641
- var createPendingAdmissions;
1642
1641
  var deleteNestedValue;
1643
- var embedChunkDocuments;
1644
1642
  var emitSyncChange;
1645
1643
  var extractVectorFromDoc;
1646
1644
  var fnv1a;
@@ -1662,7 +1660,6 @@ var prepareDocumentVectors;
1662
1660
  var prepareStoredSynonyms;
1663
1661
  var providedDocId;
1664
1662
  var readHeapStatistics;
1665
- var recordPendingAdmission;
1666
1663
  var removeDocumentVectors;
1667
1664
  var requireDocumentObject;
1668
1665
  var requireDocumentWithinSizeLimit;
@@ -1703,19 +1700,17 @@ function bindOptionalWritesHost(host, segmentCodec) {
1703
1700
  ({
1704
1701
  BATCH_CHUNK_SIZE,
1705
1702
  ErrorCodes,
1703
+ MAX_CALLS_ACROSS_INDEX_CHANGES,
1706
1704
  MIN_DOCUMENTS_FOR_SEGMENTS,
1707
1705
  NarsilError,
1708
1706
  admitBufferedUpsert,
1709
1707
  admitInsert,
1710
1708
  arrayPositionsAreStale,
1711
- asBatchInsertError,
1712
1709
  awaitWriteVisibility,
1710
+ capsEachPartition,
1713
1711
  changeIndexOutsideExternalWaits,
1714
- collectRequiredFieldFailures,
1715
1712
  createFrozenSegment,
1716
- createPendingAdmissions,
1717
1713
  deleteNestedValue,
1718
- embedChunkDocuments,
1719
1714
  emitSyncChange,
1720
1715
  extractVectorFromDoc,
1721
1716
  fnv1a,
@@ -1737,7 +1732,6 @@ function bindOptionalWritesHost(host, segmentCodec) {
1737
1732
  prepareStoredSynonyms,
1738
1733
  providedDocId,
1739
1734
  readHeapStatistics,
1740
- recordPendingAdmission,
1741
1735
  removeDocumentVectors,
1742
1736
  requireDocumentObject,
1743
1737
  requireDocumentWithinSizeLimit,
@@ -1963,6 +1957,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1963
1957
  throw recoveryFailure(metadata, err);
1964
1958
  }
1965
1959
  deps.indexRegistry.set(metadata.indexName, {
1960
+ storedSizes: deps.indexRegistry.get(metadata.indexName)?.storedSizes,
1966
1961
  config: indexConfig,
1967
1962
  language,
1968
1963
  embeddingAdapter: embedding.adapter,
@@ -1971,6 +1966,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1971
1966
  indexUuid: metadata.indexUuid ?? null,
1972
1967
  heldPartitions: metadata.heldPartitions ?? null,
1973
1968
  documentCount: metadata.documentCount ?? 0,
1969
+ savedBytes: metadata.savedBytes,
1974
1970
  partitionCount,
1975
1971
  analysisRevision: metadata.analysisRevision ?? null,
1976
1972
  arrayPositionGap: metadata.arrayPositionGap ?? null,
@@ -1993,6 +1989,326 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1993
1989
  }
1994
1990
  }
1995
1991
 
1992
+ // src/engine/mutations/batch-admissions.ts
1993
+ function createPendingAdmissions() {
1994
+ return { total: 0, byPartition: /* @__PURE__ */ new Map() };
1995
+ }
1996
+ function recordPendingAdmission(ctx, pending, manager, docId) {
1997
+ pending.total += 1;
1998
+ if (!capsEachPartition(ctx, manager)) return;
1999
+ const partitionId = manager.routePartition(docId);
2000
+ pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
2001
+ }
2002
+ function asBatchInsertError(err) {
2003
+ return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
2004
+ }
2005
+
2006
+ // src/engine/mutations/outside-hold.ts
2007
+ function runOutside(hold, work) {
2008
+ return hold === void 0 ? work() : hold.outside(work);
2009
+ }
2010
+
2011
+ // src/engine/mutations/write-preparation.ts
2012
+ function settingsOf(entry) {
2013
+ return { required: entry.config.required, embedding: entry.config.embedding, adapter: entry.embeddingAdapter };
2014
+ }
2015
+ function embedsThroughAdapter(entry) {
2016
+ return entry.config.embedding !== void 0 && entry.embeddingAdapter !== null;
2017
+ }
2018
+ function sourcesOf(sources) {
2019
+ return typeof sources === "string" ? [sources] : sources;
2020
+ }
2021
+ function sameMapping(left, right) {
2022
+ if (left === right) return true;
2023
+ const targets = Object.keys(left);
2024
+ if (targets.length !== Object.keys(right).length) return false;
2025
+ for (const target of targets) {
2026
+ if (!Object.hasOwn(right, target)) return false;
2027
+ const leftSources = sourcesOf(left[target]);
2028
+ const rightSources = sourcesOf(right[target]);
2029
+ if (leftSources.length !== rightSources.length) return false;
2030
+ if (leftSources.some((source, position) => source !== rightSources[position])) return false;
2031
+ }
2032
+ return true;
2033
+ }
2034
+ function embeddingFits(entry, embedding) {
2035
+ const embeddingConfig = entry.config.embedding;
2036
+ if (embeddingConfig === void 0 || entry.embeddingAdapter === null) return embedding === null;
2037
+ return embedding !== null && embedding.adapter === entry.embeddingAdapter && sameMapping(embedding.mapping, embeddingConfig.fields);
2038
+ }
2039
+ function embeddingOfDocuments(embedding, documents) {
2040
+ if (embedding === null) return null;
2041
+ const fields = /* @__PURE__ */ new Map();
2042
+ for (const document of documents) {
2043
+ const embedded = embedding.fields.get(document);
2044
+ if (embedded !== void 0) fields.set(document, embedded);
2045
+ }
2046
+ return { adapter: embedding.adapter, mapping: embedding.mapping, fields };
2047
+ }
2048
+ function writable(preparations) {
2049
+ return preparations.filter((preparation) => preparation.failure === void 0);
2050
+ }
2051
+ function failAll(preparations, failure) {
2052
+ for (const preparation of writable(preparations)) preparation.failure = failure;
2053
+ }
2054
+ function requireFieldsOf(required, preparations) {
2055
+ if (required === void 0 || required.length === 0) return;
2056
+ for (const preparation of writable(preparations)) {
2057
+ try {
2058
+ validateRequiredFields(preparation.document, required);
2059
+ } catch (err) {
2060
+ preparation.failure = err;
2061
+ }
2062
+ }
2063
+ }
2064
+ async function requireMappedVectors(entry, preparations) {
2065
+ const embeddingConfig = entry.config.embedding;
2066
+ if (embeddingConfig === void 0 || entry.embeddingAdapter !== null) return;
2067
+ const { assertDocumentCarriesMappedVectors } = await loadEmbedCode();
2068
+ for (const preparation of writable(preparations)) {
2069
+ try {
2070
+ assertDocumentCarriesMappedVectors(
2071
+ preparation.document,
2072
+ embeddingConfig,
2073
+ entry.embeddingAdapterName
2074
+ );
2075
+ } catch (err) {
2076
+ preparation.failure = err;
2077
+ }
2078
+ }
2079
+ }
2080
+ async function embedWith(settings, signal, preparations, earlier, together) {
2081
+ requireFieldsOf(settings.required, preparations);
2082
+ const { clearWriteEmbedding, embedDocumentsForWrite, lacksMappedVectors } = await loadEmbedCode();
2083
+ if (earlier !== null) clearWriteEmbedding(earlier);
2084
+ const { embedding: embeddingConfig, adapter } = settings;
2085
+ const embeddable = writable(preparations);
2086
+ const documentFailures = /* @__PURE__ */ new Map();
2087
+ if (embeddingConfig === void 0 || adapter === null || embeddable.length === 0) {
2088
+ return { embedding: null, called: false, documentFailures };
2089
+ }
2090
+ const fields = /* @__PURE__ */ new Map();
2091
+ const embedding = { adapter, mapping: embeddingConfig.fields, fields };
2092
+ const documents = embeddable.map((preparation) => preparation.document);
2093
+ if (!lacksMappedVectors(documents, embeddingConfig)) return { embedding, called: false, documentFailures };
2094
+ for (let chunkStart = 0; chunkStart < embeddable.length; chunkStart += BATCH_CHUNK_SIZE) {
2095
+ const chunk = embeddable.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
2096
+ const embedded = await embedDocumentsForWrite(
2097
+ chunk.map((preparation) => preparation.document),
2098
+ embeddingConfig,
2099
+ adapter,
2100
+ signal,
2101
+ together
2102
+ );
2103
+ for (const [document, embeddedFields] of embedded.embedding.fields) fields.set(document, embeddedFields);
2104
+ for (const [position, error] of embedded.failed) chunk[position].failure = error;
2105
+ for (const [position, error] of embedded.documentFailures) documentFailures.set(chunk[position], error);
2106
+ }
2107
+ return { embedding, called: true, documentFailures };
2108
+ }
2109
+ function settleDocumentFailures(entry, pass) {
2110
+ if (!embeddingFits(entry, pass.embedding)) return [...pass.documentFailures.keys()];
2111
+ for (const [preparation, failure] of pass.documentFailures) preparation.failure = failure;
2112
+ return [];
2113
+ }
2114
+ async function runBeforeWrites(preparations) {
2115
+ for (const preparation of preparations) {
2116
+ if (preparation.beforeWrite === null || preparation.failure !== void 0) continue;
2117
+ try {
2118
+ await preparation.beforeWrite();
2119
+ } catch (err) {
2120
+ preparation.failure = err;
2121
+ }
2122
+ }
2123
+ }
2124
+ async function embedThenRunHooks(settings, signal, preparations, earlier) {
2125
+ const pass = await embedWith(settings, signal, preparations, earlier, true);
2126
+ await runBeforeWrites(preparations.filter((preparation) => !pass.documentFailures.has(preparation)));
2127
+ return pass;
2128
+ }
2129
+ async function runHooksThenEmbedEach(ctx, indexName, preparations, earlier) {
2130
+ let settings = null;
2131
+ let embedding = null;
2132
+ let called = false;
2133
+ let uncleared = earlier;
2134
+ const documentFailures = /* @__PURE__ */ new Map();
2135
+ for (const preparation of preparations) {
2136
+ await runBeforeWrites([preparation]);
2137
+ settings ??= settingsOf(ctx.requireIndex(indexName));
2138
+ const pass = await embedWith(settings, ctx.abortController.signal, [preparation], uncleared, false);
2139
+ uncleared = null;
2140
+ called ||= pass.called;
2141
+ for (const [failed, failure] of pass.documentFailures) documentFailures.set(failed, failure);
2142
+ if (pass.embedding === null) continue;
2143
+ if (embedding === null) embedding = pass.embedding;
2144
+ else for (const [document, fields] of pass.embedding.fields) embedding.fields.set(document, fields);
2145
+ }
2146
+ return { embedding, called, documentFailures };
2147
+ }
2148
+ function adapterChangesRefusal(indexName, embeddings) {
2149
+ return new NarsilError(
2150
+ ErrorCodes.WORKER_BUSY,
2151
+ `Changes to index "${indexName}" replaced its embedding adapter after each of ${embeddings} embeddings of this write`,
2152
+ { indexName, embeddings }
2153
+ );
2154
+ }
2155
+ async function prepareDocumentWrites(ctx, indexName, preparations, hold, together, earlier = null) {
2156
+ let entry = ctx.requireIndex(indexName);
2157
+ let embedding = earlier;
2158
+ if (writable(preparations).length === 0) return { entry, embedding };
2159
+ const signal = ctx.abortController.signal;
2160
+ let embeddings = 0;
2161
+ let unhooked = [];
2162
+ const hooked = preparations.some((preparation) => preparation.beforeWrite !== null);
2163
+ if (hooked || embedsThroughAdapter(entry) && !embeddingFits(entry, embedding)) {
2164
+ if (together && !embedsThroughAdapter(entry)) {
2165
+ requireFieldsOf(entry.config.required, preparations);
2166
+ await requireMappedVectors(entry, preparations);
2167
+ }
2168
+ const settings = settingsOf(entry);
2169
+ const uncleared = embedding;
2170
+ const pass = await runOutside(
2171
+ hold,
2172
+ () => together ? embedThenRunHooks(settings, signal, preparations, uncleared) : runHooksThenEmbedEach(ctx, indexName, preparations, uncleared)
2173
+ );
2174
+ embedding = pass.embedding;
2175
+ if (pass.called) embeddings += 1;
2176
+ entry = ctx.requireIndex(indexName);
2177
+ unhooked = settleDocumentFailures(entry, pass);
2178
+ }
2179
+ while (writable(preparations).length > 0 && !embeddingFits(entry, embedding)) {
2180
+ if (embeddings >= MAX_CALLS_ACROSS_INDEX_CHANGES) {
2181
+ failAll(preparations, adapterChangesRefusal(indexName, embeddings));
2182
+ break;
2183
+ }
2184
+ const uncleared = embedding;
2185
+ const settings = settingsOf(entry);
2186
+ const hooking = unhooked;
2187
+ const pass = await runOutside(hold, async () => {
2188
+ const embedded = await embedWith(settings, signal, preparations, uncleared, together);
2189
+ await runBeforeWrites(hooking.filter((preparation) => !embedded.documentFailures.has(preparation)));
2190
+ return embedded;
2191
+ });
2192
+ embedding = pass.embedding;
2193
+ if (pass.called) embeddings += 1;
2194
+ entry = ctx.requireIndex(indexName);
2195
+ const stale = settleDocumentFailures(entry, pass);
2196
+ unhooked = hooking.filter((preparation) => stale.includes(preparation));
2197
+ }
2198
+ requireFieldsOf(entry.config.required, preparations);
2199
+ await requireMappedVectors(entry, preparations);
2200
+ return { entry, embedding };
2201
+ }
2202
+ async function prepareDocumentWrite(ctx, indexName, document, hold, beforeWrite) {
2203
+ const preparation = { document, beforeWrite, failure: void 0 };
2204
+ const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
2205
+ if (preparation.failure !== void 0) throw preparation.failure;
2206
+ return entry;
2207
+ }
2208
+
2209
+ // src/engine/mutations/insert-batch-admission.ts
2210
+ function pendingInsertOf(ctx, indexName, docId, document, hooked) {
2211
+ const pending = {
2212
+ docId,
2213
+ preparation: { document, beforeWrite: null, failure: void 0 },
2214
+ refusedByHook: false
2215
+ };
2216
+ if (hooked && ctx.pluginRegistry.hasHooks("beforeInsert")) {
2217
+ pending.preparation.beforeWrite = async () => {
2218
+ try {
2219
+ await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document });
2220
+ } catch (err) {
2221
+ pending.refusedByHook = true;
2222
+ throw err;
2223
+ }
2224
+ };
2225
+ }
2226
+ return pending;
2227
+ }
2228
+ async function prepareInsertDocuments(ctx, indexName, documents, start, end, hold, hooked, failed, earlier = null) {
2229
+ const pending = [];
2230
+ for (let i = start; i < end; i++) {
2231
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2232
+ if (ctx.abortController.signal.aborted) break;
2233
+ const document = documents[i];
2234
+ const docId = providedDocId(document) ?? ctx.idGenerator();
2235
+ try {
2236
+ const sizing = requireDocumentObject(document);
2237
+ if (sizing !== null) await sizing;
2238
+ validateDocId(docId);
2239
+ pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
2240
+ } catch (err) {
2241
+ failed.push({ docId, error: asBatchInsertError(err) });
2242
+ }
2243
+ }
2244
+ const { entry, embedding } = await prepareDocumentWrites(
2245
+ ctx,
2246
+ indexName,
2247
+ pending.map((item) => item.preparation),
2248
+ hold,
2249
+ true,
2250
+ earlier
2251
+ );
2252
+ const candidates = [];
2253
+ for (const item of pending) {
2254
+ const { document, failure } = item.preparation;
2255
+ if (failure === void 0) {
2256
+ candidates.push({ docId: item.docId, document });
2257
+ continue;
2258
+ }
2259
+ const docId = item.refusedByHook ? item.docId : providedDocId(document) ?? "";
2260
+ failed.push({ docId, error: asBatchInsertError(failure) });
2261
+ }
2262
+ return { entry, candidates, embedding };
2263
+ }
2264
+ async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
2265
+ const { entry, candidates, embedding } = await prepareInsertDocuments(
2266
+ ctx,
2267
+ indexName,
2268
+ documents,
2269
+ 0,
2270
+ documents.length,
2271
+ hold,
2272
+ true,
2273
+ failed
2274
+ );
2275
+ const manager = ctx.requireManager(indexName);
2276
+ const vecIndexes = manager.getVectorIndexes();
2277
+ const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
2278
+ const strict = entry.config.strict === true;
2279
+ const admitted = [];
2280
+ const admittedIds = /* @__PURE__ */ new Set();
2281
+ const pending = createPendingAdmissions();
2282
+ for (let position = 0; position < candidates.length; position++) {
2283
+ if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
2284
+ if (ctx.abortController.signal.aborted) break;
2285
+ const { docId, document } = candidates[position];
2286
+ try {
2287
+ const { partitionDoc, extractedVectors } = prepareDocumentVectors(
2288
+ document,
2289
+ vectorFieldPaths
2290
+ );
2291
+ if (extractedVectors.size > 0) {
2292
+ validateVectorDimensions(extractedVectors, vecIndexes);
2293
+ }
2294
+ admitInsert(ctx, indexName, manager, docId, pending);
2295
+ if (admittedIds.has(docId) || manager.has(docId)) {
2296
+ throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
2297
+ }
2298
+ validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
2299
+ if (strict) {
2300
+ validateDocumentStrict(partitionDoc, entry.config.schema);
2301
+ }
2302
+ admittedIds.add(docId);
2303
+ recordPendingAdmission(ctx, pending, manager, docId);
2304
+ admitted.push({ docId, document, partitionDoc, extractedVectors });
2305
+ } catch (err) {
2306
+ failed.push({ docId, error: asBatchInsertError(err) });
2307
+ }
2308
+ }
2309
+ return { admitted, embedding };
2310
+ }
2311
+
1996
2312
  // src/engine/mutations/durable-rollback.ts
1997
2313
  function holdVectorsForRollback(indexes, docId) {
1998
2314
  const releases = [];
@@ -2228,84 +2544,29 @@ async function applyInsertChunk(ctx, indexName, docs, options) {
2228
2544
  return applications;
2229
2545
  }
2230
2546
 
2231
- // src/engine/mutations/insert-batch-admission.ts
2232
- async function admitBatchDocuments(ctx, indexName, documents, failed) {
2233
- const entry = ctx.requireIndex(indexName);
2234
- const manager = ctx.requireManager(indexName);
2235
- const vecIndexes = manager.getVectorIndexes();
2236
- const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
2237
- const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
2238
- const strict = entry.config.strict === true;
2239
- const required = entry.config.required;
2240
- const admitted = [];
2241
- const admittedIds = /* @__PURE__ */ new Set();
2242
- const pending = createPendingAdmissions();
2243
- for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
2244
- if (ctx.abortController.signal.aborted) break;
2245
- const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
2246
- const chunkFailedIndexes = /* @__PURE__ */ new Set();
2247
- if (required && required.length > 0) {
2248
- collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
2249
- }
2250
- await embedChunkDocuments(
2251
- entry,
2252
- documents,
2253
- chunkStart,
2254
- chunkEnd,
2255
- ctx.abortController.signal,
2256
- chunkFailedIndexes,
2257
- failed
2258
- );
2259
- for (let i = chunkStart; i < chunkEnd; i++) {
2260
- if (ctx.abortController.signal.aborted) break;
2261
- if (chunkFailedIndexes.has(i)) continue;
2262
- const docId = providedDocId(documents[i]) ?? ctx.idGenerator();
2547
+ // src/engine/mutations/insert-batch-documents.ts
2548
+ async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
2549
+ if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
2550
+ const manager = ctx.executor.getManager(indexName);
2551
+ await runOutside(hold, async () => {
2552
+ for (const doc of inserted) {
2263
2553
  try {
2264
- const sizing = requireDocumentObject(documents[i]);
2265
- if (sizing !== null) await sizing;
2266
- validateDocId(docId);
2267
- if (hasBeforeHook) {
2268
- await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
2269
- }
2270
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
2271
- documents[i],
2272
- vectorFieldPaths
2273
- );
2274
- if (extractedVectors.size > 0) {
2275
- validateVectorDimensions(extractedVectors, vecIndexes);
2276
- }
2277
- admitInsert(ctx, indexName, manager, docId, pending);
2278
- if (admittedIds.has(docId) || manager.has(docId)) {
2279
- throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
2280
- }
2281
- validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
2282
- if (strict) {
2283
- validateDocumentStrict(partitionDoc, entry.config.schema);
2284
- }
2285
- admittedIds.add(docId);
2286
- recordPendingAdmission(ctx, pending, manager, docId);
2287
- admitted.push({ docId, document: documents[i], partitionDoc, extractedVectors });
2554
+ await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
2288
2555
  } catch (err) {
2289
- failed.push({ docId, error: asBatchInsertError(err) });
2556
+ console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
2290
2557
  }
2291
2558
  }
2292
- if (chunkEnd < documents.length) {
2293
- await new Promise((r) => setTimeout(r, 0));
2294
- }
2295
- }
2296
- return admitted;
2559
+ });
2560
+ return ctx.executor.getManager(indexName) !== manager;
2297
2561
  }
2298
-
2299
- // src/engine/mutations/insert-batch-documents.ts
2300
- async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed) {
2301
- const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
2562
+ async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
2302
2563
  const succeeded = [];
2303
- const buffered = /* @__PURE__ */ new Set();
2304
2564
  const touchedVectorFields = /* @__PURE__ */ new Set();
2305
2565
  for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
2306
2566
  if (ctx.abortController.signal.aborted) break;
2307
2567
  const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
2308
2568
  const applications = await applyInsertChunk(ctx, indexName, chunk, options);
2569
+ const inserted = [];
2309
2570
  for (let i = 0; i < chunk.length; i++) {
2310
2571
  const doc = chunk[i];
2311
2572
  const application = applications[i];
@@ -2314,30 +2575,103 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
2314
2575
  failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
2315
2576
  continue;
2316
2577
  }
2317
- if (application.status === "buffered") {
2318
- buffered.add(doc.docId);
2319
- succeeded.push(doc.docId);
2320
- continue;
2321
- }
2578
+ succeeded.push(doc.docId);
2579
+ if (application.status === "buffered") continue;
2322
2580
  for (const fieldPath of doc.extractedVectors.keys()) {
2323
2581
  touchedVectorFields.add(fieldPath);
2324
2582
  }
2325
- if (hasAfterHook) {
2326
- try {
2327
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
2328
- } catch (err) {
2329
- console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
2330
- }
2331
- }
2332
- succeeded.push(doc.docId);
2583
+ inserted.push(doc);
2333
2584
  }
2334
2585
  const applied = chunkStart + BATCH_CHUNK_SIZE;
2586
+ if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
2587
+ return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
2588
+ }
2335
2589
  if (applied < admitted.length) {
2336
2590
  ctx.checkHeapPressure(indexName);
2337
2591
  await new Promise((r) => setTimeout(r, 0));
2338
2592
  }
2339
2593
  }
2340
- return { succeeded, buffered, touchedVectorFields };
2594
+ return { succeeded, touchedVectorFields, unrecorded: [] };
2595
+ }
2596
+
2597
+ // src/engine/mutations/insert-batch-chunks.ts
2598
+ async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
2599
+ const { entry, candidates } = await prepareInsertDocuments(
2600
+ ctx,
2601
+ indexName,
2602
+ documents,
2603
+ chunkStart,
2604
+ chunkEnd,
2605
+ hold,
2606
+ hooked,
2607
+ failed,
2608
+ earlier
2609
+ );
2610
+ const manager = ctx.requireManager(indexName);
2611
+ const vecIndexes = manager.getVectorIndexes();
2612
+ const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
2613
+ const admitted = [];
2614
+ for (const { docId, document } of candidates) {
2615
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2616
+ if (ctx.abortController.signal.aborted) break;
2617
+ try {
2618
+ const { partitionDoc, extractedVectors } = prepareDocumentVectors(
2619
+ document,
2620
+ vectorFieldPaths
2621
+ );
2622
+ if (extractedVectors.size > 0) {
2623
+ validateVectorDimensions(extractedVectors, vecIndexes);
2624
+ }
2625
+ admitInsert(ctx, indexName, manager, docId);
2626
+ admitted.push({ docId, document, partitionDoc, extractedVectors });
2627
+ } catch (err) {
2628
+ failed.push({ docId, error: asBatchInsertError(err) });
2629
+ }
2630
+ }
2631
+ return admitted;
2632
+ }
2633
+ async function insertInChunks(ctx, indexName, documents, options, hold, hooked, failed, embedded = null) {
2634
+ const succeeded = [];
2635
+ const touchedVectorFields = /* @__PURE__ */ new Set();
2636
+ for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
2637
+ if (ctx.abortController.signal.aborted) break;
2638
+ const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
2639
+ const earlier = embeddingOfDocuments(embedded, documents.slice(chunkStart, chunkEnd));
2640
+ const prepared = await admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier);
2641
+ if (prepared.length > 0) {
2642
+ const applications = await applyInsertChunk(ctx, indexName, prepared, options);
2643
+ const inserted = [];
2644
+ for (let i = 0; i < prepared.length; i++) {
2645
+ const doc = prepared[i];
2646
+ const application = applications[i];
2647
+ if (application.status === "skipped") continue;
2648
+ if (application.status === "failed") {
2649
+ failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
2650
+ continue;
2651
+ }
2652
+ succeeded.push(doc.docId);
2653
+ if (application.status === "buffered") continue;
2654
+ for (const fieldPath of doc.extractedVectors.keys()) touchedVectorFields.add(fieldPath);
2655
+ inserted.push(doc);
2656
+ }
2657
+ await runAfterInsertHooks(ctx, indexName, inserted, hold);
2658
+ }
2659
+ if (chunkEnd < documents.length) {
2660
+ ctx.checkHeapPressure(indexName);
2661
+ await new Promise((r) => setTimeout(r, 0));
2662
+ }
2663
+ }
2664
+ return { succeeded, touchedVectorFields };
2665
+ }
2666
+ async function finishInsertBatch(ctx, indexName, inserts, options) {
2667
+ const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
2668
+ for (const fieldPath of inserts.touchedVectorFields) {
2669
+ vecIndexes?.get(fieldPath)?.scheduleBuild();
2670
+ }
2671
+ ctx.checkWatermark(indexName, inserts.succeeded);
2672
+ ctx.checkHeapPressure(indexName);
2673
+ await ctx.orchestrator.scaleOutReadyIndexes();
2674
+ if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2341
2675
  }
2342
2676
 
2343
2677
  // src/engine/mutations/segment-publication.ts
@@ -2442,10 +2776,6 @@ function mainStoreDocument(doc, options) {
2442
2776
  if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
2443
2777
  return structuredClone(doc.partitionDoc);
2444
2778
  }
2445
- async function applyIndividually(ctx, indexName, admitted, options, failed) {
2446
- const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
2447
- return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
2448
- }
2449
2779
  function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted) {
2450
2780
  const chunks = [];
2451
2781
  let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
@@ -2495,12 +2825,12 @@ function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberI
2495
2825
  if (chunk.docs.length > 0) chunks.push(chunk);
2496
2826
  return chunks;
2497
2827
  }
2498
- async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, failed) {
2499
- const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
2828
+ async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, hold, failed) {
2500
2829
  const succeededIds = /* @__PURE__ */ new Set();
2501
2830
  const errorOf = /* @__PURE__ */ new Map();
2502
2831
  const touchedVectorFields = /* @__PURE__ */ new Set();
2503
2832
  const chunks = chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted);
2833
+ const unrecorded = [];
2504
2834
  for (let index = 0; index < chunks.length; index++) {
2505
2835
  const { docs, attachmentOf } = chunks[index];
2506
2836
  const applies = docs.map((doc) => {
@@ -2510,6 +2840,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
2510
2840
  });
2511
2841
  const failures = await recordChunk(ctx, indexName, docs, applies);
2512
2842
  await ctx.orchestrator.settlePublications(indexName, "segments");
2843
+ const inserted = [];
2513
2844
  for (let i = 0; i < docs.length; i++) {
2514
2845
  const doc = docs[i];
2515
2846
  const failure = failures[i];
@@ -2541,15 +2872,13 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
2541
2872
  for (const fieldPath of doc.extractedVectors.keys()) {
2542
2873
  touchedVectorFields.add(fieldPath);
2543
2874
  }
2544
- if (hasAfterHook) {
2545
- try {
2546
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
2547
- } catch (err) {
2548
- console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
2549
- }
2550
- }
2875
+ inserted.push(doc);
2551
2876
  succeededIds.add(doc.docId);
2552
2877
  }
2878
+ if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
2879
+ for (const later of chunks.slice(index + 1)) unrecorded.push(...later.docs);
2880
+ break;
2881
+ }
2553
2882
  if (index + 1 < chunks.length) {
2554
2883
  ctx.checkHeapPressure(indexName);
2555
2884
  await new Promise((r) => setTimeout(r, 0));
@@ -2564,7 +2893,7 @@ async function recordMergedDocuments(ctx, indexName, partitionCount, built, froz
2564
2893
  if (!errorOf.has(doc.docId)) continue;
2565
2894
  failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
2566
2895
  }
2567
- return { succeeded, touchedVectorFields };
2896
+ return { succeeded, touchedVectorFields, unrecorded };
2568
2897
  }
2569
2898
  function frozenSegmentOf(segment, members, options) {
2570
2899
  if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
@@ -2575,14 +2904,15 @@ function frozenSegmentOf(segment, members, options) {
2575
2904
  segment.segmentId
2576
2905
  );
2577
2906
  }
2578
- async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
2907
+ async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
2579
2908
  if (admitted.length === 0) {
2580
- return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set() };
2909
+ return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] };
2581
2910
  }
2911
+ const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
2582
2912
  const manager = ctx.requireManager(indexName);
2583
2913
  const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
2584
2914
  if (admitted.length < MIN_DOCUMENTS_FOR_SEGMENTS || workers <= 0) {
2585
- return applyIndividually(ctx, indexName, admitted, options, failed);
2915
+ return applyIndividually();
2586
2916
  }
2587
2917
  const docIds = admitted.map((doc) => doc.docId);
2588
2918
  const partitionDocuments = admitted.map((doc) => doc.partitionDoc);
@@ -2595,7 +2925,7 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
2595
2925
  built = null;
2596
2926
  }
2597
2927
  if (built === null) {
2598
- return applyIndividually(ctx, indexName, admitted, options, failed);
2928
+ return applyIndividually();
2599
2929
  }
2600
2930
  const positions = new Map(admitted.map((document, index) => [document.docId, index]));
2601
2931
  const used = /* @__PURE__ */ new Set();
@@ -2604,16 +2934,15 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
2604
2934
  const members = [];
2605
2935
  for (const docId of segment.docIds) {
2606
2936
  const position = positions.get(docId);
2607
- if (position === void 0 || used.has(position))
2608
- return applyIndividually(ctx, indexName, admitted, options, failed);
2937
+ if (position === void 0 || used.has(position)) return applyIndividually();
2609
2938
  used.add(position);
2610
2939
  members.push(position);
2611
2940
  }
2612
2941
  memberIndexes.push(members);
2613
2942
  }
2614
- if (used.size !== admitted.length) return applyIndividually(ctx, indexName, admitted, options, failed);
2943
+ if (used.size !== admitted.length) return applyIndividually();
2615
2944
  if (ctx.isRebalancing(indexName)) {
2616
- return applyIndividually(ctx, indexName, admitted, options, failed);
2945
+ return applyIndividually();
2617
2946
  }
2618
2947
  const frozen = [];
2619
2948
  for (let i = 0; i < built.length; i++) {
@@ -2622,149 +2951,65 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
2622
2951
  memberIndexes[i].map((m) => admitted[m]),
2623
2952
  options
2624
2953
  );
2625
- if (segment === null) return applyIndividually(ctx, indexName, admitted, options, failed);
2954
+ if (segment === null) return applyIndividually();
2626
2955
  frozen.push(segment);
2627
2956
  }
2628
2957
  for (const doc of admitted) {
2629
- if (manager.has(doc.docId)) return applyIndividually(ctx, indexName, admitted, options, failed);
2958
+ if (manager.has(doc.docId)) return applyIndividually();
2630
2959
  }
2631
- return recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, failed);
2960
+ return recordMergedDocuments(
2961
+ ctx,
2962
+ indexName,
2963
+ partitionCount,
2964
+ built,
2965
+ frozen,
2966
+ memberIndexes,
2967
+ admitted,
2968
+ options,
2969
+ hold,
2970
+ failed
2971
+ );
2632
2972
  }
2633
- async function insertBatchViaSegments(ctx, indexName, documents, options) {
2973
+ async function insertBatchViaSegments(ctx, indexName, documents, options, hold) {
2634
2974
  const failed = [];
2635
- const admitted = await admitBatchDocuments(ctx, indexName, documents, failed);
2636
- const outcome = await ingestAdmitted(ctx, indexName, admitted, options, failed);
2637
- const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
2638
- for (const fieldPath of outcome.touchedVectorFields) {
2639
- vecIndexes.get(fieldPath)?.scheduleBuild();
2640
- }
2641
- ctx.checkWatermark(indexName, outcome.succeeded);
2642
- ctx.checkHeapPressure(indexName);
2643
- await ctx.orchestrator.scaleOutReadyIndexes();
2644
- if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2975
+ const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents, failed, hold);
2976
+ const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
2977
+ if (outcome.unrecorded.length > 0) {
2978
+ const unrecorded = outcome.unrecorded.map((doc) => doc.document);
2979
+ const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
2980
+ outcome.succeeded.push(...rest.succeeded);
2981
+ for (const fieldPath of rest.touchedVectorFields) outcome.touchedVectorFields.add(fieldPath);
2982
+ }
2983
+ await finishInsertBatch(ctx, indexName, outcome, options);
2645
2984
  return { succeeded: outcome.succeeded, failed };
2646
2985
  }
2647
2986
 
2648
2987
  // src/engine/mutations/insert-batch.ts
2649
- async function insertDocumentBatch(ctx, indexName, documents, options) {
2988
+ async function insertDocumentBatch(ctx, indexName, documents, options, hold) {
2650
2989
  ctx.guardShutdown();
2651
- const entry = ctx.requireIndex(indexName);
2990
+ ctx.requireIndex(indexName);
2652
2991
  validateBatchList(documents, "documents");
2653
2992
  await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length);
2654
2993
  const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
2655
2994
  if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
2656
- return insertBatchViaSegments(ctx, indexName, documents, options);
2995
+ return insertBatchViaSegments(ctx, indexName, documents, options, hold);
2657
2996
  }
2658
- const succeeded = [];
2659
2997
  const failed = [];
2660
- const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
2661
- const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
2662
- const required = entry.config.required;
2663
- const batchManager = ctx.requireManager(indexName);
2664
- const batchVecIndexes = batchManager.getVectorIndexes();
2665
- const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
2666
- const touchedVectorFields = /* @__PURE__ */ new Set();
2667
- for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
2668
- if (ctx.abortController.signal.aborted) break;
2669
- const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
2670
- const chunkFailedIndexes = /* @__PURE__ */ new Set();
2671
- if (required && required.length > 0) {
2672
- collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
2673
- }
2674
- await embedChunkDocuments(
2675
- entry,
2676
- documents,
2677
- chunkStart,
2678
- chunkEnd,
2679
- ctx.abortController.signal,
2680
- chunkFailedIndexes,
2681
- failed
2682
- );
2683
- async function applyPrepared(prepared2) {
2684
- const applications = await applyInsertChunk(ctx, indexName, prepared2, options);
2685
- for (let i = 0; i < prepared2.length; i++) {
2686
- const doc = prepared2[i];
2687
- const application = applications[i];
2688
- if (application.status === "skipped") continue;
2689
- if (application.status === "failed") {
2690
- failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
2691
- continue;
2692
- }
2693
- if (application.status === "buffered") {
2694
- succeeded.push(doc.docId);
2695
- continue;
2696
- }
2697
- for (const fieldPath of doc.extractedVectors.keys()) {
2698
- touchedVectorFields.add(fieldPath);
2699
- }
2700
- if (hasAfterHook) {
2701
- try {
2702
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
2703
- } catch (err) {
2704
- console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
2705
- }
2706
- }
2707
- succeeded.push(doc.docId);
2708
- }
2709
- }
2710
- const prepared = [];
2711
- for (let i = chunkStart; i < chunkEnd; i++) {
2712
- if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2713
- if (ctx.abortController.signal.aborted) break;
2714
- if (chunkFailedIndexes.has(i)) continue;
2715
- const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
2716
- try {
2717
- const sizing = requireDocumentObject(documents[i]);
2718
- if (sizing !== null) await sizing;
2719
- validateDocId(batchDocId);
2720
- if (hasBeforeHook) {
2721
- await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
2722
- }
2723
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
2724
- documents[i],
2725
- batchVectorFieldPaths
2726
- );
2727
- if (extractedVectors.size > 0) {
2728
- validateVectorDimensions(extractedVectors, batchVecIndexes);
2729
- }
2730
- admitInsert(ctx, indexName, batchManager, batchDocId);
2731
- prepared.push({
2732
- docId: batchDocId,
2733
- document: documents[i],
2734
- partitionDoc,
2735
- extractedVectors
2736
- });
2737
- } catch (err) {
2738
- failed.push({ docId: batchDocId, error: asBatchInsertError(err) });
2739
- }
2740
- }
2741
- if (prepared.length > 0) await applyPrepared(prepared);
2742
- if (chunkEnd < documents.length) {
2743
- ctx.checkHeapPressure(indexName);
2744
- await new Promise((r) => setTimeout(r, 0));
2745
- }
2746
- }
2747
- for (const fieldPath of touchedVectorFields) {
2748
- const vecIndex = batchVecIndexes.get(fieldPath);
2749
- if (vecIndex) {
2750
- vecIndex.scheduleBuild();
2751
- }
2752
- }
2753
- ctx.checkWatermark(indexName, succeeded);
2754
- ctx.checkHeapPressure(indexName);
2755
- await ctx.orchestrator.scaleOutReadyIndexes();
2756
- if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2757
- return { succeeded, failed };
2998
+ const inserts = await insertInChunks(ctx, indexName, documents, options, hold, true, failed);
2999
+ await finishInsertBatch(ctx, indexName, inserts, options);
3000
+ return { succeeded: inserts.succeeded, failed };
2758
3001
  }
2759
3002
 
2760
3003
  // src/engine/mutations/remove.ts
2761
3004
  function asRemoveError(err) {
2762
3005
  return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
2763
3006
  }
2764
- async function prepareRemoval(ctx, indexName, docId) {
3007
+ function validateRemoval(ctx, docId) {
2765
3008
  ctx.guardShutdown();
2766
3009
  validateDocId(docId);
2767
- await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
3010
+ }
3011
+ function prepareRemoval(ctx, indexName, docId) {
3012
+ ctx.guardShutdown();
2768
3013
  const rebalancing = ctx.isRebalancing(indexName);
2769
3014
  if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
2770
3015
  throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
@@ -2814,20 +3059,31 @@ function applyOfRemoval(ctx, indexName, prepared) {
2814
3059
  };
2815
3060
  return () => manager.publication.writeNow(indexChangeOutsideExternalWaits(removeFromIndex));
2816
3061
  }
2817
- async function settleRemoval(ctx, indexName, prepared) {
3062
+ async function settleRemovals(ctx, indexName, docIds, hold) {
3063
+ if (docIds.length === 0) return;
2818
3064
  const settling = ctx.orchestrator.settlePublications(indexName, "none");
2819
3065
  if (settling !== void 0) await settling;
2820
- const { docId } = prepared;
2821
- try {
2822
- await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
2823
- } catch (err) {
2824
- console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
2825
- }
3066
+ if (!ctx.pluginRegistry.hasHooks("afterRemove")) return;
3067
+ await runOutside(hold, async () => {
3068
+ for (const docId of docIds) {
3069
+ try {
3070
+ await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
3071
+ } catch (err) {
3072
+ console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
3073
+ }
3074
+ }
3075
+ });
2826
3076
  }
2827
- async function removeDocument2(ctx, indexName, docId, options) {
3077
+ async function removeDocument2(ctx, indexName, docId, options, hold) {
2828
3078
  ctx.guardShutdown();
2829
3079
  ctx.requireIndex(indexName);
2830
- const prepared = await prepareRemoval(ctx, indexName, docId);
3080
+ validateRemoval(ctx, docId);
3081
+ if (ctx.pluginRegistry.hasHooks("beforeRemove")) {
3082
+ await runOutside(hold, async () => {
3083
+ await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
3084
+ });
3085
+ }
3086
+ const prepared = prepareRemoval(ctx, indexName, docId);
2831
3087
  const apply = applyOfRemoval(ctx, indexName, prepared);
2832
3088
  if (ctx.durability) {
2833
3089
  try {
@@ -2841,25 +3097,52 @@ async function removeDocument2(ctx, indexName, docId, options) {
2841
3097
  } else {
2842
3098
  await apply();
2843
3099
  }
2844
- if (!prepared.buffered) await settleRemoval(ctx, indexName, prepared);
3100
+ if (!prepared.buffered) await settleRemovals(ctx, indexName, [docId], hold);
2845
3101
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2846
3102
  }
2847
- async function removeDocumentBatch(ctx, indexName, docIds, options) {
3103
+ async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold) {
3104
+ const refused = /* @__PURE__ */ new Map();
3105
+ for (let i = chunkStart; i < chunkEnd; i++) {
3106
+ try {
3107
+ validateRemoval(ctx, docIds[i]);
3108
+ } catch (err) {
3109
+ refused.set(i, err);
3110
+ }
3111
+ }
3112
+ if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
3113
+ await runOutside(hold, async () => {
3114
+ for (let i = chunkStart; i < chunkEnd; i++) {
3115
+ if (refused.has(i)) continue;
3116
+ try {
3117
+ await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
3118
+ } catch (err) {
3119
+ refused.set(i, err);
3120
+ }
3121
+ }
3122
+ });
3123
+ return refused;
3124
+ }
3125
+ async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
2848
3126
  ctx.guardShutdown();
2849
3127
  ctx.requireIndex(indexName);
2850
3128
  validateBatchList(docIds, "document IDs");
2851
3129
  const succeeded = [];
2852
3130
  const failed = [];
2853
- const manager = ctx.requireManager(indexName);
2854
- await manager.publication.write(() => manager.beginBatchRemove());
3131
+ const first = ctx.requireManager(indexName);
3132
+ await first.publication.write(() => first.beginBatchRemove());
3133
+ let manager = first;
2855
3134
  try {
2856
3135
  for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
2857
3136
  const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
3137
+ const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
3138
+ manager = await batchManagerAfterHooks(ctx, indexName, manager);
3139
+ const removed = [];
2858
3140
  const prepared = [];
2859
3141
  for (let i = chunkStart; i < chunkEnd; i++) {
2860
3142
  if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2861
3143
  try {
2862
- prepared.push(await prepareRemoval(ctx, indexName, docIds[i]));
3144
+ if (refused.has(i)) throw refused.get(i);
3145
+ prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
2863
3146
  } catch (err) {
2864
3147
  failed.push({ docId: docIds[i], error: asRemoveError(err) });
2865
3148
  }
@@ -2889,23 +3172,33 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
2889
3172
  continue;
2890
3173
  }
2891
3174
  removal.releaseVectorHolds?.();
2892
- if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
3175
+ if (!removal.buffered) removed.push(removal.docId);
2893
3176
  succeeded.push(removal.docId);
2894
3177
  }
2895
3178
  } finally {
2896
3179
  for (const removal of prepared) removal.releaseVectorHolds?.();
2897
3180
  }
3181
+ await settleRemovals(ctx, indexName, removed, hold);
3182
+ manager = await batchManagerAfterHooks(ctx, indexName, manager);
2898
3183
  if (chunkEnd < docIds.length) {
2899
3184
  await new Promise((r) => setTimeout(r, 0));
2900
3185
  }
2901
3186
  }
2902
3187
  } finally {
2903
- await manager.publication.write(() => manager.endBatchRemove());
3188
+ const ending = manager;
3189
+ await ending?.publication.write(() => ending.endBatchRemove());
2904
3190
  }
2905
3191
  ctx.checkHeapPressure(indexName);
2906
3192
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
2907
3193
  return { succeeded, failed };
2908
3194
  }
3195
+ async function batchManagerAfterHooks(ctx, indexName, batchManager) {
3196
+ const current = ctx.executor.getManager(indexName);
3197
+ if (current === batchManager) return batchManager;
3198
+ await batchManager?.publication.write(() => batchManager.endBatchRemove());
3199
+ await current?.publication.write(() => current.beginBatchRemove());
3200
+ return current;
3201
+ }
2909
3202
 
2910
3203
  // src/engine/mutations/update.ts
2911
3204
  function prepareUpdatePartitionDoc(document, extractedVectors) {
@@ -2939,59 +3232,47 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
2939
3232
  function asUpdateError(err) {
2940
3233
  return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
2941
3234
  }
2942
- async function settleUpdate(ctx, indexName, prepared) {
3235
+ async function runAfterUpdateHooks(ctx, indexName, updates) {
3236
+ for (const update of updates) {
3237
+ try {
3238
+ await ctx.pluginRegistry.runHook("afterUpdate", {
3239
+ indexName,
3240
+ docId: update.docId,
3241
+ oldDocument: update.oldDocument ?? {},
3242
+ newDocument: update.document
3243
+ });
3244
+ } catch (err) {
3245
+ console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
3246
+ }
3247
+ }
3248
+ }
3249
+ async function settleUpdates(ctx, indexName, updates, hold) {
3250
+ if (updates.length === 0) return;
2943
3251
  await ctx.orchestrator.settlePublications(indexName);
2944
- try {
2945
- await ctx.pluginRegistry.runHook("afterUpdate", {
2946
- indexName,
2947
- docId: prepared.docId,
2948
- oldDocument: prepared.oldDocument ?? {},
2949
- newDocument: prepared.document
2950
- });
2951
- } catch (err) {
2952
- console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
3252
+ if (ctx.pluginRegistry.hasHooks("afterUpdate")) {
3253
+ await runOutside(hold, () => runAfterUpdateHooks(ctx, indexName, updates));
2953
3254
  }
2954
- const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
2955
- for (const [fieldPath, vec] of prepared.extractedVectors) {
2956
- if (vec === null) continue;
2957
- vecIndexes.get(fieldPath)?.scheduleBuild();
3255
+ const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
3256
+ if (vecIndexes === void 0) return;
3257
+ for (const update of updates) {
3258
+ for (const [fieldPath, vec] of update.extractedVectors) {
3259
+ if (vec !== null) vecIndexes.get(fieldPath)?.scheduleBuild();
3260
+ }
2958
3261
  }
2959
3262
  }
2960
- async function prepareUpdate(ctx, indexName, entry, docId, document) {
3263
+ function updatePreparationOf(ctx, indexName, docId, document) {
2961
3264
  ctx.guardShutdown();
2962
3265
  validateDocId(docId);
2963
3266
  requireMatchingDocId(docId, document);
2964
- const manager = ctx.requireManager(indexName);
2965
- if (ctx.pluginRegistry.hasHooks("beforeUpdate")) {
2966
- await ctx.pluginRegistry.runHook("beforeUpdate", {
2967
- indexName,
2968
- docId,
2969
- oldDocument: manager.get(docId) ?? {},
2970
- newDocument: document
2971
- });
2972
- }
2973
- if (entry.config.required && entry.config.required.length > 0) {
2974
- validateRequiredFields(document, entry.config.required);
2975
- }
2976
- if (entry.config.embedding) {
2977
- const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
2978
- if (entry.embeddingAdapter) {
2979
- await embedDocumentFields(
2980
- document,
2981
- entry.config.embedding,
2982
- entry.embeddingAdapter,
2983
- ctx.abortController.signal
2984
- );
2985
- } else {
2986
- assertDocumentCarriesMappedVectors(
2987
- document,
2988
- entry.config.embedding,
2989
- entry.embeddingAdapterName
2990
- );
2991
- }
2992
- }
3267
+ if (!ctx.pluginRegistry.hasHooks("beforeUpdate")) return { document, beforeWrite: null, failure: void 0 };
3268
+ const oldDocument = ctx.requireManager(indexName).get(docId) ?? {};
3269
+ const beforeWrite = () => ctx.pluginRegistry.runHook("beforeUpdate", { indexName, docId, oldDocument, newDocument: document });
3270
+ return { document, beforeWrite, failure: void 0 };
3271
+ }
3272
+ async function finishUpdate(ctx, indexName, entry, docId, document) {
2993
3273
  const sizing = requireDocumentWithinSizeLimit(document);
2994
3274
  if (sizing !== null) await sizing;
3275
+ const manager = ctx.requireManager(indexName);
2995
3276
  const vecIndexes = manager.getVectorIndexes();
2996
3277
  const extractedVectors = /* @__PURE__ */ new Map();
2997
3278
  if (vecIndexes.size > 0) {
@@ -3085,10 +3366,13 @@ function applyOfUpdate(ctx, indexName, prepared) {
3085
3366
  };
3086
3367
  return () => manager.publication.write(indexChangeOutsideExternalWaits(updateInIndex));
3087
3368
  }
3088
- async function updateDocument(ctx, indexName, docId, document, options) {
3369
+ async function updateDocument(ctx, indexName, docId, document, options, hold) {
3089
3370
  ctx.guardShutdown();
3090
- const entry = ctx.requireIndex(indexName);
3091
- const prepared = await prepareUpdate(ctx, indexName, entry, docId, document);
3371
+ ctx.requireIndex(indexName);
3372
+ const preparation = updatePreparationOf(ctx, indexName, docId, document);
3373
+ const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
3374
+ if (preparation.failure !== void 0) throw preparation.failure;
3375
+ const prepared = await finishUpdate(ctx, indexName, entry, docId, document);
3092
3376
  const apply = applyOfUpdate(ctx, indexName, prepared);
3093
3377
  if (ctx.durability) {
3094
3378
  try {
@@ -3111,10 +3395,12 @@ async function updateDocument(ctx, indexName, docId, document, options) {
3111
3395
  } else {
3112
3396
  await apply();
3113
3397
  }
3114
- if (!prepared.buffered) await settleUpdate(ctx, indexName, prepared);
3398
+ if (!prepared.buffered) await settleUpdates(ctx, indexName, [prepared], hold);
3115
3399
  ctx.checkHeapPressure(indexName);
3116
3400
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
3117
3401
  }
3402
+
3403
+ // src/engine/mutations/update-batch.ts
3118
3404
  function reportedDocIdOf(update) {
3119
3405
  const docId = update !== null && typeof update === "object" ? update.docId : void 0;
3120
3406
  return typeof docId === "string" ? docId : "";
@@ -3131,29 +3417,47 @@ function requireUpdateEntry(update) {
3131
3417
  "Each entry of a batch update must be an object whose docId is a string and whose document is an object"
3132
3418
  );
3133
3419
  }
3134
- async function updateDocumentBatch(ctx, indexName, updates, options) {
3420
+ async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed) {
3421
+ const preparations = [];
3422
+ for (let i = chunkStart; i < chunkEnd; i++) {
3423
+ const update = updates[i];
3424
+ const docId = reportedDocIdOf(update);
3425
+ try {
3426
+ const { document } = requireUpdateEntry(update);
3427
+ preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
3428
+ } catch (err) {
3429
+ failed.push({ docId, error: asUpdateError(err) });
3430
+ }
3431
+ }
3432
+ const { entry } = await prepareDocumentWrites(
3433
+ ctx,
3434
+ indexName,
3435
+ preparations.map((pending) => pending.preparation),
3436
+ hold,
3437
+ false
3438
+ );
3439
+ const prepared = [];
3440
+ for (const { docId, preparation } of preparations) {
3441
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
3442
+ try {
3443
+ if (preparation.failure !== void 0) throw preparation.failure;
3444
+ prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
3445
+ } catch (err) {
3446
+ failed.push({ docId, error: asUpdateError(err) });
3447
+ }
3448
+ }
3449
+ return prepared;
3450
+ }
3451
+ async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
3135
3452
  ctx.guardShutdown();
3136
- const entry = ctx.requireIndex(indexName);
3453
+ ctx.requireIndex(indexName);
3137
3454
  validateBatchList(updates, "updates");
3138
3455
  const succeeded = [];
3139
3456
  const failed = [];
3140
- const updateBatchManager = ctx.requireManager(indexName);
3141
- const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
3142
3457
  const touchedVectorFields = /* @__PURE__ */ new Set();
3143
3458
  for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
3144
3459
  const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
3145
- const prepared = [];
3146
- for (let i = chunkStart; i < chunkEnd; i++) {
3147
- if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
3148
- const update = updates[i];
3149
- const docId = reportedDocIdOf(update);
3150
- try {
3151
- const { document } = requireUpdateEntry(update);
3152
- prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document));
3153
- } catch (err) {
3154
- failed.push({ docId, error: asUpdateError(err) });
3155
- }
3156
- }
3460
+ const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
3157
3461
  const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
3158
3462
  let failures;
3159
3463
  try {
@@ -3180,6 +3484,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
3180
3484
  } finally {
3181
3485
  for (const update of prepared) update.releaseVectorHolds?.();
3182
3486
  }
3487
+ const settled = [];
3183
3488
  for (let i = 0; i < prepared.length; i++) {
3184
3489
  const update = prepared[i];
3185
3490
  const failure = failures[i];
@@ -3187,19 +3492,21 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
3187
3492
  failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
3188
3493
  continue;
3189
3494
  }
3190
- if (!update.buffered) await settleUpdate(ctx, indexName, update);
3495
+ if (!update.buffered) settled.push(update);
3191
3496
  for (const [fieldPath, vec] of update.extractedVectors) {
3192
3497
  if (vec !== null) touchedVectorFields.add(fieldPath);
3193
3498
  }
3194
3499
  succeeded.push(update.docId);
3195
3500
  }
3501
+ await settleUpdates(ctx, indexName, settled, hold);
3196
3502
  if (chunkEnd < updates.length) {
3197
3503
  ctx.checkHeapPressure(indexName);
3198
3504
  await new Promise((r) => setTimeout(r, 0));
3199
3505
  }
3200
3506
  }
3507
+ const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
3201
3508
  for (const fieldPath of touchedVectorFields) {
3202
- updateBatchVecIndexes.get(fieldPath)?.scheduleBuild();
3509
+ vectorIndexes?.get(fieldPath)?.scheduleBuild();
3203
3510
  }
3204
3511
  ctx.checkHeapPressure(indexName);
3205
3512
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
@@ -3245,4 +3552,4 @@ async function updateEnginePartitionConfig(core, indexName, partitionConfig) {
3245
3552
  core.watermarkNotifier.check(indexName);
3246
3553
  }
3247
3554
 
3248
- export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
3555
+ export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, prepareDocumentWrite, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };