@harperfast/harper 5.3.0 → 5.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/bin/copyDb.ts +5 -1
  2. package/bin/deployRenderer.ts +3 -0
  3. package/components/Application.ts +66 -22
  4. package/components/DESIGN.md +89 -36
  5. package/components/componentLoader.ts +2 -0
  6. package/components/deploymentRecorder.ts +18 -1
  7. package/components/installFingerprint.ts +123 -0
  8. package/components/mcp/DESIGN.md +4 -4
  9. package/components/operations.js +29 -0
  10. package/config/DESIGN.md +5 -5
  11. package/dataLayer/DESIGN.md +37 -22
  12. package/dataLayer/backupRepository.ts +248 -0
  13. package/dataLayer/restoreMarker.ts +30 -75
  14. package/dataLayer/rocksdbBackup.ts +253 -110
  15. package/dist/bin/copyDb.js +3 -0
  16. package/dist/bin/copyDb.js.map +1 -1
  17. package/dist/bin/deployRenderer.js +3 -0
  18. package/dist/bin/deployRenderer.js.map +1 -1
  19. package/dist/components/Application.d.ts +5 -0
  20. package/dist/components/Application.js +51 -19
  21. package/dist/components/Application.js.map +1 -1
  22. package/dist/components/componentLoader.js +2 -0
  23. package/dist/components/componentLoader.js.map +1 -1
  24. package/dist/components/deploymentRecorder.js +18 -1
  25. package/dist/components/deploymentRecorder.js.map +1 -1
  26. package/dist/components/installFingerprint.d.ts +39 -0
  27. package/dist/components/installFingerprint.js +113 -0
  28. package/dist/components/installFingerprint.js.map +1 -0
  29. package/dist/components/operations.d.ts +2 -0
  30. package/dist/components/operations.js +31 -0
  31. package/dist/components/operations.js.map +1 -1
  32. package/dist/dataLayer/backupRepository.d.ts +33 -0
  33. package/dist/dataLayer/backupRepository.js +232 -0
  34. package/dist/dataLayer/backupRepository.js.map +1 -0
  35. package/dist/dataLayer/restoreMarker.d.ts +4 -1
  36. package/dist/dataLayer/restoreMarker.js +29 -68
  37. package/dist/dataLayer/restoreMarker.js.map +1 -1
  38. package/dist/dataLayer/rocksdbBackup.js +221 -112
  39. package/dist/dataLayer/rocksdbBackup.js.map +1 -1
  40. package/dist/index.d.ts +1 -0
  41. package/dist/index.js.map +1 -1
  42. package/dist/json/systemSchema.json +14 -0
  43. package/dist/resources/DatabaseTransaction.d.ts +140 -11
  44. package/dist/resources/DatabaseTransaction.js +1055 -464
  45. package/dist/resources/DatabaseTransaction.js.map +1 -1
  46. package/dist/resources/IterableEventQueue.d.ts +2 -1
  47. package/dist/resources/IterableEventQueue.js +16 -8
  48. package/dist/resources/IterableEventQueue.js.map +1 -1
  49. package/dist/resources/LMDBTransaction.d.ts +13 -2
  50. package/dist/resources/LMDBTransaction.js +219 -93
  51. package/dist/resources/LMDBTransaction.js.map +1 -1
  52. package/dist/resources/RecordEncoder.js +2 -3
  53. package/dist/resources/RecordEncoder.js.map +1 -1
  54. package/dist/resources/Resource.js +5 -6
  55. package/dist/resources/Resource.js.map +1 -1
  56. package/dist/resources/ResourceInterface.d.ts +3 -1
  57. package/dist/resources/ResourceInterface.js.map +1 -1
  58. package/dist/resources/RocksIndexStore.d.ts +2 -1
  59. package/dist/resources/RocksIndexStore.js +3 -0
  60. package/dist/resources/RocksIndexStore.js.map +1 -1
  61. package/dist/resources/Table.js +116 -12
  62. package/dist/resources/Table.js.map +1 -1
  63. package/dist/resources/analytics/write.js +6 -2
  64. package/dist/resources/analytics/write.js.map +1 -1
  65. package/dist/resources/auditStore.d.ts +71 -17
  66. package/dist/resources/auditStore.js +248 -64
  67. package/dist/resources/auditStore.js.map +1 -1
  68. package/dist/resources/branchDatabase.js +2 -0
  69. package/dist/resources/branchDatabase.js.map +1 -1
  70. package/dist/resources/databases.js +8 -2
  71. package/dist/resources/databases.js.map +1 -1
  72. package/dist/resources/indexes/fullTextNativeBinding.d.ts +6 -2
  73. package/dist/resources/indexes/fullTextNativeBinding.js +12 -2
  74. package/dist/resources/indexes/fullTextNativeBinding.js.map +1 -1
  75. package/dist/resources/indexes/fullTextQueryIndex.d.ts +4 -0
  76. package/dist/resources/indexes/fullTextQueryIndex.js +207 -10
  77. package/dist/resources/indexes/fullTextQueryIndex.js.map +1 -1
  78. package/dist/resources/models/Models.d.ts +11 -0
  79. package/dist/resources/models/Models.js +95 -4
  80. package/dist/resources/models/Models.js.map +1 -1
  81. package/dist/resources/models/backendRegistry.d.ts +8 -0
  82. package/dist/resources/models/backendRegistry.js +15 -1
  83. package/dist/resources/models/backendRegistry.js.map +1 -1
  84. package/dist/resources/models/bootstrap.d.ts +2 -0
  85. package/dist/resources/models/bootstrap.js +6 -0
  86. package/dist/resources/models/bootstrap.js.map +1 -1
  87. package/dist/resources/models/calibration.d.ts +88 -0
  88. package/dist/resources/models/calibration.js +244 -0
  89. package/dist/resources/models/calibration.js.map +1 -0
  90. package/dist/resources/models/calibrationStore.d.ts +163 -0
  91. package/dist/resources/models/calibrationStore.js +986 -0
  92. package/dist/resources/models/calibrationStore.js.map +1 -0
  93. package/dist/resources/models/decisionStore.d.ts +24 -0
  94. package/dist/resources/models/decisionStore.js +45 -0
  95. package/dist/resources/models/decisionStore.js.map +1 -1
  96. package/dist/resources/models/generativeDecision.js +44 -15
  97. package/dist/resources/models/generativeDecision.js.map +1 -1
  98. package/dist/resources/models/types.d.ts +7 -0
  99. package/dist/resources/recordLock.d.ts +8 -5
  100. package/dist/resources/recordLock.js +6 -4
  101. package/dist/resources/recordLock.js.map +1 -1
  102. package/dist/resources/replayLogs.js +8 -1
  103. package/dist/resources/replayLogs.js.map +1 -1
  104. package/dist/resources/scheduler/engine.d.ts +4 -0
  105. package/dist/resources/scheduler/engine.js +30 -2
  106. package/dist/resources/scheduler/engine.js.map +1 -1
  107. package/dist/resources/search.d.ts +16 -4
  108. package/dist/resources/search.js +117 -13
  109. package/dist/resources/search.js.map +1 -1
  110. package/dist/resources/transaction.js +85 -5
  111. package/dist/resources/transaction.js.map +1 -1
  112. package/dist/resources/transactionBroadcast.d.ts +8 -0
  113. package/dist/resources/transactionBroadcast.js +65 -0
  114. package/dist/resources/transactionBroadcast.js.map +1 -1
  115. package/dist/security/keys.d.ts +1 -1
  116. package/dist/security/keys.js +170 -120
  117. package/dist/security/keys.js.map +1 -1
  118. package/dist/server/http.js +10 -0
  119. package/dist/server/http.js.map +1 -1
  120. package/dist/server/serverHelpers/Request.js +11 -2
  121. package/dist/server/serverHelpers/Request.js.map +1 -1
  122. package/dist/server/serverHelpers/contentTypes.js +8 -3
  123. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  124. package/dist/server/serverHelpers/uwsServer.js +38 -7
  125. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  126. package/dist/server/threads/manageThreads.d.ts +3 -3
  127. package/dist/server/threads/manageThreads.js +15 -13
  128. package/dist/server/threads/manageThreads.js.map +1 -1
  129. package/dist/server/threads/threadServer.d.ts +3 -0
  130. package/dist/server/threads/threadServer.js +24 -6
  131. package/dist/server/threads/threadServer.js.map +1 -1
  132. package/dist/upgrade/directives/5-3-0.js +9 -3
  133. package/dist/upgrade/directives/5-3-0.js.map +1 -1
  134. package/dist/utility/durableFile.d.ts +36 -0
  135. package/dist/utility/durableFile.js +106 -0
  136. package/dist/utility/durableFile.js.map +1 -0
  137. package/dist/utility/errors/hdbError.d.ts +4 -0
  138. package/dist/utility/errors/hdbError.js +10 -1
  139. package/dist/utility/errors/hdbError.js.map +1 -1
  140. package/dist/utility/globalSchema.d.ts +9 -0
  141. package/dist/utility/hdbTerms.d.ts +1 -0
  142. package/dist/utility/hdbTerms.js +1 -0
  143. package/dist/utility/hdbTerms.js.map +1 -1
  144. package/dist/validation/configValidator.js +29 -0
  145. package/dist/validation/configValidator.js.map +1 -1
  146. package/index.ts +7 -0
  147. package/json/systemSchema.json +14 -0
  148. package/npm-shrinkwrap.json +22 -22
  149. package/package.json +2 -2
  150. package/resources/DESIGN.md +57 -272
  151. package/resources/DatabaseTransaction.ts +708 -87
  152. package/resources/FULL_TEXT.md +6 -2
  153. package/resources/IterableEventQueue.ts +14 -7
  154. package/resources/LMDBTransaction.ts +212 -84
  155. package/resources/RecordEncoder.ts +2 -3
  156. package/resources/Resource.ts +5 -6
  157. package/resources/ResourceInterface.ts +3 -1
  158. package/resources/RocksIndexStore.ts +5 -0
  159. package/resources/Table.ts +118 -15
  160. package/resources/analytics/write.ts +6 -2
  161. package/resources/audit-retention.md +203 -0
  162. package/resources/auditStore.ts +306 -84
  163. package/resources/branchDatabase.ts +2 -0
  164. package/resources/databases.ts +6 -2
  165. package/resources/indexes/DESIGN.md +120 -177
  166. package/resources/indexes/fullTextNativeBinding.ts +22 -3
  167. package/resources/indexes/fullTextQueryIndex.ts +274 -11
  168. package/resources/models/DESIGN.md +11 -3
  169. package/resources/models/Models.ts +127 -4
  170. package/resources/models/backendRegistry.ts +17 -0
  171. package/resources/models/bootstrap.ts +9 -1
  172. package/resources/models/calibration.ts +298 -0
  173. package/resources/models/calibrationStore.ts +1167 -0
  174. package/resources/models/decisionStore.ts +57 -0
  175. package/resources/models/generativeDecision.ts +54 -16
  176. package/resources/models/types.ts +6 -0
  177. package/resources/record-locks.md +155 -141
  178. package/resources/recordLock.ts +12 -6
  179. package/resources/replayLogs.ts +9 -2
  180. package/resources/scheduler/DESIGN.md +4 -3
  181. package/resources/scheduler/engine.ts +35 -2
  182. package/resources/search.ts +154 -16
  183. package/resources/transaction.ts +50 -5
  184. package/resources/transactionBroadcast.ts +69 -0
  185. package/security/DESIGN.md +9 -30
  186. package/security/keys.ts +175 -127
  187. package/server/DESIGN.md +70 -36
  188. package/server/http.ts +10 -0
  189. package/server/serverHelpers/Request.ts +10 -2
  190. package/server/serverHelpers/contentTypes.ts +5 -3
  191. package/server/serverHelpers/uwsServer.ts +34 -7
  192. package/server/threads/manageThreads.js +16 -14
  193. package/server/threads/threadServer.js +24 -6
  194. package/studio/web/assets/{Chat-CNEKMllJ.js → Chat-4LMxI0yB.js} +1 -1
  195. package/studio/web/assets/{FloatingChat-CEnWhHeR.js → FloatingChat-kWHApEtF.js} +3 -3
  196. package/studio/web/assets/{SearchDialog-CwhJdEdY.js → SearchDialog-B7ymna97.js} +1 -1
  197. package/studio/web/assets/{apiToken-DC5vsaaO.js → apiToken-CE3AAt9f.js} +1 -1
  198. package/studio/web/assets/{applications-DQZxDy0x.js → applications-DdN7CqsB.js} +1 -1
  199. package/studio/web/assets/{cssMode-Dz2DxF4e.js → cssMode-BpySZUSt.js} +1 -1
  200. package/studio/web/assets/{editor-B3mjd3tz.js → editor-myLCVv9C.js} +1 -1
  201. package/studio/web/assets/{html-BsRGM7tp.js → html-DGNYO55O.js} +1 -1
  202. package/studio/web/assets/{htmlMode-tUKJaA3o.js → htmlMode-BUgpQr41.js} +1 -1
  203. package/studio/web/assets/{index-BuqFjN5K.js → index-FN4KC9va.js} +39 -39
  204. package/studio/web/assets/index-QLZw6xrU.css +1 -0
  205. package/studio/web/assets/{index.lazy-CcpfQFY_.js → index.lazy-DZFuaumG.js} +1 -1
  206. package/studio/web/assets/{javascript-BZ_uRGHl.js → javascript-DiBzh4pj.js} +1 -1
  207. package/studio/web/assets/{jsonMode-DgB-aFMj.js → jsonMode-Bm312nmu.js} +1 -1
  208. package/studio/web/assets/{languageServices-BleuBw17.js → languageServices-Dz2K9rZN.js} +1 -1
  209. package/studio/web/assets/{lspLanguageFeatures-ByVYLSsQ.js → lspLanguageFeatures-CNu1L5xn.js} +1 -1
  210. package/studio/web/assets/{notifications-TuzN0IMi.js → notifications--nwFLGmM.js} +1 -1
  211. package/studio/web/assets/{notifications-D1Nf-Edn.js → notifications-BelKw5bR.js} +1 -1
  212. package/studio/web/assets/{profile-D7C7Lupz.js → profile-AU5FjOBx.js} +1 -1
  213. package/studio/web/assets/{regions-nGQCm-v8.js → regions-C0sTSEHH.js} +1 -1
  214. package/studio/web/assets/{register-C3x9Ze6P.js → register-DyNR2rGk.js} +2 -2
  215. package/studio/web/assets/{setComponentFile-DpjrRqeG.js → setComponentFile-BkWx8AN3.js} +1 -1
  216. package/studio/web/assets/{setup-CGiKLZ9e.js → setup-D6LZS78d.js} +2 -2
  217. package/studio/web/assets/{status-CGJwrNPH.js → status-BCWnn70o.js} +1 -1
  218. package/studio/web/assets/{toggleHighContrast-C-Ne6P7T.js → toggleHighContrast-pyeGJ6W1.js} +1 -1
  219. package/studio/web/assets/{tsMode-CSreYTys.js → tsMode-CGV3iaWC.js} +1 -1
  220. package/studio/web/assets/{typescript-MnxBk89W.js → typescript-Ci7Khvwo.js} +1 -1
  221. package/studio/web/assets/{useEntityRestURL-TNrh77-s.js → useEntityRestURL-DNiz3aZ0.js} +1 -1
  222. package/studio/web/assets/{workers-BfZlUNVI.js → workers-byzxjbqy.js} +1 -1
  223. package/studio/web/assets/{xml-BHd6qGH7.js → xml-DGnHQ3CX.js} +1 -1
  224. package/studio/web/assets/{yaml-C53R8OQq.js → yaml-BUoTnPWb.js} +1 -1
  225. package/studio/web/index.html +2 -2
  226. package/upgrade/directives/5-3-0.ts +10 -3
  227. package/utility/durableFile.ts +96 -0
  228. package/utility/errors/hdbError.ts +9 -0
  229. package/utility/hdbTerms.ts +1 -0
  230. package/validation/configValidator.ts +31 -0
  231. package/studio/web/assets/index-72cEH474.css +0 -1
@@ -49,6 +49,7 @@ exports.replicationConfirmation = replicationConfirmation;
49
49
  exports.transactionOpenTooLongError = transactionOpenTooLongError;
50
50
  exports.getReadTransactionGuard = getReadTransactionGuard;
51
51
  exports.trackReadRange = trackReadRange;
52
+ exports.requestAbortedError = requestAbortedError;
52
53
  exports.closeWriteInstance = closeWriteInstance;
53
54
  exports.validateWrite = validateWrite;
54
55
  exports.getAppliedWriteVersion = getAppliedWriteVersion;
@@ -57,6 +58,7 @@ exports.writeKeyId = writeKeyId;
57
58
  exports.shouldSpareCommitPhase = shouldSpareCommitPhase;
58
59
  exports.isReleasedTransaction = isReleasedTransaction;
59
60
  exports.isJoinableScope = isJoinableScope;
61
+ exports.deferForCommitInFlight = deferForCommitInFlight;
60
62
  exports.resetReplayedWritesWarning = resetReplayedWritesWarning;
61
63
  exports.setMaxOutstandingTxnDuration = setMaxOutstandingTxnDuration;
62
64
  exports.isWriteSupervised = isWriteSupervised;
@@ -79,6 +81,7 @@ const trackedTxns = new Set();
79
81
  const readTransactionOwners = new WeakMap();
80
82
  // Read options for a rotated generation's native transactions; shared because they never vary.
81
83
  const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
84
+ const ONCE = Object.freeze({ once: true });
82
85
  // Logical transactions the monitor supervises for their WRITES, kept apart from trackedTxns because the
83
86
  // two have different units and different consumers: trackedTxns is per-link, bounds a read snapshot, and
84
87
  // is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the
@@ -500,6 +503,10 @@ function trackReadRange(transaction, createRange) {
500
503
  };
501
504
  return range;
502
505
  }
506
+ function requestAbortedError() {
507
+ // 499 (client closed request) rather than a retryable 503/408: no client is left to retry.
508
+ return new hdbError_ts_1.ServerError('Transaction was aborted because the client disconnected', 499);
509
+ }
503
510
  function closeWriteInstance(operation) {
504
511
  if (operation && !operation.instanceClosed) {
505
512
  operation.instanceClosed = true;
@@ -552,6 +559,11 @@ function writeKeyId(key) {
552
559
  return key;
553
560
  return (0, ordered_binary_1.toBufferKey)(key).toString('latin1');
554
561
  }
562
+ function clearAttemptState(txn) {
563
+ txn.poisonedMidCommit = false;
564
+ txn.postSubmitPoisoned = false;
565
+ txn.stalledCommitResourcesReleased = false;
566
+ }
555
567
  class DatabaseTransaction {
556
568
  #context;
557
569
  // Whether a resources/transaction.ts scope owns this instance — i.e. a final commit or abort is
@@ -611,6 +623,20 @@ class DatabaseTransaction {
611
623
  txn.commitPhaseTicks = 0;
612
624
  }
613
625
  }
626
+ // Depth of commit() attempts on this link that have not reached a native outcome.
627
+ commitsInFlight = 0;
628
+ // Per-link submission state. A commit method can still be parked in pre-commit work, where an abort
629
+ // is safe; once storage work has been submitted, its outcome is unknown and monitor cleanup must not
630
+ // clear writes or blobs that the eventual commit can reference.
631
+ nativeCommitSubmitted = false;
632
+ // Root-only aggregate, retained until every attempt in the multi-store chain settles. An earlier
633
+ // link can have submitted before a later link parks in pre-commit work, so a per-link check alone
634
+ // cannot prove that aborting the logical transaction is safe.
635
+ commitSubmitted = false;
636
+ // Keeps even a single-link submitted commit visible to the long-transaction monitor after its
637
+ // native handle has been detached from the read registry.
638
+ submittedCommitSupervised = false;
639
+ committingWrites = false;
614
640
  rangeReadActive = false;
615
641
  renewReadTimeout() {
616
642
  // The limit is an IDLE limit. Writes always re-arm it (see addWrite), but reads only do so
@@ -658,6 +684,8 @@ class DatabaseTransaction {
658
684
  coordinatedRetry: true,
659
685
  disableSnapshot: disableSnapshot || this.snapshotFree,
660
686
  }));
687
+ if (disableSnapshot || this.snapshotFree)
688
+ this.transaction.snapshotDisabled = true;
661
689
  if (this.timestamp) {
662
690
  this.transaction.setTimestamp(this.timestamp);
663
691
  }
@@ -710,6 +738,8 @@ class DatabaseTransaction {
710
738
  if (link.writeSupervised)
711
739
  return;
712
740
  }
741
+ if (root.submittedCommitSupervised)
742
+ return;
713
743
  supervisedWriteRoots.delete(root);
714
744
  }
715
745
  detachOwnedTransaction() {
@@ -723,6 +753,49 @@ class DatabaseTransaction {
723
753
  this.handleOpenedAt = 0;
724
754
  return transaction;
725
755
  }
756
+ /**
757
+ * Record that `iterator` owes a doneReadTxn(). Without the record, a retained handle's only
758
+ * reclamation path is an onDone() an abandoned request leaves nobody to call.
759
+ */
760
+ registerReadIterator(iterator) {
761
+ // A transaction with no handle of its own (an ImmediateTransaction, which a context reuses for
762
+ // every search it makes) has nothing to reclaim and is in no monitor's registry, so an entry here
763
+ // would never be swept — it would just pin every partially-consumed result set on that context.
764
+ if (!this.transaction && !this.readTxn)
765
+ return;
766
+ (this.ownedReadIterators ??= new Set()).add(iterator);
767
+ }
768
+ unregisterReadIterator(iterator) {
769
+ this.ownedReadIterators?.delete(iterator);
770
+ }
771
+ /**
772
+ * Close every read iterator this transaction still owns, returning how many references came back.
773
+ * Idempotent — each onDone() clears itself — but only safe once nothing can still be consuming them.
774
+ */
775
+ closeOwnedReadIterators() {
776
+ let closed = 0;
777
+ // The whole multi-store chain: a handler that searched a second database registered that
778
+ // iterator on the `next` link that owns its handle, not on the head.
779
+ for (let txn = this; txn; txn = txn.next) {
780
+ const iterators = txn.ownedReadIterators;
781
+ if (!iterators?.size)
782
+ continue;
783
+ // Snapshot first: onDone() deregisters, mutating the set as we walk it.
784
+ for (const iterator of [...iterators]) {
785
+ iterators.delete(iterator);
786
+ try {
787
+ if (iterator.onDone) {
788
+ iterator.onDone();
789
+ closed++;
790
+ }
791
+ }
792
+ catch (error) {
793
+ harperLogger.warn?.('Failed to close a read iterator owned by an abandoned transaction', error);
794
+ }
795
+ }
796
+ }
797
+ return closed;
798
+ }
726
799
  useReadTxn(disableSnapshot) {
727
800
  const readTxn = this.getReadTxn(disableSnapshot);
728
801
  // stackTraces is seeded by the same getReadTxn branch that registers the transaction with
@@ -759,6 +832,17 @@ class DatabaseTransaction {
759
832
  this.completeDeferredContextRelease();
760
833
  }
761
834
  }
835
+ releaseRetainedWriteIntents() {
836
+ if (this.writesAbandoned)
837
+ return;
838
+ this.writesAbandoned = true;
839
+ try {
840
+ this.transaction?.abandonWrites?.();
841
+ }
842
+ catch (error) {
843
+ harperLogger.warn?.('Failed to release write intents on a retained read transaction', error);
844
+ }
845
+ }
762
846
  /**
763
847
  * Force-release the retained native read handle without touching staged writes. Used by the
764
848
  * long-transaction monitor for a CLOSED (already-acknowledged) transaction whose iterators have
@@ -1051,8 +1135,11 @@ class DatabaseTransaction {
1051
1135
  completion.then(undefined, (error) => harperLogger.warn?.('A staged transaction completion failed after the transaction was aborted', error));
1052
1136
  }
1053
1137
  addWrite(operation) {
1054
- if (this.timedOut)
1138
+ if (this.timedOut || this.postSubmitPoisoned)
1055
1139
  throw transactionOpenTooLongError();
1140
+ if (this.disconnected)
1141
+ throw requestAbortedError();
1142
+ this.admitRequestWrite();
1056
1143
  // A write is activity: it re-arms the idle limit on this link even though the reads it
1057
1144
  // performs no longer do (see getReadTxn), so a transaction that keeps writing stays alive
1058
1145
  // and only an idle one holding write intents is reaped.
@@ -1091,6 +1178,20 @@ class DatabaseTransaction {
1091
1178
  return operation;
1092
1179
  }
1093
1180
  save(operation, transaction, reloadEntry = false, options) {
1181
+ if (!transaction && !options) {
1182
+ if (this.timedOut || this.postSubmitPoisoned)
1183
+ throw transactionOpenTooLongError();
1184
+ if (this.disconnected)
1185
+ throw requestAbortedError();
1186
+ // A deferred save is staged by addWrite() but admitted again when it is finally saved.
1187
+ this.admitRequestWrite();
1188
+ }
1189
+ if (!transaction && this.open !== exports.TRANSACTION_STATE.OPEN) {
1190
+ if (this.timedOut || this.postSubmitPoisoned)
1191
+ throw transactionOpenTooLongError();
1192
+ if (this.disconnected)
1193
+ throw requestAbortedError();
1194
+ }
1094
1195
  const lockHandle = operation.lockHandle;
1095
1196
  // Guard: a write staged through an expired or released lock handle must not land.
1096
1197
  // The handle's lease timer already unlocked the native key; another holder may have taken it.
@@ -1195,10 +1296,21 @@ class DatabaseTransaction {
1195
1296
  // against; a snapshot-free transaction (this.snapshotFree, after a mid-scope-commit rotation)
1196
1297
  // has no snapshot for rocksdb-js to validate the Put against, so it only narrows the window to
1197
1298
  // the read-to-put span rather than closing it (open follow-up, tracked in the PR description).
1198
- // Replays keep their pre-read base — their convergence contract is the replay pass itself.
1299
+ // Replay bypasses the cache: WeakRef targets survive the entire synchronous job, even after eviction.
1199
1300
  const reloadsCommitBase = operation.reloadCommitBase && !operation.saved && !this.isReplay;
1200
- if (reloadEntry || operation.entry === undefined || reloadsCommitBase) {
1201
- const uncachedRead = (!!operation.reloadCommitBase && !this.isReplay) || reloadEntry;
1301
+ // An entry read uncached through this same pinned-snapshot handle is already that base, until a retry resets
1302
+ // the snapshot or an earlier staged write to the key changes what a read returns; the resource withholds the
1303
+ // handle once its entry was replaced or evicted, and locked, snapshot-free and replayed writes always reload.
1304
+ const reusesBaseRead = operation.baseReadTxn === transaction &&
1305
+ this.retries === 0 &&
1306
+ !reloadEntry &&
1307
+ !operation.priorWrite &&
1308
+ !operation.lockHandle &&
1309
+ !this.snapshotFree &&
1310
+ !transaction.snapshotDisabled &&
1311
+ !this.isReplay;
1312
+ if (!reusesBaseRead && (reloadEntry || operation.entry === undefined || reloadsCommitBase)) {
1313
+ const uncachedRead = !!operation.reloadCommitBase || reloadEntry || this.isReplay;
1202
1314
  operation.entry = operation.store.getEntry(operation.key, { transaction, uncachedRead });
1203
1315
  }
1204
1316
  if (!operation.saved) {
@@ -1253,11 +1365,44 @@ class DatabaseTransaction {
1253
1365
  }
1254
1366
  }
1255
1367
  /**
1256
- * Resolves with information on the timestamp and success of the commit
1368
+ * Whether any link of this multi-store chain has a commit attempt that has not reached a native
1369
+ * outcome. Asked of the whole chain, not just this link: the head commits its own writes and then
1370
+ * cascades into `next`, so that cascade is a continuation of one logical commit, not a fresh one.
1371
+ * Walked from here as well when a settling commit has already detached this link from that chain.
1372
+ */
1373
+ isChainCommitting() {
1374
+ let attached = false;
1375
+ const root = this.root ?? this;
1376
+ for (let txn = root; txn; txn = txn.next) {
1377
+ if (txn.commitsInFlight)
1378
+ return true;
1379
+ if (txn === this)
1380
+ attached = true;
1381
+ }
1382
+ if (submittedCommitStillInFlight(root))
1383
+ return true;
1384
+ if (attached)
1385
+ return false;
1386
+ // Only a link a settling commit already detached needs its own walk, and its own attempt is
1387
+ // exactly the one that must not be torn out from under it. The ordinary case — asked of the root,
1388
+ // or of a link still on the chain — is answered by the single walk above.
1389
+ for (let txn = this; txn; txn = txn.next) {
1390
+ if (txn.commitsInFlight)
1391
+ return true;
1392
+ }
1393
+ return false;
1394
+ }
1395
+ /**
1396
+ * A submitted native write is this transaction's point of no destructive return. Only a FRESH
1397
+ * attempt — none already in flight, and no `options.transaction` from an immediate-commit re-entry —
1398
+ * is rejected by the poison flags. Everything that continues an already-submitted attempt (a
1399
+ * RETRY_NOW/ERR_BUSY/ERR_TRY_AGAIN recursion, the extra-writes recursion inside performCommit) must
1400
+ * reach its native outcome instead of abandoning unknown state; a poison that lands before submission
1401
+ * can still abort safely (see abortAndPoison).
1402
+ *
1403
+ * Resolves with information on the timestamp and success of the commit.
1257
1404
  */
1258
1405
  commit(options = {}) {
1259
- if (this.timedOut)
1260
- throw transactionOpenTooLongError();
1261
1406
  if (this.monitorCommit && !options.transaction) {
1262
1407
  return this.monitorCommit.then(() => this.commit(options), (error) => {
1263
1408
  // The failed commit ran its terminal cleanup, but as a non-final commit it kept the context.
@@ -1268,6 +1413,233 @@ class DatabaseTransaction {
1268
1413
  throw error;
1269
1414
  });
1270
1415
  }
1416
+ if (!(options.transaction && this.commitsInFlight) && !(options.continuation && this.poisonedMidCommit)) {
1417
+ if (this.timedOut || this.postSubmitPoisoned)
1418
+ throw transactionOpenTooLongError();
1419
+ if (this.disconnected)
1420
+ throw requestAbortedError();
1421
+ }
1422
+ this.commitsInFlight = (this.commitsInFlight ?? 0) + 1;
1423
+ // Only a top-level attempt records the intent: a continuation would re-read hasPendingWrites()
1424
+ // after clearWrites() has already emptied it.
1425
+ if (this.commitsInFlight === 1 && !options.continuation && this.hasPendingWrites()) {
1426
+ (this.root ?? this).committingWrites = true;
1427
+ }
1428
+ let resolution;
1429
+ try {
1430
+ resolution = this.performCommit(options);
1431
+ }
1432
+ catch (error) {
1433
+ this.endCommitAttempt();
1434
+ throw error;
1435
+ }
1436
+ if (!resolution?.then) {
1437
+ this.endCommitAttempt();
1438
+ return resolution;
1439
+ }
1440
+ return resolution.then((settled) => {
1441
+ this.endCommitAttempt();
1442
+ return settled;
1443
+ }, (error) => {
1444
+ this.endCommitAttempt();
1445
+ throw error;
1446
+ });
1447
+ }
1448
+ markNativeCommitSubmitted() {
1449
+ this.nativeCommitSubmitted = true;
1450
+ const root = this.root ?? this;
1451
+ root.commitSubmitted = true;
1452
+ if (!root.submittedLink)
1453
+ root.submittedLink = this;
1454
+ else if (root.submittedLink !== this)
1455
+ (root.submittedLinks ??= new Set()).add(this);
1456
+ // LMDB has its own monitor registry and overrides this method to enroll there. Do not put an
1457
+ // LMDB link into the RocksDB monitor as well (stagesWriteOnSave is the engine discriminator).
1458
+ if (this.stagesWriteOnSave) {
1459
+ root.submittedCommitSupervised = true;
1460
+ supervisedWriteRoots.add(root);
1461
+ }
1462
+ }
1463
+ /** Engine hook for a submission that has reached a native outcome. */
1464
+ nativeCommitAttemptEnded() { }
1465
+ /**
1466
+ * True when a commit attempt is parked on this link that can no longer land: it never submitted, and
1467
+ * the link is poisoned, so performCommit()'s post-await poison check throws before it commits. The
1468
+ * monitor's commit-phase grace exists to let real pre-commit work finish; extending it to an attempt
1469
+ * with no outcome left only pins this link's read snapshot for the rest of the grace.
1470
+ */
1471
+ commitAttemptDoomedByPoison() {
1472
+ return !this.nativeCommitSubmitted && !this.poisonedMidCommit && Boolean(this.timedOut || this.disconnected);
1473
+ }
1474
+ /**
1475
+ * Admission for a new write on behalf of a request. A request already cancelled poisons the chain
1476
+ * (releasing whatever it staged) and is refused; otherwise the chain subscribes to the cancellation
1477
+ * for as long as it owns writes, so a disconnect releases its intents at once instead of when its
1478
+ * scope or pre-commit work happens to finish. Reads never subscribe.
1479
+ */
1480
+ admitRequestWrite() {
1481
+ const root = this.root ?? this;
1482
+ const signal = root.requestSignal;
1483
+ if (!signal)
1484
+ return;
1485
+ if (signal.aborted) {
1486
+ this.abortDueToDisconnect();
1487
+ throw requestAbortedError();
1488
+ }
1489
+ if (root.requestAbortListener)
1490
+ return;
1491
+ const listener = () => {
1492
+ root.requestAbortListener = undefined;
1493
+ try {
1494
+ // isCommittingWrites() is not redundant: commit() marks itself CLOSED and clears its staged
1495
+ // writes before the native commit settles, so for that window the chain reads as idle.
1496
+ if ((root.open === exports.TRANSACTION_STATE.OPEN && root.hasPendingWrites()) || root.isCommittingWrites())
1497
+ root.abortDueToDisconnect();
1498
+ }
1499
+ catch (error) {
1500
+ harperLogger.debug?.('aborting transaction on client disconnect', error);
1501
+ }
1502
+ };
1503
+ root.requestAbortListener = listener;
1504
+ signal.addEventListener('abort', listener, ONCE);
1505
+ }
1506
+ releaseRequestAbortListener() {
1507
+ const listener = this.requestAbortListener;
1508
+ if (!listener)
1509
+ return;
1510
+ this.requestAbortListener = undefined;
1511
+ this.requestSignal.removeEventListener('abort', listener);
1512
+ }
1513
+ /** Protected work has no proven resume path, so the monitor may observe but never poison it. */
1514
+ isProtectedCommit() {
1515
+ const root = this.root ?? this;
1516
+ let attached = false;
1517
+ for (let txn = root; txn; txn = txn.next) {
1518
+ if (txn.sourceApply || txn.isReplay)
1519
+ return true;
1520
+ if (txn === this)
1521
+ attached = true;
1522
+ }
1523
+ return !attached && Boolean(this.sourceApply || this.isReplay);
1524
+ }
1525
+ /**
1526
+ * Reclaim only read resources that are separate from already-submitted writes. Never run this on a
1527
+ * pre-submit link: its native read handle can still carry the write batch the commit needs.
1528
+ */
1529
+ releaseStalledCommitReadResources() {
1530
+ const release = (txn) => {
1531
+ if (!txn.nativeCommitSubmitted || txn.stalledCommitResourcesReleased)
1532
+ return;
1533
+ txn.stalledCommitResourcesReleased = true;
1534
+ if (txn.closeOwnedReadIterators() > 0)
1535
+ harperLogger.warn?.(`Read iterators held a submitted transaction's snapshot after its native commit stalled; releasing it, from table: ${txn.db?.name}`);
1536
+ if (txn.transaction || txn.readTxn)
1537
+ txn.releaseReadTxn();
1538
+ };
1539
+ const root = this.root ?? this;
1540
+ let attached = false;
1541
+ for (let txn = root; txn; txn = txn.next) {
1542
+ release(txn);
1543
+ if (txn === this)
1544
+ attached = true;
1545
+ }
1546
+ if (!attached)
1547
+ release(this);
1548
+ }
1549
+ endCommitAttempt() {
1550
+ this.commitsInFlight--;
1551
+ if (!this.commitsInFlight) {
1552
+ this.nativeCommitSubmitted = false;
1553
+ this.nativeCommitAttemptEnded();
1554
+ }
1555
+ // A link closes its OWN iterators the moment its own attempt settles, without waiting for the
1556
+ // chain: LMDBTransaction's commit detaches `next` before awaiting the child, so a scope abandoned
1557
+ // in that window never captured this link and the chain-wide cleanup below cannot reach it. Safe
1558
+ // here and not earlier because this link's attempt is the one that just reached its outcome.
1559
+ if ((this.root ?? this).scopeAbandoned)
1560
+ this.closeOwnedReadIterators();
1561
+ if (this.isChainCommitting())
1562
+ return;
1563
+ // The window has closed: a commit entered from here on is a fresh one, and the flags that landed
1564
+ // during the finished attempt must reject it like any other poisoned commit.
1565
+ const root = this.root ?? this;
1566
+ root.committingWrites = false;
1567
+ root.commitSubmitted = false;
1568
+ root.submittedCommitSupervised = false;
1569
+ root.deferredPoisonDeadline = undefined;
1570
+ root.unsubmittedCommitDeadline = undefined;
1571
+ root.stalledCommitLogged = false;
1572
+ const submittedLink = root.submittedLink;
1573
+ root.submittedLink = undefined;
1574
+ const submittedLinks = root.submittedLinks;
1575
+ root.submittedLinks = undefined;
1576
+ if (submittedLink)
1577
+ clearAttemptState(submittedLink);
1578
+ if (submittedLinks)
1579
+ for (const txn of submittedLinks)
1580
+ clearAttemptState(txn);
1581
+ let stillWriteSupervised = false;
1582
+ for (let txn = root; txn; txn = txn.next) {
1583
+ clearAttemptState(txn);
1584
+ if (txn.writeSupervised)
1585
+ stillWriteSupervised = true;
1586
+ }
1587
+ if (!stillWriteSupervised)
1588
+ supervisedWriteRoots.delete(root);
1589
+ if (!root.hasPendingWrites())
1590
+ root.releaseRequestAbortListener();
1591
+ if (root.scopeAbandoned) {
1592
+ root.scopeAbandoned = false;
1593
+ // The links captured when the scope was abandoned, not the current `next` chain: a successful
1594
+ // multi-store commit clears `next` before this runs, which would otherwise leave a former
1595
+ // child's iterators owning a snapshot nothing can reach.
1596
+ const abandoned = root.abandonedLinks;
1597
+ root.abandonedLinks = undefined;
1598
+ if (abandoned)
1599
+ for (const link of abandoned)
1600
+ link.closeOwnedReadIterators();
1601
+ else
1602
+ root.closeOwnedReadIterators();
1603
+ }
1604
+ }
1605
+ /**
1606
+ * The scope that owned this transaction ended while a commit attempt was still in flight, so the
1607
+ * wrapper could not abort it. Surrender scope ownership now, or the attempt's own
1608
+ * rotateAfterMidScopeCommit reopens an instance with no wrapper left to commit or abort it and the
1609
+ * next transaction() on that context joins it and never commits. Iterator cleanup is deferred to
1610
+ * endCommitAttempt(), since closing them mid-attempt can pull the handle the commit is still
1611
+ * staging through.
1612
+ */
1613
+ abandonScope() {
1614
+ const root = this.root ?? this;
1615
+ root.scopeAbandoned = true;
1616
+ const links = [];
1617
+ for (let txn = this; txn; txn = txn.next) {
1618
+ links.push(txn);
1619
+ txn.endScopeOwnership();
1620
+ // Ending ownership alone only stops the attempt's own rotation; until it reaches its native
1621
+ // outcome the instance is still OPEN, and a hung `before` hook makes that window arbitrarily
1622
+ // long. Submitted attempts are monitor-deferred; a pre-submit attempt remains safely abortable.
1623
+ // Not on its own sufficient:
1624
+ // LMDBTransaction's own commit reassigns `open`, so the release below is what actually keeps
1625
+ // the next write on this context off an instance with no wrapper behind it.
1626
+ txn.open = exports.TRANSACTION_STATE.CLOSED;
1627
+ }
1628
+ root.abandonedLinks = links;
1629
+ // By abort()'s own rule, so a poisoned chain stays joined and throws while an ordinary one lets
1630
+ // the next write start fresh.
1631
+ this.releaseContext(!this.timedOut && !this.disconnected);
1632
+ }
1633
+ /**
1634
+ * Whether this chain has a write-bearing commit attempt in flight. `hasPendingWrites()` cannot
1635
+ * answer that: commit() marks itself CLOSED and clears its staged writes well before the native
1636
+ * commit settles, so between those points a write-bearing transaction looks read-only and idle to
1637
+ * anything reading its public state — including transaction.ts's disconnect listener.
1638
+ */
1639
+ isCommittingWrites() {
1640
+ return Boolean((this.root ?? this).committingWrites) && this.isChainCommitting();
1641
+ }
1642
+ performCommit(options = {}) {
1271
1643
  // reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain —
1272
1644
  // but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction
1273
1645
  let transaction = options.transaction ?? this.transaction;
@@ -1295,447 +1667,455 @@ class DatabaseTransaction {
1295
1667
  if (completions.length > 0) {
1296
1668
  this.setCommitPhase(true);
1297
1669
  }
1298
- return (0, when_ts_1.when)(completions.length > 0 ? Promise.all(completions) : null, () => {
1299
- if (completions.length > 0)
1300
- this.setCommitPhase(false);
1301
- // The transaction can be aborted underneath us while we are parked in the await above — by the
1302
- // monitor once the commit phase outlives its grace, or through the multi-store poison chain.
1303
- // abort() cleared the write set and released the handle, so resuming would commit nothing and
1304
- // resolve as SUCCESS: the caller is told its write landed when it was dropped, and a write
1305
- // carrying a blob is left holding an instance whose file was unlinked (issue #2062).
1306
- if (this.timedOut)
1307
- throw transactionOpenTooLongError();
1308
- if (stagedWrites > 0 && this.writes.length === 0 && this.open === exports.TRANSACTION_STATE.CLOSED)
1309
- throw new hdbError_ts_1.ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
1310
- if (this.writes.length > this.validated) {
1311
- // check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
1312
- return this.commit(options);
1313
- }
1314
- // The save loop above can be what opened this transaction's native handle — save() attaches
1315
- // one when it had none, which is every ImmediateTransaction commit since its getReadTxn
1316
- // opens none — leaving the local captured before the loop empty while that handle holds
1317
- // every staged write, for the detach below to drop uncommitted (issue #2288). Only when
1318
- // the local is empty: a truthy one is what the loop staged into, and the retained-handle
1319
- // and replay branches below deliberately commit a handle other than this.transaction.
1320
- if (!transaction)
1321
- transaction = this.transaction;
1322
- if (!options.transaction && this.writes.some((write) => write))
1323
- this.renewChainForNativeCommit();
1324
- this.open = exports.TRANSACTION_STATE.CLOSED;
1325
- // RocksTransaction.commit() resolves with RETRY_NOW_VALUE (a number) under
1326
- // coordinatedRetry, or void on a normal commit/abort.
1327
- let commitResolution;
1328
- // Consume this commit's own read reference — exactly once per read handle: retry
1329
- // recursions, immediate-commit re-entries, and a second top-level commit() (wrapper
1330
- // commit after an explicit in-handler commit) must not steal a reference owned by an
1331
- // outstanding iterator (doneReadTxn() would then never release the native handle).
1332
- if (!this.baseReadRefConsumed) {
1333
- this.baseReadRefConsumed = true;
1334
- this.readTxnsUsed--;
1335
- }
1336
- if (this.readTxnsUsed > 0) {
1337
- // Outstanding iterators still stream through this.transaction — their native iterators
1338
- // live inside it (GetIterator wraps its write batch + snapshot), so committing or
1339
- // aborting the handle now would invalidate them mid-stream. Leave it open for the
1340
- // iterators (doneReadTxn() aborts it when the last one finishes) and commit the writes
1341
- // NOW by replaying them onto a fresh transaction, the same shape as the ERR_TRY_AGAIN
1342
- // replay below: entries reload through the new transaction and re-resolve against
1343
- // current state, and conflicts surface through the normal retry ladder. Deferring the
1344
- // native commit to doneReadTxn() instead (the old LINGERING state) meant anything that
1345
- // kept the last iterator from finishing cleanly — a hung stream, the long-transaction
1346
- // monitor's timeout abort — dropped writes the caller had already been told committed.
1347
- this.writes = this.writes.filter((write) => write); // filter out removed entries
1348
- if (this.writes.length > 0) {
1349
- if (!options.transaction) {
1350
- if (!replayedWritesWarned) {
1351
- replayedWritesWarned = true;
1352
- harperLogger.warn?.(`Committing while read iterators are still open: ${this.writes.length} staged write(s) must be re-staged and committed on a second transaction, doubling their write work` +
1353
- (this.startedFrom ? `, from ${this.startedFrom.resourceName}.${this.startedFrom.method}` : '') +
1354
- `. Fully consume (or close) iterators before committing to avoid this. Logged once per process.`);
1355
- }
1356
- // Deliberately NOT marked isRetry and NOT carrying over the original's onCommit:
1357
- // audit/txn-log entries batch natively on the transaction they were staged into and
1358
- // are only durably written by that transaction's commit attempt (an abort discards
1359
- // them — unlike the ERR_TRY_AGAIN replay below, where the original's FAILED commit
1360
- // attempt already wrote its log batch). The original handle here never attempts a
1361
- // commit, so the replayed stagings must re-append their entries into the replay
1362
- // transaction's own batch — which also installs the replay's own commit hook.
1363
- const replayTransaction = new rocksdb_js_1.Transaction((this.writes[0].store.store ?? this.db.store), { coordinatedRetry: true });
1364
- if (this.timestamp)
1365
- replayTransaction.setTimestamp(this.timestamp);
1366
- this.retries++; // a replay round: commit handlers re-base on the reloaded entries
1367
- try {
1368
- for (const operation of this.writes) {
1369
- this.save(operation, replayTransaction, true, options);
1370
- }
1371
- }
1372
- catch (error) {
1373
- abortNativeTransaction(replayTransaction, 'aborting a replay transaction whose re-save threw');
1374
- this.abort();
1375
- throw error;
1376
- }
1377
- transaction = replayTransaction;
1378
- }
1379
- // with options.transaction set this is a retry round — the save loop above already
1380
- // re-staged the writes into it
1381
- this.assertDatabaseCommitAllowed(transaction);
1382
- commitResolution = transaction.commit();
1383
- recordCommitLatency(commitResolution, performance.now());
1384
- // Write-queue-depth accounting for this replay commit happens uniformly below, via
1385
- // trackOutstandingCommit(commitResolution) — see that function's comment. Omitting
1386
- // dedicated accounting here (as a prior version of this replay path did) used to leave
1387
- // write-transaction-queue-depth, the one metric that can observe a commit that never
1388
- // settles (harper#2001), reading zero for exactly this path.
1670
+ let commitResult;
1671
+ try {
1672
+ commitResult = (0, when_ts_1.when)(completions.length > 0 ? Promise.all(completions) : null, () => {
1673
+ if (completions.length > 0)
1674
+ this.setCommitPhase(false);
1675
+ // The transaction can be aborted underneath us while we are parked in the await above — by the
1676
+ // monitor once the commit phase outlives its grace, or through the multi-store poison chain.
1677
+ // abort() cleared the write set and released the handle, so resuming would commit nothing and
1678
+ // resolve as SUCCESS: the caller is told its write landed when it was dropped, and a write
1679
+ // carrying a blob is left holding an instance whose file was unlinked (issue #2062).
1680
+ if (!this.poisonedMidCommit) {
1681
+ if (this.timedOut)
1682
+ throw transactionOpenTooLongError();
1683
+ if (this.disconnected)
1684
+ throw requestAbortedError();
1389
1685
  }
1390
- // No commit will ever run on the retained handle — the replay above owns these
1391
- // writes — so this is the only place its write intents can be released. Left in
1392
- // place, other writers' coordinated-retry commits park on them until the last
1393
- // iterator finishes (harper#2001). Reads through the handle, including
1394
- // read-your-own-writes, keep working. Once only: a coordinated-retry or backoff
1395
- // round re-enters this branch on the same retained handle. Fenced like the other
1396
- // post-submit steps here: the replay commit is already in flight, so a throw must
1397
- // not skip onCommit/the chain-store commit below. Optional: rocksdb-js < 2.7
1398
- // lacks the method.
1399
- if (!this.writesAbandoned) {
1400
- this.writesAbandoned = true;
1401
- try {
1402
- this.transaction?.abandonWrites?.();
1403
- }
1404
- catch (error) {
1405
- harperLogger.warn?.('Failed to release write intents on a retained read transaction', error);
1406
- }
1686
+ if (stagedWrites > 0 && this.writes.length === 0 && this.open === exports.TRANSACTION_STATE.CLOSED)
1687
+ throw new hdbError_ts_1.ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
1688
+ if (this.writes.length > this.validated) {
1689
+ // check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
1690
+ return this.commit({ ...options, continuation: true });
1407
1691
  }
1408
- }
1409
- else {
1410
- // no more reads need to be performed, just commit/abort based if there are any writes
1411
- this.detachOwnedTransaction(); // any further operations operate immediately
1412
- if (transaction) {
1692
+ // The save loop above can be what opened this transaction's native handle — save() attaches
1693
+ // one when it had none, which is every ImmediateTransaction commit since its getReadTxn
1694
+ // opens none — leaving the local captured before the loop empty while that handle holds
1695
+ // every staged write, for the detach below to drop uncommitted (issue #2288). Only when
1696
+ // the local is empty: a truthy one is what the loop staged into, and the retained-handle
1697
+ // and replay branches below deliberately commit a handle other than this.transaction.
1698
+ if (!transaction)
1699
+ transaction = this.transaction;
1700
+ if (!options.transaction && this.writes.some((write) => write))
1701
+ this.renewChainForNativeCommit();
1702
+ this.open = exports.TRANSACTION_STATE.CLOSED;
1703
+ // RocksTransaction.commit() resolves with RETRY_NOW_VALUE (a number) under
1704
+ // coordinatedRetry, or void on a normal commit/abort.
1705
+ let commitResolution;
1706
+ // Consume this commit's own read reference — exactly once per read handle: retry
1707
+ // recursions, immediate-commit re-entries, and a second top-level commit() (wrapper
1708
+ // commit after an explicit in-handler commit) must not steal a reference owned by an
1709
+ // outstanding iterator (doneReadTxn() would then never release the native handle).
1710
+ if (!this.baseReadRefConsumed) {
1711
+ this.baseReadRefConsumed = true;
1712
+ this.readTxnsUsed--;
1713
+ }
1714
+ if (this.readTxnsUsed > 0) {
1715
+ // Outstanding iterators still stream through this.transaction — their native iterators
1716
+ // live inside it (GetIterator wraps its write batch + snapshot), so committing or
1717
+ // aborting the handle now would invalidate them mid-stream. Leave it open for the
1718
+ // iterators (doneReadTxn() aborts it when the last one finishes) and commit the writes
1719
+ // NOW by replaying them onto a fresh transaction, the same shape as the ERR_TRY_AGAIN
1720
+ // replay below: entries reload through the new transaction and re-resolve against
1721
+ // current state, and conflicts surface through the normal retry ladder. Deferring the
1722
+ // native commit to doneReadTxn() instead (the old LINGERING state) meant anything that
1723
+ // kept the last iterator from finishing cleanly — a hung stream, the long-transaction
1724
+ // monitor's timeout abort — dropped writes the caller had already been told committed.
1413
1725
  this.writes = this.writes.filter((write) => write); // filter out removed entries
1414
1726
  if (this.writes.length > 0) {
1415
- // Commit retries can construct fresh ranges on this live handle after read ownership ends.
1416
- readTransactionOwners.delete(transaction);
1417
- // Re-fence before submitting: the loop above skips operations already marked saved, so
1418
- // save()'s own check cannot see a holder that stalled between staging and commit. The
1419
- // retry/replay path re-saves every operation and is fenced there instead. Lease expiry
1420
- // only: an unlock() inside the lease leaves the staged write valid, an elapsed lease
1421
- // does not, whether or not the caller also unlocked.
1422
- for (let i = 0; this.hasLeaseProtectedWrite && i < this.writes.length; i++) {
1423
- const lapsed = this.writes[i].lockHandle;
1424
- if (!lapsed?.isLeaseExpired())
1425
- continue;
1426
- try {
1427
- transaction.abort();
1727
+ if (!options.transaction) {
1728
+ if (!replayedWritesWarned) {
1729
+ replayedWritesWarned = true;
1730
+ harperLogger.warn?.(`Committing while read iterators are still open: ${this.writes.length} staged write(s) must be re-staged and committed on a second transaction, doubling their write work` +
1731
+ (this.startedFrom ? `, from ${this.startedFrom.resourceName}.${this.startedFrom.method}` : '') +
1732
+ `. Fully consume (or close) iterators before committing to avoid this. Logged once per process.`);
1428
1733
  }
1429
- catch { }
1430
- // Every other terminal exit from commit() runs the logical cleanup too. Throwing
1431
- // straight out would strand the OTHER locks this transaction holds, its staged
1432
- // blobs, and the context's back-reference to a CLOSED transaction — and
1433
- // transaction()'s onComplete has no rejection path to run it later.
1734
+ // Deliberately NOT marked isRetry and NOT carrying over the original's onCommit:
1735
+ // audit/txn-log entries batch natively on the transaction they were staged into and
1736
+ // are only durably written by that transaction's commit attempt (an abort discards
1737
+ // them — unlike the ERR_TRY_AGAIN replay below, where the original's FAILED commit
1738
+ // attempt already wrote its log batch). The original handle here never attempts a
1739
+ // commit, so the replayed stagings must re-append their entries into the replay
1740
+ // transaction's own batch — which also installs the replay's own commit hook.
1741
+ const replayTransaction = new rocksdb_js_1.Transaction((this.writes[0].store.store ?? this.db.store), { coordinatedRetry: true });
1742
+ if (this.timestamp)
1743
+ replayTransaction.setTimestamp(this.timestamp);
1744
+ this.retries++; // a replay round: commit handlers re-base on the reloaded entries
1434
1745
  try {
1435
- this.abort();
1746
+ for (const operation of this.writes) {
1747
+ this.save(operation, replayTransaction, true, options);
1748
+ }
1436
1749
  }
1437
1750
  catch (error) {
1438
- harperLogger.debug?.('cleaning up a transaction whose record lock lapsed', error);
1751
+ abortNativeTransaction(replayTransaction, 'aborting a replay transaction whose re-save threw');
1752
+ this.abort();
1753
+ throw error;
1439
1754
  }
1440
- throw (0, recordLock_ts_1.lockNotHeldError)(lapsed);
1755
+ transaction = replayTransaction;
1441
1756
  }
1442
- // The transaction was created with coordinatedRetry:true (see
1443
- // getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
1444
- // sentinel (a number) is why commitResolution is typed
1445
- // Promise<number | void>; it is handled in the resolve callback below.
1757
+ // with options.transaction set this is a retry round — the save loop above already
1758
+ // re-staged the writes into it
1446
1759
  this.assertDatabaseCommitAllowed(transaction);
1760
+ this.markNativeCommitSubmitted();
1447
1761
  commitResolution = transaction.commit();
1448
- // Record how long this commit stays outstanding (submit → settle) as a distribution
1449
- // metric. This is the same clock the overload check uses (trackOutstandingCommit
1450
- // stamps each attempt at submit), so a rising p99/p999 is the leading indicator for the
1451
- // "Outstanding write transactions have too long of queue" (503) rejection. A transient-
1452
- // conflict retry rejects this promise and issues a fresh commit(), which is tracked as
1453
- // its own attempt, so recording per attempt matches the overload semantics.
1454
- // commitResolution's declared type (Promise<number | void> | void) doesn't narrow to
1455
- // Promise<void> here because the widening union defeats flow analysis on the prior
1456
- // cast assignment; re-assert it — this branch's commit() result is always a Promise.
1457
1762
  recordCommitLatency(commitResolution, performance.now());
1458
- // Write-queue-depth accounting for this commit happens uniformly below, via
1459
- // trackOutstandingCommit(commitResolution) — see that function's comment. A
1460
- // transient-conflict retry rejects this promise and issues a fresh commit()
1461
- // (re-entering here), which trackOutstandingCommit tracks as its own attempt.
1462
- }
1463
- else {
1464
- try {
1465
- commitResolution = transaction.abort();
1466
- }
1467
- catch {
1468
- // The transaction has uncommitted writes that were already cleared from
1469
- // this.writes by a concurrent immediate-commit path (e.g. writes made with
1470
- // an explicitly-reused closed transaction). Those writes are handled by the
1471
- // concurrent commit, so there is nothing left to do here.
1472
- }
1763
+ // Write-queue-depth accounting for this replay commit happens uniformly below, via
1764
+ // trackOutstandingCommit(commitResolution) — see that function's comment. Omitting
1765
+ // dedicated accounting here (as a prior version of this replay path did) used to leave
1766
+ // write-transaction-queue-depth, the one metric that can observe a commit that never
1767
+ // settles (harper#2001), reading zero for exactly this path.
1473
1768
  }
1769
+ // No commit will ever run on the retained handle — the replay above owns these
1770
+ // writes — so this is the only place its write intents can be released. Left in
1771
+ // place, other writers' coordinated-retry commits park on them until the last
1772
+ // iterator finishes (harper#2001). Reads through the handle, including
1773
+ // read-your-own-writes, keep working. Once only: a coordinated-retry or backoff
1774
+ // round re-enters this branch on the same retained handle. Fenced like the other
1775
+ // post-submit steps here: the replay commit is already in flight, so a throw must
1776
+ // not skip onCommit/the chain-store commit below. Optional: rocksdb-js < 2.7
1777
+ // lacks the method.
1778
+ this.releaseRetainedWriteIntents();
1474
1779
  }
1475
- }
1476
- if (commitResolution) {
1477
- // Read the table off the write itself, not this.db, which is whichever table first
1478
- // claimed this per-database transaction in txnForContext and so can name the wrong
1479
- // table when a transaction spans more than one table in the same database.
1480
- trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
1481
- // Every retry round and every chained store re-enters here and must inherit the chain
1482
- // root's clock rather than restart it, so only the first submission stamps.
1483
- const chainRoot = this.root ?? this;
1484
- if (chainRoot.commitStartedAt == null)
1485
- chainRoot.commitStartedAt = performance.now();
1486
- const completions = [];
1487
- const commitOutcome = commitResolution.then((commitResult) => {
1488
- if (commitResult === RETRY_NOW_VALUE) {
1489
- this.retries++;
1490
- harperLogger.debug?.('coordinated retry', transaction.id, this.retries);
1491
- // Mark this specific native transaction as a retry so RocksTransactionLogStore
1492
- // skips re-writing its already-staged txn-log entries (#2).
1493
- transaction.isRetry = true;
1494
- const pastBudget = this.elapsedPastCommitBudget();
1495
- if (pastBudget)
1496
- this.abandonCommitAfterDeadline(transaction, pastBudget);
1497
- // Mirror the ERR_BUSY cap/warn policy: non-sourceApply transactions abort
1498
- // at MAX_RETRIES; sourceApply transactions keep retrying with periodic warn.
1499
- if (this.retries > MAX_RETRIES) {
1500
- if (!this.sourceApply) {
1501
- // giving up: poison and abort the whole linked chain so no link leaks its native
1502
- // handle / read snapshot — or any unpublished transaction-log position — until GC.
1503
- this.abortChainAfterRetries(transaction);
1504
- throw new hdbError_ts_1.ServerError(`After ${MAX_RETRIES} coordinated retries, unable to commit transaction, transaction is in conflict with ongoing writes`);
1505
- }
1506
- if (this.retries % MAX_RETRIES === 0) {
1507
- harperLogger.warn?.(`Source-applied transaction ${transaction.id} still in conflict after ${this.retries} coordinated retries; continuing to retry`);
1780
+ else {
1781
+ // no more reads need to be performed, just commit/abort based if there are any writes
1782
+ this.detachOwnedTransaction(); // any further operations operate immediately
1783
+ if (transaction) {
1784
+ this.writes = this.writes.filter((write) => write); // filter out removed entries
1785
+ if (this.writes.length > 0) {
1786
+ // Commit retries can construct fresh ranges on this live handle after read ownership ends.
1787
+ readTransactionOwners.delete(transaction);
1788
+ // Re-fence before submitting: the loop above skips operations already marked saved, so
1789
+ // save()'s own check cannot see a holder that stalled between staging and commit. The
1790
+ // retry/replay path re-saves every operation and is fenced there instead. Lease expiry
1791
+ // only: an unlock() inside the lease leaves the staged write valid, an elapsed lease
1792
+ // does not, whether or not the caller also unlocked.
1793
+ for (let i = 0; this.hasLeaseProtectedWrite && i < this.writes.length; i++) {
1794
+ const lapsed = this.writes[i].lockHandle;
1795
+ if (!lapsed?.isLeaseExpired())
1796
+ continue;
1797
+ try {
1798
+ transaction.abort();
1799
+ }
1800
+ catch { }
1801
+ // Every other terminal exit from commit() runs the logical cleanup too. Throwing
1802
+ // straight out would strand the OTHER locks this transaction holds, its staged
1803
+ // blobs, and the context's back-reference to a CLOSED transaction — and
1804
+ // transaction()'s onComplete has no rejection path to run it later.
1805
+ try {
1806
+ this.abort();
1807
+ }
1808
+ catch (error) {
1809
+ harperLogger.debug?.('cleaning up a transaction whose record lock lapsed', error);
1810
+ }
1811
+ throw (0, recordLock_ts_1.lockNotHeldError)(lapsed);
1508
1812
  }
1813
+ // The transaction was created with coordinatedRetry:true (see
1814
+ // getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
1815
+ // sentinel (a number) is why commitResolution is typed
1816
+ // Promise<number | void>; it is handled in the resolve callback below.
1817
+ this.assertDatabaseCommitAllowed(transaction);
1818
+ this.markNativeCommitSubmitted();
1819
+ commitResolution = transaction.commit();
1820
+ // Record how long this commit stays outstanding (submit → settle) as a distribution
1821
+ // metric. This is the same clock the overload check uses (trackOutstandingCommit
1822
+ // stamps each attempt at submit), so a rising p99/p999 is the leading indicator for the
1823
+ // "Outstanding write transactions have too long of queue" (503) rejection. A transient-
1824
+ // conflict retry rejects this promise and issues a fresh commit(), which is tracked as
1825
+ // its own attempt, so recording per attempt matches the overload semantics.
1826
+ // commitResolution's declared type (Promise<number | void> | void) doesn't narrow to
1827
+ // Promise<void> here because the widening union defeats flow analysis on the prior
1828
+ // cast assignment; re-assert it — this branch's commit() result is always a Promise.
1829
+ recordCommitLatency(commitResolution, performance.now());
1830
+ // Write-queue-depth accounting for this commit happens uniformly below, via
1831
+ // trackOutstandingCommit(commitResolution) — see that function's comment. A
1832
+ // transient-conflict retry rejects this promise and issues a fresh commit()
1833
+ // (re-entering here), which trackOutstandingCommit tracks as its own attempt.
1509
1834
  }
1510
- return this.commit({ ...options, transaction });
1511
- }
1512
- // onCommit may be async (e.g. RocksTransactionLogStore emits 'aftercommit'). Surface a
1513
- // rejection — or a synchronous throw — via logging rather than failing the commit, since
1514
- // the write is already durable.
1515
- try {
1516
- const onCommitResult = transaction.onCommit?.();
1517
- if (onCommitResult?.then)
1518
- onCommitResult.catch((error) => harperLogger.warn?.('onCommit handler failed after commit', error));
1519
- }
1520
- catch (error) {
1521
- harperLogger.warn?.('onCommit handler failed after commit', error);
1522
- }
1523
- if (this.next) {
1524
- // never forward options.transaction (a retry/replay round's HEAD-store handle) to
1525
- // the next store — it must commit its own writes through its own transaction
1526
- let nextCommit;
1527
- try {
1528
- nextCommit = this.next.commit(options.transaction ? { ...options, transaction: undefined } : options);
1529
- }
1530
- catch (error) {
1531
- // This store has landed; its bookkeeping below must run before the failure surfaces.
1532
- nextCommit = Promise.reject(error);
1533
- nextCommit.catch(() => { }); // still rejects Promise.all, even if bookkeeping throws first
1534
- }
1535
- completions.push(nextCommit);
1536
- }
1537
- if (options?.flush) {
1538
- completions.push(this.writes[0].store.flushed);
1539
- }
1540
- if (this.replicatedConfirmation) {
1541
- // if we want to wait for replication confirmation, we need to track the transaction times
1542
- // and when replication notifications come in, we count the number of confirms until we reach the desired number
1543
- const databaseName = this.writes[0].store.rootStore.databaseName;
1544
- const lastEntry = this.lastConfirmableEntry();
1545
- if (confirmReplication && lastEntry) {
1546
- completions.push(confirmReplication(databaseName, lastEntry.version, this.replicatedConfirmation));
1835
+ else {
1836
+ try {
1837
+ commitResolution = transaction.abort();
1838
+ }
1839
+ catch {
1840
+ // The transaction has uncommitted writes that were already cleared from
1841
+ // this.writes by a concurrent immediate-commit path (e.g. writes made with
1842
+ // an explicitly-reused closed transaction). Those writes are handled by the
1843
+ // concurrent commit, so there is nothing left to do here.
1844
+ }
1547
1845
  }
1548
1846
  }
1549
- // commit succeeded; clean up files for any writes whose commit-handler took an early-return,
1550
- // or whose stored record a later write to the same key replaced without an audit entry
1551
- // keeping its blobs reachable (checked against the final committed record, so a blob the
1552
- // later write retained survives). Deferred until here so a retry that *would* have
1553
- // referenced the blob can flip skipped/superseded back to false first.
1554
- for (const write of this.writes) {
1555
- if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
1556
- (0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
1557
- }
1558
- if (this.recordLocks)
1559
- this.noteCommittedLockVersions();
1560
- // now reset transactions tracking; this transaction be reused and committed again
1561
- this.retries = 0; // reset per-native-transaction retry counter so a reused DatabaseTransaction's next batch starts fresh
1562
- this.clearWrites();
1563
- this.releaseRecordLocks();
1564
- if (options.doneWriting)
1565
- this.endScopeOwnership();
1566
- this.releaseContext(!!options.doneWriting);
1567
- let txnTime = this.timestamp;
1568
- this.timestamp = 0; // reset the timestamp as well
1569
- return Promise.all(completions).then(() => {
1570
- // Only once the chained store's commit has settled, as on the synchronous path: a
1571
- // partially failed mid-scope commit must not leave the scope resumable.
1572
- this.completeMidScopeCommit(options);
1573
- return {
1574
- txnTime,
1575
- };
1576
- }, (error) => {
1577
- // As on the synchronous path: a completion that failed (a chained store's commit,
1578
- // a replication confirmation) leaves this commit partly landed, so ownership goes
1579
- // with it rather than letting a later commit rotate on top.
1580
- this.endScopeOwnership();
1581
- throw error;
1582
- });
1583
- }, (error) => {
1584
- // Coordinated transactions surface conflicts as RETRY_NOW (handled in the
1585
- // resolve branch above) and never reach here with ERR_BUSY. But not every
1586
- // write transaction is coordinated — a write that reaches save() with no
1587
- // prior getReadTxn() (immediate/publish/invalidate writes) is created
1588
- // without coordinatedRetry and still rejects with ERR_BUSY on conflict.
1589
- // Keep the backoff retry as the fallback for those paths.
1590
- //
1591
- // ERR_BUSY: optimistic-transaction write conflict. ERR_TRY_AGAIN: RocksDB kTryAgain —
1592
- // the transaction's snapshot sequence fell outside the memtable conflict-check window
1593
- // (max_write_buffer_size_to_maintain), which happens under bulk-ingest bursts such as a
1594
- // migration full-table copy. Both are transient and retryable. Before ERR_TRY_AGAIN was
1595
- // retried here, the rejection propagated out of the unawaited onCommit() handler as an
1596
- // unhandled rejection and the write was silently dropped — records lost mid-copy (#308).
1597
- if (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') {
1598
- // if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
1599
- // for future transactions
1600
- this.retries++;
1601
- harperLogger.debug?.('retrying', transaction.id, this.retries);
1602
- // ERR_BUSY and ERR_TRY_AGAIN are both retried by recommitting the SAME native
1603
- // transaction. ERR_BUSY recovers because the save loop re-writes each key, re-tracking
1604
- // it at the current sequence. ERR_TRY_AGAIN — a snapshot stranded outside the memtable
1605
- // conflict-check window after a bulk-ingest flush — used to fail forever on recommit
1606
- // because the native layer left the stranded snapshot in place; rocksdb-js now resets
1607
- // the transaction onto a fresh snapshot on the failed TryAgain commit, exactly as it
1608
- // always did for IsBusy, so the re-run's save loop re-resolves against current state and
1609
- // converges. Keeping the same transaction means its committedPosition survives the reset
1610
- // (WAL write-once, rocksdb-js#668) and its onCommit hook stays attached, so the
1611
- // already-staged change-feed entry publishes only when the retry really commits — no
1612
- // premature publish, and no fresh-transaction replay that would drop the entry.
1613
- // Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
1614
- transaction.isRetry = true;
1615
- // Before the backoff gate below: the budget can already be spent on the first
1616
- // retry when an earlier store in this chain consumed it.
1617
- const pastBudget = this.elapsedPastCommitBudget();
1618
- if (pastBudget)
1619
- this.abandonCommitAfterDeadline(transaction, pastBudget);
1620
- if (this.retries > 2) {
1621
- // Transactions applying data from a canonical source of truth (replication peer or
1622
- // external caching source) must never drop a write on a transient conflict: there is no
1623
- // re-subscribe / sequence-id-resume path, so a dropped write would leave this node
1624
- // permanently diverged (harper-pro#348). Such transactions retry without a cap; the source
1625
- // apply loop serializes commits (backpressure), so contention clears rather than
1626
- // compounding. Request-path transactions keep the MAX_RETRIES cap and surface a loud error.
1627
- const neverDropOnConflict = this.sourceApply;
1847
+ }
1848
+ if (commitResolution) {
1849
+ // Read the table off the write itself, not this.db, which is whichever table first
1850
+ // claimed this per-database transaction in txnForContext and so can name the wrong
1851
+ // table when a transaction spans more than one table in the same database.
1852
+ trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
1853
+ // Every retry round and every chained store re-enters here and must inherit the chain
1854
+ // root's clock rather than restart it, so only the first submission stamps.
1855
+ const chainRoot = this.root ?? this;
1856
+ if (chainRoot.commitStartedAt == null)
1857
+ chainRoot.commitStartedAt = performance.now();
1858
+ const completions = [];
1859
+ const commitOutcome = commitResolution.then((commitResult) => {
1860
+ if (commitResult === RETRY_NOW_VALUE) {
1861
+ this.retries++;
1862
+ harperLogger.debug?.('coordinated retry', transaction.id, this.retries);
1863
+ // Mark this specific native transaction as a retry so RocksTransactionLogStore
1864
+ // skips re-writing its already-staged txn-log entries (#2).
1865
+ transaction.isRetry = true;
1866
+ const pastBudget = this.elapsedPastCommitBudget();
1867
+ if (pastBudget)
1868
+ this.abandonCommitAfterDeadline(transaction, pastBudget);
1869
+ // Mirror the ERR_BUSY cap/warn policy: non-sourceApply transactions abort
1870
+ // at MAX_RETRIES; sourceApply transactions keep retrying with periodic warn.
1628
1871
  if (this.retries > MAX_RETRIES) {
1629
- if (!neverDropOnConflict) {
1872
+ if (!this.sourceApply) {
1630
1873
  // giving up: poison and abort the whole linked chain so no link leaks its native
1631
- // handle / read snapshot until GC.
1874
+ // handle / read snapshot — or any unpublished transaction-log position — until GC.
1632
1875
  this.abortChainAfterRetries(transaction);
1633
- throw new hdbError_ts_1.ServerError(`After ${MAX_RETRIES} retries, unable to commit transaction, transaction is in conflict with ongoing writes`);
1876
+ throw new hdbError_ts_1.ServerError(`After ${MAX_RETRIES} coordinated retries, unable to commit transaction, transaction is in conflict with ongoing writes`);
1634
1877
  }
1635
- // Uncapped retry can otherwise stall silently (debug logging is off in production);
1636
- // surface periodic visibility into a stalled source-apply commit.
1637
1878
  if (this.retries % MAX_RETRIES === 0) {
1638
- harperLogger.warn?.(`Source-applied transaction ${transaction.id} still in conflict after ${this.retries} retries; continuing to retry`);
1879
+ harperLogger.warn?.(`Source-applied transaction ${transaction.id} still in conflict after ${this.retries} coordinated retries; continuing to retry`);
1639
1880
  }
1640
1881
  }
1641
- // start delaying, back off to try to space out transactions and avoid excessive conflicts
1642
- return (0, promises_1.setTimeout)(Math.min(this.retries * this.retries, MAX_RETRY_DELAY_MS)).then(() => this.commit({ ...options, transaction }));
1882
+ return this.commit({ ...options, transaction });
1643
1883
  }
1644
- return this.commit({ ...options, transaction }); // try again
1645
- }
1646
- else {
1647
- // terminal (non-conflict) failure: release the native handle so it doesn't leak;
1648
- // usually already released by the failed commit itself, abort for the unexpected
1649
- // case (same defensive pattern as the retry-exhaustion give-up above)
1884
+ // onCommit may be async (e.g. RocksTransactionLogStore emits 'aftercommit'). Surface a
1885
+ // rejection — or a synchronous throw — via logging rather than failing the commit, since
1886
+ // the write is already durable.
1650
1887
  try {
1651
- transaction.abort();
1888
+ const onCommitResult = transaction.onCommit?.();
1889
+ if (onCommitResult?.then)
1890
+ onCommitResult.catch((error) => harperLogger.warn?.('onCommit handler failed after commit', error));
1652
1891
  }
1653
- catch (abortError) {
1654
- harperLogger.debug?.('aborting transaction after failed commit', abortError);
1892
+ catch (error) {
1893
+ harperLogger.warn?.('onCommit handler failed after commit', error);
1655
1894
  }
1656
- // A terminal failure is just as final as a success — release the context's
1657
- // back-reference here too, or transaction.ts's onComplete() (which has no
1658
- // rejection handler of its own) would leave a long-lived context pinning this
1659
- // CLOSED wrapper forever.
1660
- // A failed commit must never be followed by a resumed segment: this generation is
1661
- // finished and its durability is unknown, so ownership goes with it.
1662
- this.endScopeOwnership();
1663
- this.releaseContext(!!options.doneWriting);
1895
+ if (this.next) {
1896
+ // never forward options.transaction (a retry/replay round's HEAD-store handle) to
1897
+ // the next store — it must commit its own writes through its own transaction
1898
+ let nextCommit;
1899
+ try {
1900
+ nextCommit = this.next.commit({ ...options, transaction: undefined, continuation: true });
1901
+ }
1902
+ catch (error) {
1903
+ // This store has landed; its bookkeeping below must run before the failure surfaces.
1904
+ nextCommit = Promise.reject(error);
1905
+ nextCommit.catch(() => { }); // still rejects Promise.all, even if bookkeeping throws first
1906
+ }
1907
+ completions.push(nextCommit);
1908
+ }
1909
+ if (options?.flush) {
1910
+ const store = this.writes[0]?.store;
1911
+ if (store)
1912
+ completions.push(store.flushed);
1913
+ }
1914
+ if (this.replicatedConfirmation) {
1915
+ // if we want to wait for replication confirmation, we need to track the transaction times
1916
+ // and when replication notifications come in, we count the number of confirms until we reach the desired number
1917
+ const databaseName = this.writes[0]?.store.rootStore.databaseName;
1918
+ const lastEntry = this.lastConfirmableEntry();
1919
+ if (confirmReplication && databaseName && lastEntry) {
1920
+ completions.push(confirmReplication(databaseName, lastEntry.version, this.replicatedConfirmation));
1921
+ }
1922
+ }
1923
+ // commit succeeded; clean up files for any writes whose commit-handler took an early-return,
1924
+ // or whose stored record a later write to the same key replaced without an audit entry
1925
+ // keeping its blobs reachable (checked against the final committed record, so a blob the
1926
+ // later write retained survives). Deferred until here so a retry that *would* have
1927
+ // referenced the blob can flip skipped/superseded back to false first.
1928
+ for (const write of this.writes) {
1929
+ if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
1930
+ (0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
1931
+ }
1932
+ if (this.recordLocks)
1933
+ this.noteCommittedLockVersions();
1934
+ // now reset transactions tracking; this transaction be reused and committed again
1935
+ this.retries = 0; // reset per-native-transaction retry counter so a reused DatabaseTransaction's next batch starts fresh
1936
+ this.clearWrites();
1664
1937
  this.releaseRecordLocks();
1665
- this.timestamp = 0;
1938
+ if (options.doneWriting)
1939
+ this.endScopeOwnership();
1940
+ this.releaseContext(!!options.doneWriting);
1941
+ let txnTime = this.timestamp;
1942
+ this.timestamp = 0; // reset the timestamp as well
1943
+ return Promise.all(completions).then(() => {
1944
+ // Only once the chained store's commit has settled, as on the synchronous path: a
1945
+ // partially failed mid-scope commit must not leave the scope resumable.
1946
+ this.completeMidScopeCommit(options);
1947
+ return {
1948
+ txnTime,
1949
+ };
1950
+ }, (error) => {
1951
+ // As on the synchronous path: a completion that failed (a chained store's commit,
1952
+ // a replication confirmation) leaves this commit partly landed, so ownership goes
1953
+ // with it rather than letting a later commit rotate on top.
1954
+ this.endScopeOwnership();
1955
+ throw error;
1956
+ });
1957
+ }, (error) => {
1958
+ // Coordinated transactions surface conflicts as RETRY_NOW (handled in the
1959
+ // resolve branch above) and never reach here with ERR_BUSY. But not every
1960
+ // write transaction is coordinated — a write that reaches save() with no
1961
+ // prior getReadTxn() (immediate/publish/invalidate writes) is created
1962
+ // without coordinatedRetry and still rejects with ERR_BUSY on conflict.
1963
+ // Keep the backoff retry as the fallback for those paths.
1964
+ //
1965
+ // ERR_BUSY: optimistic-transaction write conflict. ERR_TRY_AGAIN: RocksDB kTryAgain —
1966
+ // the transaction's snapshot sequence fell outside the memtable conflict-check window
1967
+ // (max_write_buffer_size_to_maintain), which happens under bulk-ingest bursts such as a
1968
+ // migration full-table copy. Both are transient and retryable. Before ERR_TRY_AGAIN was
1969
+ // retried here, the rejection propagated out of the unawaited onCommit() handler as an
1970
+ // unhandled rejection and the write was silently dropped — records lost mid-copy (#308).
1971
+ if (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') {
1972
+ // if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
1973
+ // for future transactions
1974
+ this.retries++;
1975
+ harperLogger.debug?.('retrying', transaction.id, this.retries);
1976
+ // ERR_BUSY and ERR_TRY_AGAIN are both retried by recommitting the SAME native
1977
+ // transaction. ERR_BUSY recovers because the save loop re-writes each key, re-tracking
1978
+ // it at the current sequence. ERR_TRY_AGAIN — a snapshot stranded outside the memtable
1979
+ // conflict-check window after a bulk-ingest flush — used to fail forever on recommit
1980
+ // because the native layer left the stranded snapshot in place; rocksdb-js now resets
1981
+ // the transaction onto a fresh snapshot on the failed TryAgain commit, exactly as it
1982
+ // always did for IsBusy, so the re-run's save loop re-resolves against current state and
1983
+ // converges. Keeping the same transaction means its committedPosition survives the reset
1984
+ // (WAL write-once, rocksdb-js#668) and its onCommit hook stays attached, so the
1985
+ // already-staged change-feed entry publishes only when the retry really commits — no
1986
+ // premature publish, and no fresh-transaction replay that would drop the entry.
1987
+ // Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
1988
+ transaction.isRetry = true;
1989
+ // Before the backoff gate below: the budget can already be spent on the first
1990
+ // retry when an earlier store in this chain consumed it.
1991
+ const pastBudget = this.elapsedPastCommitBudget();
1992
+ if (pastBudget)
1993
+ this.abandonCommitAfterDeadline(transaction, pastBudget);
1994
+ if (this.retries > 2) {
1995
+ // Transactions applying data from a canonical source of truth (replication peer or
1996
+ // external caching source) must never drop a write on a transient conflict: there is no
1997
+ // re-subscribe / sequence-id-resume path, so a dropped write would leave this node
1998
+ // permanently diverged (harper-pro#348). Such transactions retry without a cap; the source
1999
+ // apply loop serializes commits (backpressure), so contention clears rather than
2000
+ // compounding. Request-path transactions keep the MAX_RETRIES cap and surface a loud error.
2001
+ const neverDropOnConflict = this.sourceApply;
2002
+ if (this.retries > MAX_RETRIES) {
2003
+ if (!neverDropOnConflict) {
2004
+ // giving up: poison and abort the whole linked chain so no link leaks its native
2005
+ // handle / read snapshot until GC.
2006
+ this.abortChainAfterRetries(transaction);
2007
+ throw new hdbError_ts_1.ServerError(`After ${MAX_RETRIES} retries, unable to commit transaction, transaction is in conflict with ongoing writes`);
2008
+ }
2009
+ // Uncapped retry can otherwise stall silently (debug logging is off in production);
2010
+ // surface periodic visibility into a stalled source-apply commit.
2011
+ if (this.retries % MAX_RETRIES === 0) {
2012
+ harperLogger.warn?.(`Source-applied transaction ${transaction.id} still in conflict after ${this.retries} retries; continuing to retry`);
2013
+ }
2014
+ }
2015
+ // start delaying, back off to try to space out transactions and avoid excessive conflicts
2016
+ return (0, promises_1.setTimeout)(Math.min(this.retries * this.retries, MAX_RETRY_DELAY_MS)).then(() => this.commit({ ...options, transaction }));
2017
+ }
2018
+ return this.commit({ ...options, transaction }); // try again
2019
+ }
2020
+ else {
2021
+ // terminal (non-conflict) failure: release the native handle so it doesn't leak;
2022
+ // usually already released by the failed commit itself, abort for the unexpected
2023
+ // case (same defensive pattern as the retry-exhaustion give-up above)
2024
+ try {
2025
+ transaction.abort();
2026
+ }
2027
+ catch (abortError) {
2028
+ harperLogger.debug?.('aborting transaction after failed commit', abortError);
2029
+ }
2030
+ // A terminal failure is just as final as a success — release the context's
2031
+ // back-reference while handling the native transaction. performCommit's outer
2032
+ // rejection handler then aborts the wrapper and its linked chain.
2033
+ // A failed commit must never be followed by a resumed segment: this generation is
2034
+ // finished and its durability is unknown, so ownership goes with it.
2035
+ this.endScopeOwnership();
2036
+ this.releaseContext(!!options.doneWriting);
2037
+ this.releaseRecordLocks();
2038
+ this.timestamp = 0;
2039
+ throw error;
2040
+ }
2041
+ });
2042
+ // `commitOutcome` settles when the LOGICAL commit ends — its success branch awaits the
2043
+ // chained stores' own commits — so releasing here covers every terminal exit (success,
2044
+ // retry exhaustion, abandonment, terminal failure) in one place rather than five.
2045
+ // Released through the RETURNED promise rather than a second subscriber on
2046
+ // `commitOutcome`: a subscriber would mark a dropped commit rejection as handled and
2047
+ // silence the unhandled-rejection that surfaces it.
2048
+ return commitOutcome.then((resolution) => {
2049
+ chainRoot.commitStartedAt = undefined;
2050
+ return resolution;
2051
+ }, (error) => {
2052
+ chainRoot.commitStartedAt = undefined;
1666
2053
  throw error;
1667
- }
1668
- });
1669
- // `commitOutcome` settles when the LOGICAL commit ends — its success branch awaits the
1670
- // chained stores' own commits — so releasing here covers every terminal exit (success,
1671
- // retry exhaustion, abandonment, terminal failure) in one place rather than five.
1672
- // Released through the RETURNED promise rather than a second subscriber on
1673
- // `commitOutcome`: a subscriber would mark a dropped commit rejection as handled and
1674
- // silence the unhandled-rejection that surfaces it.
1675
- return commitOutcome.then((resolution) => {
1676
- chainRoot.commitStartedAt = undefined;
1677
- return resolution;
1678
- }, (error) => {
1679
- chainRoot.commitStartedAt = undefined;
1680
- throw error;
1681
- });
1682
- }
1683
- for (const write of this.writes) {
1684
- if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
1685
- (0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
1686
- }
1687
- if (this.recordLocks)
1688
- this.noteCommittedLockVersions();
1689
- this.clearWrites();
1690
- this.releaseRecordLocks();
1691
- if (options.doneWriting)
1692
- this.endScopeOwnership();
1693
- this.releaseContext(!!options.doneWriting);
1694
- const txnResolution = {
1695
- txnTime: this.timestamp,
1696
- };
1697
- this.timestamp = 0; // reset like the async path (~1279) so stale lock stamps don't persist
1698
- if (this.next) {
1699
- // now run any other transactions
1700
- options.timestamp = txnResolution.txnTime;
1701
- // as above: the next store must not inherit this store's explicit native transaction
1702
- let nextResolution;
1703
- try {
1704
- nextResolution = this.next?.commit(options.transaction ? { ...options, transaction: undefined } : options);
2054
+ });
1705
2055
  }
1706
- catch (error) {
1707
- // A synchronous throw reaches neither rejection handler below, and the head has already
1708
- // committed — surrender ownership here too, or the scope stays resumable on top of a
1709
- // half-landed multi-store commit.
1710
- this.endScopeOwnership();
1711
- throw error;
2056
+ for (const write of this.writes) {
2057
+ if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
2058
+ (0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
1712
2059
  }
1713
- if (nextResolution?.then)
1714
- return nextResolution?.then((nextResolution) => {
1715
- // Only once the chained store's own commit has SETTLED: rotating first would leave the
1716
- // scope resumable after a partially failed mid-scope commit.
1717
- this.completeMidScopeCommit(options);
1718
- return {
1719
- txnTime: txnResolution.txnTime,
1720
- next: nextResolution,
1721
- };
1722
- }, (error) => {
1723
- // A chained store's commit failed, so this multi-store commit half-landed. Surrender
1724
- // ownership as the head's own failure branch does: a handler that catches this and
1725
- // commits again must not rotate on top of it, and must not have the failed link
1726
- // dropped from the chain before its abort can clean up its blobs.
2060
+ if (this.recordLocks)
2061
+ this.noteCommittedLockVersions();
2062
+ this.clearWrites();
2063
+ this.releaseRecordLocks();
2064
+ if (options.doneWriting)
2065
+ this.endScopeOwnership();
2066
+ this.releaseContext(!!options.doneWriting);
2067
+ const txnResolution = {
2068
+ txnTime: this.timestamp,
2069
+ };
2070
+ this.timestamp = 0; // reset like the async path (~1279) so stale lock stamps don't persist
2071
+ if (this.next) {
2072
+ // now run any other transactions
2073
+ options.timestamp = txnResolution.txnTime;
2074
+ // as above: the next store must not inherit this store's explicit native transaction
2075
+ let nextResolution;
2076
+ try {
2077
+ nextResolution = this.next?.commit({ ...options, transaction: undefined, continuation: true });
2078
+ }
2079
+ catch (error) {
2080
+ // A synchronous throw reaches neither rejection handler below, and the head has already
2081
+ // committed — surrender ownership here too, or the scope stays resumable on top of a
2082
+ // half-landed multi-store commit.
1727
2083
  this.endScopeOwnership();
1728
2084
  throw error;
1729
- });
1730
- txnResolution.next = nextResolution;
1731
- }
1732
- this.completeMidScopeCommit(options);
1733
- return txnResolution;
1734
- }, (error) => {
2085
+ }
2086
+ if (nextResolution?.then)
2087
+ return nextResolution?.then((nextResolution) => {
2088
+ // Only once the chained store's own commit has SETTLED: rotating first would leave the
2089
+ // scope resumable after a partially failed mid-scope commit.
2090
+ this.completeMidScopeCommit(options);
2091
+ return {
2092
+ txnTime: txnResolution.txnTime,
2093
+ next: nextResolution,
2094
+ };
2095
+ }, (error) => {
2096
+ // A chained store's commit failed, so this multi-store commit half-landed. Surrender
2097
+ // ownership as the head's own failure branch does: a handler that catches this and
2098
+ // commits again must not rotate on top of it, and must not have the failed link
2099
+ // dropped from the chain before its abort can clean up its blobs.
2100
+ this.endScopeOwnership();
2101
+ throw error;
2102
+ });
2103
+ txnResolution.next = nextResolution;
2104
+ }
2105
+ this.completeMidScopeCommit(options);
2106
+ return txnResolution;
2107
+ });
2108
+ }
2109
+ catch (error) {
1735
2110
  this.setCommitPhase(false);
1736
- this.abort();
1737
- throw error;
1738
- });
2111
+ this.abortAfterCommitError(error);
2112
+ }
2113
+ if (commitResult?.then)
2114
+ return commitResult.catch((error) => {
2115
+ this.setCommitPhase(false);
2116
+ this.abortAfterCommitError(error);
2117
+ });
2118
+ return commitResult;
1739
2119
  }
1740
2120
  /**
1741
2121
  * A successful commit that is NOT the scope's final one leaves the scope still running and still
@@ -1767,7 +2147,12 @@ class DatabaseTransaction {
1767
2147
  }
1768
2148
  /** See completeMidScopeCommit, which is the only caller and carries the reasoning. */
1769
2149
  rotateAfterMidScopeCommit(options) {
1770
- if (options.doneWriting || this.timedOut || this.transaction || !this.#scopeOwned)
2150
+ if (options.doneWriting ||
2151
+ this.timedOut ||
2152
+ this.disconnected ||
2153
+ this.postSubmitPoisoned ||
2154
+ this.transaction ||
2155
+ !this.#scopeOwned)
1771
2156
  return;
1772
2157
  this.open = exports.TRANSACTION_STATE.OPEN;
1773
2158
  this.snapshotFree = true;
@@ -1789,40 +2174,74 @@ class DatabaseTransaction {
1789
2174
  }
1790
2175
  throw new hdbError_ts_1.DatabaseClosingError(rootStore?.databaseName ?? rootStore?.path ?? 'unknown', retryable);
1791
2176
  }
1792
- abort() {
1793
- while (this.readTxnsUsed > 0)
1794
- this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
1795
- // Defensively release any native handle whose reference bookkeeping was already consumed.
1796
- if (this.transaction)
1797
- this.releaseReadTxn();
1798
- this.open = exports.TRANSACTION_STATE.CLOSED;
1799
- this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
1800
- this.drainCompletions();
2177
+ abortAfterCommitError(error) {
2178
+ try {
2179
+ // Not "always retain": abort(true) keeps the native handle only while read iterators still
2180
+ // own it. resources/transaction.ts's own post-error abort passes the same argument, so the
2181
+ // two layers cannot contradict each other one frame apart.
2182
+ this.abort(true);
2183
+ }
2184
+ catch (abortError) {
2185
+ harperLogger.debug?.('aborting transaction after a failed commit', abortError);
2186
+ }
2187
+ throw error;
2188
+ }
2189
+ abort(retainReadTransaction = false, cascade = true) {
2190
+ const next = cascade ? this.next : undefined;
2191
+ if (cascade && !retainReadTransaction)
2192
+ this.next = null;
1801
2193
  try {
2194
+ const hasOpenReadIterator = retainReadTransaction &&
2195
+ this.transaction &&
2196
+ (this.readTxnsUsed > 1 || (this.baseReadRefConsumed && this.readTxnsUsed > 0));
2197
+ if (hasOpenReadIterator) {
2198
+ this.releaseRetainedWriteIntents();
2199
+ if (!this.baseReadRefConsumed) {
2200
+ this.doneReadTxn();
2201
+ this.baseReadRefConsumed = true;
2202
+ }
2203
+ }
2204
+ else {
2205
+ while (this.readTxnsUsed > 0)
2206
+ this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
2207
+ }
2208
+ // A write-only transaction never took a read reference (getReadTxn was never called), so the
2209
+ // loop above releases nothing even though save() created a native handle. Keep an iterator's
2210
+ // handle alive, but release every other remaining handle through the shared cleanup path.
2211
+ if (this.transaction && !hasOpenReadIterator)
2212
+ this.releaseReadTxn();
2213
+ this.open = exports.TRANSACTION_STATE.CLOSED;
2214
+ this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
2215
+ this.drainCompletions();
1802
2216
  for (const write of this.writes) {
1803
2217
  if (write?.savedBlobs)
1804
2218
  (0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
1805
2219
  }
1806
2220
  }
1807
2221
  finally {
1808
- this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
1809
- this.clearWrites();
1810
- this.releaseRecordLocks();
1811
- // A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
1812
- // Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
1813
- // starting a fresh one) so the rest of the logical operation fails atomically via the
1814
- // poison check in addWrite()/commit(), rather than silently landing a later write on a
1815
- // brand-new transaction after an earlier one was rolled back (#1411). Releasing here would
1816
- // make that check see `undefined?.timedOut` and take the "start fresh" branch instead.
1817
- this.releaseContext(!this.timedOut);
1818
- const next = this.next;
1819
- this.next = null;
1820
- if (next) {
1821
- try {
1822
- next.abort();
1823
- }
1824
- catch (error) {
1825
- harperLogger.debug?.('cleaning up a chained transaction during abort', error);
2222
+ try {
2223
+ this.open = exports.TRANSACTION_STATE.CLOSED;
2224
+ this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
2225
+ this.clearWrites();
2226
+ this.releaseRecordLocks();
2227
+ // A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
2228
+ // Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
2229
+ // starting a fresh one) so the rest of the logical operation fails atomically via the
2230
+ // poison check in addWrite()/commit(), rather than silently landing a later write on a
2231
+ // brand-new transaction after an earlier one was rolled back (#1411). Releasing here would
2232
+ // make that check see `undefined?.timedOut` and take the "start fresh" branch instead.
2233
+ this.releaseContext(!this.timedOut && !this.disconnected);
2234
+ if (!this.root)
2235
+ this.releaseRequestAbortListener();
2236
+ }
2237
+ finally {
2238
+ if (next && !(next.nativeCommitSubmitted && next.commitsInFlight)) {
2239
+ try {
2240
+ next.abort(retainReadTransaction);
2241
+ }
2242
+ catch (error) {
2243
+ harperLogger.debug?.('cleaning up a chained transaction during abort', error);
2244
+ }
1826
2245
  }
1827
2246
  }
1828
2247
  }
@@ -1933,35 +2352,107 @@ class DatabaseTransaction {
1933
2352
  */
1934
2353
  hasPendingWrites() {
1935
2354
  for (let txn = this; txn; txn = txn.next) {
1936
- if (txn.writes.some((write) => write))
1937
- return true;
2355
+ for (let i = 0; i < txn.writes.length; i++) {
2356
+ if (txn.writes[i])
2357
+ return true;
2358
+ }
1938
2359
  }
1939
2360
  return false;
1940
2361
  }
1941
2362
  /**
1942
- * Abort and poison this transaction because it exceeded the open-transaction limit. The next write or
1943
- * commit throws transactionOpenTooLongError so the request fails cleanly and rolls back, rather than the
1944
- * monitor force-committing a partial write set on the application's behalf (issue #1407). The whole
1945
- * multi-store `next` chain is poisoned and aborted: writes to a second database live on `next`, so
1946
- * leaving it un-poisoned would let the head's commit cascade force-commit them (or orphan its resources
1947
- * until it self-times-out).
2363
+ * Abort and poison this transaction (and its multi-store `next` chain) for `reason`, throwing on any
2364
+ * further addWrite/commit (transactionOpenTooLongError / requestAbortedError) rather than the monitor
2365
+ * force-committing a partial write set on the application's behalf (issue #1407) or a chain link
2366
+ * silently committing on behalf of a request that was supposed to have been cut off (harper#2001).
2367
+ * Poisons every link first, then aborts each — so a throw from one link's abort() can't leave a later
2368
+ * link un-poisoned and eligible for a commit cascade to force-commit it or leak its native handle.
2369
+ *
2370
+ * A link whose native commit was submitted is past the point of no destructive return: poison it for
2371
+ * everything that comes after, but let that attempt reach its outcome rather than racing cleanup
2372
+ * against unknown durability. Accepted consequence: a submitted write can land after the client is gone.
2373
+ *
2374
+ * For a disconnect that point covers the whole chain: its unsubmitted links ride the cascade already
2375
+ * under way, so the logical commit lands whole instead of splitting across stores (#1407). The monitor
2376
+ * aborts them independently instead, because it only gets here for pre-commit work that outlived its grace.
1948
2377
  */
1949
- abortDueToTimeout() {
1950
- // Poison every link first, then abort each, so a throw from one link's abort() can't leave later links
1951
- // in the chain un-poisoned (and thus eligible to be force-committed by a later commit cascade).
1952
- for (let txn = this; txn; txn = txn.next) {
1953
- txn.timedOut = true;
1954
- txn.open = exports.TRANSACTION_STATE.CLOSED;
2378
+ abortAndPoison(reason) {
2379
+ const root = this.root ?? this;
2380
+ const submittedCommit = root.commitSubmitted && this.isChainCommitting();
2381
+ const spareWholeChain = submittedCommit && reason === 'disconnected';
2382
+ const links = new Set();
2383
+ const collect = (start) => {
2384
+ for (let txn = start; txn; txn = txn.next)
2385
+ links.add(txn);
2386
+ };
2387
+ collect(root);
2388
+ collect(this);
2389
+ collect(root.submittedLink);
2390
+ if (root.submittedLinks)
2391
+ for (const txn of root.submittedLinks)
2392
+ collect(txn);
2393
+ for (const txn of links) {
2394
+ txn[reason] = true;
2395
+ if (spareWholeChain || (submittedCommit && txn.nativeCommitSubmitted))
2396
+ txn.poisonedMidCommit = true;
2397
+ else {
2398
+ txn.poisonedMidCommit = false;
2399
+ txn.open = exports.TRANSACTION_STATE.CLOSED;
2400
+ }
1955
2401
  }
1956
- for (let txn = this; txn; txn = txn.next) {
2402
+ if (spareWholeChain)
2403
+ return;
2404
+ for (const txn of links) {
2405
+ if (submittedCommit && txn.nativeCommitSubmitted)
2406
+ continue;
1957
2407
  try {
1958
- txn.abort();
2408
+ // An iterator's native cursor is owned by this transaction. Drop the staged writes now, but
2409
+ // leave its handle for doneReadTxn() so the iterator can finish without using a freed handle.
2410
+ // Abort only this link: a later link may already have submitted work whose outcome is unknown.
2411
+ txn.abort(true, false);
1959
2412
  }
1960
2413
  catch (error) {
1961
- harperLogger.debug?.(`Error aborting timed-out transaction in chain: ${error.message}`);
2414
+ harperLogger.debug?.(`Error aborting ${reason} transaction in chain`, error);
1962
2415
  }
1963
2416
  }
1964
2417
  }
2418
+ /** The long-transaction monitor calls this when a write-bearing transaction stays open too long. */
2419
+ abortDueToTimeout() {
2420
+ this.abortAndPoison('timedOut');
2421
+ }
2422
+ /** A request this chain writes for was cancelled (admitRequestWrite, harper#2001). */
2423
+ abortDueToDisconnect() {
2424
+ this.abortAndPoison('disconnected');
2425
+ }
2426
+ /**
2427
+ * Poison fresh ordinary work after a submitted commit stalls, without aborting any unknown native
2428
+ * outcome. Start at the root so every still-attached link receives the poison; also cover a link an
2429
+ * LMDB commit detached before its child settled.
2430
+ */
2431
+ poisonAfterStalledSubmittedCommit() {
2432
+ const root = this.root ?? this;
2433
+ const links = new Set();
2434
+ const collect = (start) => {
2435
+ for (let txn = start; txn; txn = txn.next)
2436
+ links.add(txn);
2437
+ };
2438
+ collect(root);
2439
+ collect(this);
2440
+ collect(root.submittedLink);
2441
+ if (root.submittedLinks)
2442
+ for (const txn of root.submittedLinks)
2443
+ collect(txn);
2444
+ for (const txn of links) {
2445
+ txn.postSubmitPoisoned = true;
2446
+ // Not on a link an earlier poison already aborted: abort() cleared its write set, so excusing
2447
+ // its continuation from the poison check would let it resume onto nothing and report success.
2448
+ if (!txn.commitAttemptDoomedByPoison())
2449
+ txn.poisonedMidCommit = true;
2450
+ if (txn.nativeCommitSubmitted) {
2451
+ txn.timedOut = true;
2452
+ }
2453
+ }
2454
+ root.timedOut = true;
2455
+ }
1965
2456
  directCommitSync() {
1966
2457
  const transaction = this.transaction;
1967
2458
  if (transaction)
@@ -2020,6 +2511,7 @@ class ImmediateTransaction extends DatabaseTransaction {
2020
2511
  super.save(operation, args[1], true);
2021
2512
  }
2022
2513
  else {
2514
+ this.admitRequestWrite(); // a deferred save reaches here without passing through addWrite()
2023
2515
  this.isCommitting = true;
2024
2516
  // A synchronous throw from commit() (e.g. a 409 from an expired lock handle) would
2025
2517
  // otherwise leave isCommitting latched at true, causing every subsequent save() in this
@@ -2209,6 +2701,94 @@ function reportIfLongLived(txn, thresholdMs, now, reportBudget) {
2209
2701
  reportLongLivedLink(link, thresholdMs, now, reportBudget);
2210
2702
  }
2211
2703
  }
2704
+ /** Diagnostic grace after a native submission outlives the ordinary transaction timeout. */
2705
+ const MAX_DEFERRED_POISON_TICKS = 2;
2706
+ const STALLED_COMMIT_LOG_MIN_INTERVAL_MS = 1000;
2707
+ let lastStalledCommitWarningAt = -Infinity;
2708
+ let lastStalledCommitErrorAt = -Infinity;
2709
+ function submittedCommitStillInFlight(root) {
2710
+ if (root.submittedLink?.nativeCommitSubmitted && root.submittedLink.commitsInFlight)
2711
+ return true;
2712
+ if (root.submittedLinks) {
2713
+ for (const txn of root.submittedLinks) {
2714
+ if (txn.nativeCommitSubmitted && txn.commitsInFlight)
2715
+ return true;
2716
+ }
2717
+ }
2718
+ return false;
2719
+ }
2720
+ /**
2721
+ * True when this tick must leave `txn` alone because some link has submitted writes whose native
2722
+ * outcome is not known yet. Merely entering commit() is not enough: a wholly pre-submit transaction
2723
+ * remains subject to shouldSpareCommitPhase() and its ordinary bounded grace.
2724
+ */
2725
+ function deferForCommitInFlight(txn, url, expiration = txnExpiration) {
2726
+ const root = txn.root ?? txn;
2727
+ if (!txn.isChainCommitting() || !root.commitSubmitted)
2728
+ return false;
2729
+ // A sibling's submitted commit is protected by the links that own it. This one is poisoned and never
2730
+ // submitted, so deferring it only holds its retained snapshot for the diagnostic and commit-phase
2731
+ // graces on behalf of an attempt that is going to throw.
2732
+ if (txn.commitAttemptDoomedByPoison())
2733
+ return false;
2734
+ // performance.now(), not Date.now(): a clock correction must not extend a stalled commit's hold on its
2735
+ // diagnostic grace, nor report a legitimately slow one early.
2736
+ const now = performance.now();
2737
+ const deadline = (root.deferredPoisonDeadline ??= now + MAX_DEFERRED_POISON_TICKS * expiration);
2738
+ let hasUnsubmittedCommit = false;
2739
+ let unsubmittedCommitGraceExpired = false;
2740
+ if (now >= deadline) {
2741
+ // Only reclaim after the commit has genuinely stalled. Submitted writes no longer depend on these
2742
+ // separate snapshots; pre-submit handles can still carry writes needed by the attempt.
2743
+ txn.releaseStalledCommitReadResources();
2744
+ if (!txn.isProtectedCommit()) {
2745
+ if (!root.postSubmitPoisoned)
2746
+ txn.poisonAfterStalledSubmittedCommit();
2747
+ const links = new Set();
2748
+ const collect = (start) => {
2749
+ for (let link = start; link; link = link.next)
2750
+ links.add(link);
2751
+ };
2752
+ collect(root);
2753
+ collect(txn);
2754
+ collect(root.submittedLink);
2755
+ if (root.submittedLinks)
2756
+ for (const link of root.submittedLinks)
2757
+ collect(link);
2758
+ for (const link of links) {
2759
+ if (link.committing && !link.nativeCommitSubmitted) {
2760
+ hasUnsubmittedCommit = true;
2761
+ break;
2762
+ }
2763
+ }
2764
+ if (hasUnsubmittedCommit) {
2765
+ const unsubmittedDeadline = (root.unsubmittedCommitDeadline ??= now + exports.COMMIT_PHASE_GRACE * expiration);
2766
+ if (now >= unsubmittedDeadline) {
2767
+ unsubmittedCommitGraceExpired = true;
2768
+ txn.abortDueToTimeout();
2769
+ }
2770
+ }
2771
+ else
2772
+ root.unsubmittedCommitDeadline = undefined;
2773
+ }
2774
+ if (!txn.nativeCommitSubmitted &&
2775
+ txn.open === exports.TRANSACTION_STATE.CLOSED &&
2776
+ !(txn === root && submittedCommitStillInFlight(root)))
2777
+ return hasUnsubmittedCommit && !unsubmittedCommitGraceExpired;
2778
+ if (!root.stalledCommitLogged && now - lastStalledCommitErrorAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS) {
2779
+ lastStalledCommitErrorAt = now;
2780
+ root.stalledCommitLogged = true;
2781
+ harperLogger.error(`A native commit has not settled after exceeding the open-transaction limit by a further ${MAX_DEFERRED_POISON_TICKS * expiration}ms. Its outcome is unknown, so Harper will not abort or delete its writes; fresh ordinary work ` +
2782
+ `on this transaction is rejected until it settles, and the worker may require a restart, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
2783
+ }
2784
+ }
2785
+ else if (now - lastStalledCommitWarningAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS) {
2786
+ lastStalledCommitWarningAt = now;
2787
+ harperLogger.warn?.(`Transaction exceeded the open-transaction limit after submitting a native commit; preserving its unknown outcome, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
2788
+ }
2789
+ txn.timeout = Math.max(expiration, txn.timeoutBudget ?? 0);
2790
+ return true;
2791
+ }
2212
2792
  function startMonitoringTxns() {
2213
2793
  timer = setInterval(function () {
2214
2794
  monitorTick++;
@@ -2241,6 +2821,15 @@ function startMonitoringTxns() {
2241
2821
  txn.writeTimeout -= txnExpiration;
2242
2822
  if (txn.timeout <= 0) {
2243
2823
  const url = txn.getContext()?.url;
2824
+ if (deferForCommitInFlight(txn, url, txnExpiration))
2825
+ return;
2826
+ if (txn.open === exports.TRANSACTION_STATE.CLOSED &&
2827
+ !txn.commitAttemptDoomedByPoison() &&
2828
+ shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
2829
+ harperLogger.warn?.(`Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work; letting it complete, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
2830
+ txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
2831
+ return;
2832
+ }
2244
2833
  if (txn.open === exports.TRANSACTION_STATE.CLOSED) {
2245
2834
  if (!txn.transaction) {
2246
2835
  // Nothing left to supervise, and this is the registry's only unconditional exit:
@@ -2250,14 +2839,16 @@ function startMonitoringTxns() {
2250
2839
  txn.dropWriteSupervision();
2251
2840
  return;
2252
2841
  }
2253
- // The commit was already acknowledged; any staged writes are riding an in-flight
2254
- // replay commit (see the outstanding-iterators branch in commit()) and are not the
2255
- // monitor's to abort — dropping them here would re-introduce the silent
2256
- // write-loss-after-ack this branch structure exists to prevent. Only the read
2257
- // snapshot's lifetime is left to enforce: release the retained native handle that
2258
- // the overdue iterators are holding open.
2259
- harperLogger.warn?.(`Read iterators held a committed transaction's snapshot past the open-transaction limit; releasing it, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
2260
- txn.releaseReadTxn();
2842
+ // Staged writes are not the monitor's to abort here: on the committed path they ride an
2843
+ // in-flight replay commit (see the outstanding-iterators branch in commit()) and dropping
2844
+ // them would re-introduce the silent write-loss-after-ack this structure prevents; on the
2845
+ // poisoned path abort(true) already discarded them. Only the snapshot's lifetime is left.
2846
+ // Close the owning iterators first so the handle comes back through doneReadTxn() — a
2847
+ // transaction poisoned outside transaction() has no callback settlement to do it.
2848
+ const closedIterators = txn.closeOwnedReadIterators();
2849
+ harperLogger.warn?.(`Read iterators held a ${txn.timedOut || txn.disconnected ? 'poisoned' : 'committed'} transaction's snapshot past the open-transaction limit; ${txn.transaction ? 'releasing it' : `closed ${closedIterators} abandoned iterator(s)`}, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
2850
+ if (txn.transaction)
2851
+ txn.releaseReadTxn();
2261
2852
  }
2262
2853
  else if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
2263
2854
  // Parked in commit()'s pre-commit await — a `before`/`beforeIntermediate` hook, in practice a