@harperfast/harper 5.3.0 → 5.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/copyDb.ts +5 -1
- package/bin/deployRenderer.ts +3 -0
- package/components/Application.ts +66 -22
- package/components/DESIGN.md +89 -36
- package/components/componentLoader.ts +2 -0
- package/components/deploymentRecorder.ts +18 -1
- package/components/installFingerprint.ts +123 -0
- package/components/mcp/DESIGN.md +4 -4
- package/components/operations.js +29 -0
- package/config/DESIGN.md +5 -5
- package/dataLayer/DESIGN.md +37 -22
- package/dataLayer/backupRepository.ts +248 -0
- package/dataLayer/restoreMarker.ts +30 -75
- package/dataLayer/rocksdbBackup.ts +253 -110
- package/dist/bin/copyDb.js +3 -0
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/bin/deployRenderer.js +3 -0
- package/dist/bin/deployRenderer.js.map +1 -1
- package/dist/components/Application.d.ts +5 -0
- package/dist/components/Application.js +51 -19
- package/dist/components/Application.js.map +1 -1
- package/dist/components/componentLoader.js +2 -0
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/deploymentRecorder.js +18 -1
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/installFingerprint.d.ts +39 -0
- package/dist/components/installFingerprint.js +113 -0
- package/dist/components/installFingerprint.js.map +1 -0
- package/dist/components/operations.d.ts +2 -0
- package/dist/components/operations.js +31 -0
- package/dist/components/operations.js.map +1 -1
- package/dist/dataLayer/backupRepository.d.ts +33 -0
- package/dist/dataLayer/backupRepository.js +232 -0
- package/dist/dataLayer/backupRepository.js.map +1 -0
- package/dist/dataLayer/restoreMarker.d.ts +4 -1
- package/dist/dataLayer/restoreMarker.js +29 -68
- package/dist/dataLayer/restoreMarker.js.map +1 -1
- package/dist/dataLayer/rocksdbBackup.js +221 -112
- package/dist/dataLayer/rocksdbBackup.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +14 -0
- package/dist/resources/DatabaseTransaction.d.ts +140 -11
- package/dist/resources/DatabaseTransaction.js +1055 -464
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/IterableEventQueue.d.ts +2 -1
- package/dist/resources/IterableEventQueue.js +16 -8
- package/dist/resources/IterableEventQueue.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +13 -2
- package/dist/resources/LMDBTransaction.js +219 -93
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/RecordEncoder.js +2 -3
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/Resource.js +5 -6
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +3 -1
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/RocksIndexStore.d.ts +2 -1
- package/dist/resources/RocksIndexStore.js +3 -0
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/Table.js +116 -12
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/write.js +6 -2
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +71 -17
- package/dist/resources/auditStore.js +248 -64
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/branchDatabase.js +2 -0
- package/dist/resources/branchDatabase.js.map +1 -1
- package/dist/resources/databases.js +8 -2
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/indexes/fullTextNativeBinding.d.ts +6 -2
- package/dist/resources/indexes/fullTextNativeBinding.js +12 -2
- package/dist/resources/indexes/fullTextNativeBinding.js.map +1 -1
- package/dist/resources/indexes/fullTextQueryIndex.d.ts +4 -0
- package/dist/resources/indexes/fullTextQueryIndex.js +207 -10
- package/dist/resources/indexes/fullTextQueryIndex.js.map +1 -1
- package/dist/resources/models/Models.d.ts +11 -0
- package/dist/resources/models/Models.js +95 -4
- package/dist/resources/models/Models.js.map +1 -1
- package/dist/resources/models/backendRegistry.d.ts +8 -0
- package/dist/resources/models/backendRegistry.js +15 -1
- package/dist/resources/models/backendRegistry.js.map +1 -1
- package/dist/resources/models/bootstrap.d.ts +2 -0
- package/dist/resources/models/bootstrap.js +6 -0
- package/dist/resources/models/bootstrap.js.map +1 -1
- package/dist/resources/models/calibration.d.ts +88 -0
- package/dist/resources/models/calibration.js +244 -0
- package/dist/resources/models/calibration.js.map +1 -0
- package/dist/resources/models/calibrationStore.d.ts +163 -0
- package/dist/resources/models/calibrationStore.js +986 -0
- package/dist/resources/models/calibrationStore.js.map +1 -0
- package/dist/resources/models/decisionStore.d.ts +24 -0
- package/dist/resources/models/decisionStore.js +45 -0
- package/dist/resources/models/decisionStore.js.map +1 -1
- package/dist/resources/models/generativeDecision.js +44 -15
- package/dist/resources/models/generativeDecision.js.map +1 -1
- package/dist/resources/models/types.d.ts +7 -0
- package/dist/resources/recordLock.d.ts +8 -5
- package/dist/resources/recordLock.js +6 -4
- package/dist/resources/recordLock.js.map +1 -1
- package/dist/resources/replayLogs.js +8 -1
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/scheduler/engine.d.ts +4 -0
- package/dist/resources/scheduler/engine.js +30 -2
- package/dist/resources/scheduler/engine.js.map +1 -1
- package/dist/resources/search.d.ts +16 -4
- package/dist/resources/search.js +117 -13
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transaction.js +85 -5
- package/dist/resources/transaction.js.map +1 -1
- package/dist/resources/transactionBroadcast.d.ts +8 -0
- package/dist/resources/transactionBroadcast.js +65 -0
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/keys.d.ts +1 -1
- package/dist/security/keys.js +170 -120
- package/dist/security/keys.js.map +1 -1
- package/dist/server/http.js +10 -0
- package/dist/server/http.js.map +1 -1
- package/dist/server/serverHelpers/Request.js +11 -2
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.js +8 -3
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/uwsServer.js +38 -7
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/threads/manageThreads.d.ts +3 -3
- package/dist/server/threads/manageThreads.js +15 -13
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/threadServer.d.ts +3 -0
- package/dist/server/threads/threadServer.js +24 -6
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/upgrade/directives/5-3-0.js +9 -3
- package/dist/upgrade/directives/5-3-0.js.map +1 -1
- package/dist/utility/durableFile.d.ts +36 -0
- package/dist/utility/durableFile.js +106 -0
- package/dist/utility/durableFile.js.map +1 -0
- package/dist/utility/errors/hdbError.d.ts +4 -0
- package/dist/utility/errors/hdbError.js +10 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/globalSchema.d.ts +9 -0
- package/dist/utility/hdbTerms.d.ts +1 -0
- package/dist/utility/hdbTerms.js +1 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/validation/configValidator.js +29 -0
- package/dist/validation/configValidator.js.map +1 -1
- package/index.ts +7 -0
- package/json/systemSchema.json +14 -0
- package/npm-shrinkwrap.json +22 -22
- package/package.json +2 -2
- package/resources/DESIGN.md +57 -272
- package/resources/DatabaseTransaction.ts +708 -87
- package/resources/FULL_TEXT.md +6 -2
- package/resources/IterableEventQueue.ts +14 -7
- package/resources/LMDBTransaction.ts +212 -84
- package/resources/RecordEncoder.ts +2 -3
- package/resources/Resource.ts +5 -6
- package/resources/ResourceInterface.ts +3 -1
- package/resources/RocksIndexStore.ts +5 -0
- package/resources/Table.ts +118 -15
- package/resources/analytics/write.ts +6 -2
- package/resources/audit-retention.md +203 -0
- package/resources/auditStore.ts +306 -84
- package/resources/branchDatabase.ts +2 -0
- package/resources/databases.ts +6 -2
- package/resources/indexes/DESIGN.md +120 -177
- package/resources/indexes/fullTextNativeBinding.ts +22 -3
- package/resources/indexes/fullTextQueryIndex.ts +274 -11
- package/resources/models/DESIGN.md +11 -3
- package/resources/models/Models.ts +127 -4
- package/resources/models/backendRegistry.ts +17 -0
- package/resources/models/bootstrap.ts +9 -1
- package/resources/models/calibration.ts +298 -0
- package/resources/models/calibrationStore.ts +1167 -0
- package/resources/models/decisionStore.ts +57 -0
- package/resources/models/generativeDecision.ts +54 -16
- package/resources/models/types.ts +6 -0
- package/resources/record-locks.md +155 -141
- package/resources/recordLock.ts +12 -6
- package/resources/replayLogs.ts +9 -2
- package/resources/scheduler/DESIGN.md +4 -3
- package/resources/scheduler/engine.ts +35 -2
- package/resources/search.ts +154 -16
- package/resources/transaction.ts +50 -5
- package/resources/transactionBroadcast.ts +69 -0
- package/security/DESIGN.md +9 -30
- package/security/keys.ts +175 -127
- package/server/DESIGN.md +70 -36
- package/server/http.ts +10 -0
- package/server/serverHelpers/Request.ts +10 -2
- package/server/serverHelpers/contentTypes.ts +5 -3
- package/server/serverHelpers/uwsServer.ts +34 -7
- package/server/threads/manageThreads.js +16 -14
- package/server/threads/threadServer.js +24 -6
- package/studio/web/assets/{Chat-CNEKMllJ.js → Chat-4LMxI0yB.js} +1 -1
- package/studio/web/assets/{FloatingChat-CEnWhHeR.js → FloatingChat-kWHApEtF.js} +3 -3
- package/studio/web/assets/{SearchDialog-CwhJdEdY.js → SearchDialog-B7ymna97.js} +1 -1
- package/studio/web/assets/{apiToken-DC5vsaaO.js → apiToken-CE3AAt9f.js} +1 -1
- package/studio/web/assets/{applications-DQZxDy0x.js → applications-DdN7CqsB.js} +1 -1
- package/studio/web/assets/{cssMode-Dz2DxF4e.js → cssMode-BpySZUSt.js} +1 -1
- package/studio/web/assets/{editor-B3mjd3tz.js → editor-myLCVv9C.js} +1 -1
- package/studio/web/assets/{html-BsRGM7tp.js → html-DGNYO55O.js} +1 -1
- package/studio/web/assets/{htmlMode-tUKJaA3o.js → htmlMode-BUgpQr41.js} +1 -1
- package/studio/web/assets/{index-BuqFjN5K.js → index-FN4KC9va.js} +39 -39
- package/studio/web/assets/index-QLZw6xrU.css +1 -0
- package/studio/web/assets/{index.lazy-CcpfQFY_.js → index.lazy-DZFuaumG.js} +1 -1
- package/studio/web/assets/{javascript-BZ_uRGHl.js → javascript-DiBzh4pj.js} +1 -1
- package/studio/web/assets/{jsonMode-DgB-aFMj.js → jsonMode-Bm312nmu.js} +1 -1
- package/studio/web/assets/{languageServices-BleuBw17.js → languageServices-Dz2K9rZN.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-ByVYLSsQ.js → lspLanguageFeatures-CNu1L5xn.js} +1 -1
- package/studio/web/assets/{notifications-TuzN0IMi.js → notifications--nwFLGmM.js} +1 -1
- package/studio/web/assets/{notifications-D1Nf-Edn.js → notifications-BelKw5bR.js} +1 -1
- package/studio/web/assets/{profile-D7C7Lupz.js → profile-AU5FjOBx.js} +1 -1
- package/studio/web/assets/{regions-nGQCm-v8.js → regions-C0sTSEHH.js} +1 -1
- package/studio/web/assets/{register-C3x9Ze6P.js → register-DyNR2rGk.js} +2 -2
- package/studio/web/assets/{setComponentFile-DpjrRqeG.js → setComponentFile-BkWx8AN3.js} +1 -1
- package/studio/web/assets/{setup-CGiKLZ9e.js → setup-D6LZS78d.js} +2 -2
- package/studio/web/assets/{status-CGJwrNPH.js → status-BCWnn70o.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-C-Ne6P7T.js → toggleHighContrast-pyeGJ6W1.js} +1 -1
- package/studio/web/assets/{tsMode-CSreYTys.js → tsMode-CGV3iaWC.js} +1 -1
- package/studio/web/assets/{typescript-MnxBk89W.js → typescript-Ci7Khvwo.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-TNrh77-s.js → useEntityRestURL-DNiz3aZ0.js} +1 -1
- package/studio/web/assets/{workers-BfZlUNVI.js → workers-byzxjbqy.js} +1 -1
- package/studio/web/assets/{xml-BHd6qGH7.js → xml-DGnHQ3CX.js} +1 -1
- package/studio/web/assets/{yaml-C53R8OQq.js → yaml-BUoTnPWb.js} +1 -1
- package/studio/web/index.html +2 -2
- package/upgrade/directives/5-3-0.ts +10 -3
- package/utility/durableFile.ts +96 -0
- package/utility/errors/hdbError.ts +9 -0
- package/utility/hdbTerms.ts +1 -0
- package/validation/configValidator.ts +31 -0
- package/studio/web/assets/index-72cEH474.css +0 -1
|
@@ -49,6 +49,7 @@ exports.replicationConfirmation = replicationConfirmation;
|
|
|
49
49
|
exports.transactionOpenTooLongError = transactionOpenTooLongError;
|
|
50
50
|
exports.getReadTransactionGuard = getReadTransactionGuard;
|
|
51
51
|
exports.trackReadRange = trackReadRange;
|
|
52
|
+
exports.requestAbortedError = requestAbortedError;
|
|
52
53
|
exports.closeWriteInstance = closeWriteInstance;
|
|
53
54
|
exports.validateWrite = validateWrite;
|
|
54
55
|
exports.getAppliedWriteVersion = getAppliedWriteVersion;
|
|
@@ -57,6 +58,7 @@ exports.writeKeyId = writeKeyId;
|
|
|
57
58
|
exports.shouldSpareCommitPhase = shouldSpareCommitPhase;
|
|
58
59
|
exports.isReleasedTransaction = isReleasedTransaction;
|
|
59
60
|
exports.isJoinableScope = isJoinableScope;
|
|
61
|
+
exports.deferForCommitInFlight = deferForCommitInFlight;
|
|
60
62
|
exports.resetReplayedWritesWarning = resetReplayedWritesWarning;
|
|
61
63
|
exports.setMaxOutstandingTxnDuration = setMaxOutstandingTxnDuration;
|
|
62
64
|
exports.isWriteSupervised = isWriteSupervised;
|
|
@@ -79,6 +81,7 @@ const trackedTxns = new Set();
|
|
|
79
81
|
const readTransactionOwners = new WeakMap();
|
|
80
82
|
// Read options for a rotated generation's native transactions; shared because they never vary.
|
|
81
83
|
const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
|
|
84
|
+
const ONCE = Object.freeze({ once: true });
|
|
82
85
|
// Logical transactions the monitor supervises for their WRITES, kept apart from trackedTxns because the
|
|
83
86
|
// two have different units and different consumers: trackedTxns is per-link, bounds a read snapshot, and
|
|
84
87
|
// is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the
|
|
@@ -500,6 +503,10 @@ function trackReadRange(transaction, createRange) {
|
|
|
500
503
|
};
|
|
501
504
|
return range;
|
|
502
505
|
}
|
|
506
|
+
function requestAbortedError() {
|
|
507
|
+
// 499 (client closed request) rather than a retryable 503/408: no client is left to retry.
|
|
508
|
+
return new hdbError_ts_1.ServerError('Transaction was aborted because the client disconnected', 499);
|
|
509
|
+
}
|
|
503
510
|
function closeWriteInstance(operation) {
|
|
504
511
|
if (operation && !operation.instanceClosed) {
|
|
505
512
|
operation.instanceClosed = true;
|
|
@@ -552,6 +559,11 @@ function writeKeyId(key) {
|
|
|
552
559
|
return key;
|
|
553
560
|
return (0, ordered_binary_1.toBufferKey)(key).toString('latin1');
|
|
554
561
|
}
|
|
562
|
+
function clearAttemptState(txn) {
|
|
563
|
+
txn.poisonedMidCommit = false;
|
|
564
|
+
txn.postSubmitPoisoned = false;
|
|
565
|
+
txn.stalledCommitResourcesReleased = false;
|
|
566
|
+
}
|
|
555
567
|
class DatabaseTransaction {
|
|
556
568
|
#context;
|
|
557
569
|
// Whether a resources/transaction.ts scope owns this instance — i.e. a final commit or abort is
|
|
@@ -611,6 +623,20 @@ class DatabaseTransaction {
|
|
|
611
623
|
txn.commitPhaseTicks = 0;
|
|
612
624
|
}
|
|
613
625
|
}
|
|
626
|
+
// Depth of commit() attempts on this link that have not reached a native outcome.
|
|
627
|
+
commitsInFlight = 0;
|
|
628
|
+
// Per-link submission state. A commit method can still be parked in pre-commit work, where an abort
|
|
629
|
+
// is safe; once storage work has been submitted, its outcome is unknown and monitor cleanup must not
|
|
630
|
+
// clear writes or blobs that the eventual commit can reference.
|
|
631
|
+
nativeCommitSubmitted = false;
|
|
632
|
+
// Root-only aggregate, retained until every attempt in the multi-store chain settles. An earlier
|
|
633
|
+
// link can have submitted before a later link parks in pre-commit work, so a per-link check alone
|
|
634
|
+
// cannot prove that aborting the logical transaction is safe.
|
|
635
|
+
commitSubmitted = false;
|
|
636
|
+
// Keeps even a single-link submitted commit visible to the long-transaction monitor after its
|
|
637
|
+
// native handle has been detached from the read registry.
|
|
638
|
+
submittedCommitSupervised = false;
|
|
639
|
+
committingWrites = false;
|
|
614
640
|
rangeReadActive = false;
|
|
615
641
|
renewReadTimeout() {
|
|
616
642
|
// The limit is an IDLE limit. Writes always re-arm it (see addWrite), but reads only do so
|
|
@@ -658,6 +684,8 @@ class DatabaseTransaction {
|
|
|
658
684
|
coordinatedRetry: true,
|
|
659
685
|
disableSnapshot: disableSnapshot || this.snapshotFree,
|
|
660
686
|
}));
|
|
687
|
+
if (disableSnapshot || this.snapshotFree)
|
|
688
|
+
this.transaction.snapshotDisabled = true;
|
|
661
689
|
if (this.timestamp) {
|
|
662
690
|
this.transaction.setTimestamp(this.timestamp);
|
|
663
691
|
}
|
|
@@ -710,6 +738,8 @@ class DatabaseTransaction {
|
|
|
710
738
|
if (link.writeSupervised)
|
|
711
739
|
return;
|
|
712
740
|
}
|
|
741
|
+
if (root.submittedCommitSupervised)
|
|
742
|
+
return;
|
|
713
743
|
supervisedWriteRoots.delete(root);
|
|
714
744
|
}
|
|
715
745
|
detachOwnedTransaction() {
|
|
@@ -723,6 +753,49 @@ class DatabaseTransaction {
|
|
|
723
753
|
this.handleOpenedAt = 0;
|
|
724
754
|
return transaction;
|
|
725
755
|
}
|
|
756
|
+
/**
|
|
757
|
+
* Record that `iterator` owes a doneReadTxn(). Without the record, a retained handle's only
|
|
758
|
+
* reclamation path is an onDone() an abandoned request leaves nobody to call.
|
|
759
|
+
*/
|
|
760
|
+
registerReadIterator(iterator) {
|
|
761
|
+
// A transaction with no handle of its own (an ImmediateTransaction, which a context reuses for
|
|
762
|
+
// every search it makes) has nothing to reclaim and is in no monitor's registry, so an entry here
|
|
763
|
+
// would never be swept — it would just pin every partially-consumed result set on that context.
|
|
764
|
+
if (!this.transaction && !this.readTxn)
|
|
765
|
+
return;
|
|
766
|
+
(this.ownedReadIterators ??= new Set()).add(iterator);
|
|
767
|
+
}
|
|
768
|
+
unregisterReadIterator(iterator) {
|
|
769
|
+
this.ownedReadIterators?.delete(iterator);
|
|
770
|
+
}
|
|
771
|
+
/**
|
|
772
|
+
* Close every read iterator this transaction still owns, returning how many references came back.
|
|
773
|
+
* Idempotent — each onDone() clears itself — but only safe once nothing can still be consuming them.
|
|
774
|
+
*/
|
|
775
|
+
closeOwnedReadIterators() {
|
|
776
|
+
let closed = 0;
|
|
777
|
+
// The whole multi-store chain: a handler that searched a second database registered that
|
|
778
|
+
// iterator on the `next` link that owns its handle, not on the head.
|
|
779
|
+
for (let txn = this; txn; txn = txn.next) {
|
|
780
|
+
const iterators = txn.ownedReadIterators;
|
|
781
|
+
if (!iterators?.size)
|
|
782
|
+
continue;
|
|
783
|
+
// Snapshot first: onDone() deregisters, mutating the set as we walk it.
|
|
784
|
+
for (const iterator of [...iterators]) {
|
|
785
|
+
iterators.delete(iterator);
|
|
786
|
+
try {
|
|
787
|
+
if (iterator.onDone) {
|
|
788
|
+
iterator.onDone();
|
|
789
|
+
closed++;
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
catch (error) {
|
|
793
|
+
harperLogger.warn?.('Failed to close a read iterator owned by an abandoned transaction', error);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
return closed;
|
|
798
|
+
}
|
|
726
799
|
useReadTxn(disableSnapshot) {
|
|
727
800
|
const readTxn = this.getReadTxn(disableSnapshot);
|
|
728
801
|
// stackTraces is seeded by the same getReadTxn branch that registers the transaction with
|
|
@@ -759,6 +832,17 @@ class DatabaseTransaction {
|
|
|
759
832
|
this.completeDeferredContextRelease();
|
|
760
833
|
}
|
|
761
834
|
}
|
|
835
|
+
releaseRetainedWriteIntents() {
|
|
836
|
+
if (this.writesAbandoned)
|
|
837
|
+
return;
|
|
838
|
+
this.writesAbandoned = true;
|
|
839
|
+
try {
|
|
840
|
+
this.transaction?.abandonWrites?.();
|
|
841
|
+
}
|
|
842
|
+
catch (error) {
|
|
843
|
+
harperLogger.warn?.('Failed to release write intents on a retained read transaction', error);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
762
846
|
/**
|
|
763
847
|
* Force-release the retained native read handle without touching staged writes. Used by the
|
|
764
848
|
* long-transaction monitor for a CLOSED (already-acknowledged) transaction whose iterators have
|
|
@@ -1051,8 +1135,11 @@ class DatabaseTransaction {
|
|
|
1051
1135
|
completion.then(undefined, (error) => harperLogger.warn?.('A staged transaction completion failed after the transaction was aborted', error));
|
|
1052
1136
|
}
|
|
1053
1137
|
addWrite(operation) {
|
|
1054
|
-
if (this.timedOut)
|
|
1138
|
+
if (this.timedOut || this.postSubmitPoisoned)
|
|
1055
1139
|
throw transactionOpenTooLongError();
|
|
1140
|
+
if (this.disconnected)
|
|
1141
|
+
throw requestAbortedError();
|
|
1142
|
+
this.admitRequestWrite();
|
|
1056
1143
|
// A write is activity: it re-arms the idle limit on this link even though the reads it
|
|
1057
1144
|
// performs no longer do (see getReadTxn), so a transaction that keeps writing stays alive
|
|
1058
1145
|
// and only an idle one holding write intents is reaped.
|
|
@@ -1091,6 +1178,20 @@ class DatabaseTransaction {
|
|
|
1091
1178
|
return operation;
|
|
1092
1179
|
}
|
|
1093
1180
|
save(operation, transaction, reloadEntry = false, options) {
|
|
1181
|
+
if (!transaction && !options) {
|
|
1182
|
+
if (this.timedOut || this.postSubmitPoisoned)
|
|
1183
|
+
throw transactionOpenTooLongError();
|
|
1184
|
+
if (this.disconnected)
|
|
1185
|
+
throw requestAbortedError();
|
|
1186
|
+
// A deferred save is staged by addWrite() but admitted again when it is finally saved.
|
|
1187
|
+
this.admitRequestWrite();
|
|
1188
|
+
}
|
|
1189
|
+
if (!transaction && this.open !== exports.TRANSACTION_STATE.OPEN) {
|
|
1190
|
+
if (this.timedOut || this.postSubmitPoisoned)
|
|
1191
|
+
throw transactionOpenTooLongError();
|
|
1192
|
+
if (this.disconnected)
|
|
1193
|
+
throw requestAbortedError();
|
|
1194
|
+
}
|
|
1094
1195
|
const lockHandle = operation.lockHandle;
|
|
1095
1196
|
// Guard: a write staged through an expired or released lock handle must not land.
|
|
1096
1197
|
// The handle's lease timer already unlocked the native key; another holder may have taken it.
|
|
@@ -1195,10 +1296,21 @@ class DatabaseTransaction {
|
|
|
1195
1296
|
// against; a snapshot-free transaction (this.snapshotFree, after a mid-scope-commit rotation)
|
|
1196
1297
|
// has no snapshot for rocksdb-js to validate the Put against, so it only narrows the window to
|
|
1197
1298
|
// the read-to-put span rather than closing it (open follow-up, tracked in the PR description).
|
|
1198
|
-
//
|
|
1299
|
+
// Replay bypasses the cache: WeakRef targets survive the entire synchronous job, even after eviction.
|
|
1199
1300
|
const reloadsCommitBase = operation.reloadCommitBase && !operation.saved && !this.isReplay;
|
|
1200
|
-
|
|
1201
|
-
|
|
1301
|
+
// An entry read uncached through this same pinned-snapshot handle is already that base, until a retry resets
|
|
1302
|
+
// the snapshot or an earlier staged write to the key changes what a read returns; the resource withholds the
|
|
1303
|
+
// handle once its entry was replaced or evicted, and locked, snapshot-free and replayed writes always reload.
|
|
1304
|
+
const reusesBaseRead = operation.baseReadTxn === transaction &&
|
|
1305
|
+
this.retries === 0 &&
|
|
1306
|
+
!reloadEntry &&
|
|
1307
|
+
!operation.priorWrite &&
|
|
1308
|
+
!operation.lockHandle &&
|
|
1309
|
+
!this.snapshotFree &&
|
|
1310
|
+
!transaction.snapshotDisabled &&
|
|
1311
|
+
!this.isReplay;
|
|
1312
|
+
if (!reusesBaseRead && (reloadEntry || operation.entry === undefined || reloadsCommitBase)) {
|
|
1313
|
+
const uncachedRead = !!operation.reloadCommitBase || reloadEntry || this.isReplay;
|
|
1202
1314
|
operation.entry = operation.store.getEntry(operation.key, { transaction, uncachedRead });
|
|
1203
1315
|
}
|
|
1204
1316
|
if (!operation.saved) {
|
|
@@ -1253,11 +1365,44 @@ class DatabaseTransaction {
|
|
|
1253
1365
|
}
|
|
1254
1366
|
}
|
|
1255
1367
|
/**
|
|
1256
|
-
*
|
|
1368
|
+
* Whether any link of this multi-store chain has a commit attempt that has not reached a native
|
|
1369
|
+
* outcome. Asked of the whole chain, not just this link: the head commits its own writes and then
|
|
1370
|
+
* cascades into `next`, so that cascade is a continuation of one logical commit, not a fresh one.
|
|
1371
|
+
* Walked from here as well when a settling commit has already detached this link from that chain.
|
|
1372
|
+
*/
|
|
1373
|
+
isChainCommitting() {
|
|
1374
|
+
let attached = false;
|
|
1375
|
+
const root = this.root ?? this;
|
|
1376
|
+
for (let txn = root; txn; txn = txn.next) {
|
|
1377
|
+
if (txn.commitsInFlight)
|
|
1378
|
+
return true;
|
|
1379
|
+
if (txn === this)
|
|
1380
|
+
attached = true;
|
|
1381
|
+
}
|
|
1382
|
+
if (submittedCommitStillInFlight(root))
|
|
1383
|
+
return true;
|
|
1384
|
+
if (attached)
|
|
1385
|
+
return false;
|
|
1386
|
+
// Only a link a settling commit already detached needs its own walk, and its own attempt is
|
|
1387
|
+
// exactly the one that must not be torn out from under it. The ordinary case — asked of the root,
|
|
1388
|
+
// or of a link still on the chain — is answered by the single walk above.
|
|
1389
|
+
for (let txn = this; txn; txn = txn.next) {
|
|
1390
|
+
if (txn.commitsInFlight)
|
|
1391
|
+
return true;
|
|
1392
|
+
}
|
|
1393
|
+
return false;
|
|
1394
|
+
}
|
|
1395
|
+
/**
|
|
1396
|
+
* A submitted native write is this transaction's point of no destructive return. Only a FRESH
|
|
1397
|
+
* attempt — none already in flight, and no `options.transaction` from an immediate-commit re-entry —
|
|
1398
|
+
* is rejected by the poison flags. Everything that continues an already-submitted attempt (a
|
|
1399
|
+
* RETRY_NOW/ERR_BUSY/ERR_TRY_AGAIN recursion, the extra-writes recursion inside performCommit) must
|
|
1400
|
+
* reach its native outcome instead of abandoning unknown state; a poison that lands before submission
|
|
1401
|
+
* can still abort safely (see abortAndPoison).
|
|
1402
|
+
*
|
|
1403
|
+
* Resolves with information on the timestamp and success of the commit.
|
|
1257
1404
|
*/
|
|
1258
1405
|
commit(options = {}) {
|
|
1259
|
-
if (this.timedOut)
|
|
1260
|
-
throw transactionOpenTooLongError();
|
|
1261
1406
|
if (this.monitorCommit && !options.transaction) {
|
|
1262
1407
|
return this.monitorCommit.then(() => this.commit(options), (error) => {
|
|
1263
1408
|
// The failed commit ran its terminal cleanup, but as a non-final commit it kept the context.
|
|
@@ -1268,6 +1413,233 @@ class DatabaseTransaction {
|
|
|
1268
1413
|
throw error;
|
|
1269
1414
|
});
|
|
1270
1415
|
}
|
|
1416
|
+
if (!(options.transaction && this.commitsInFlight) && !(options.continuation && this.poisonedMidCommit)) {
|
|
1417
|
+
if (this.timedOut || this.postSubmitPoisoned)
|
|
1418
|
+
throw transactionOpenTooLongError();
|
|
1419
|
+
if (this.disconnected)
|
|
1420
|
+
throw requestAbortedError();
|
|
1421
|
+
}
|
|
1422
|
+
this.commitsInFlight = (this.commitsInFlight ?? 0) + 1;
|
|
1423
|
+
// Only a top-level attempt records the intent: a continuation would re-read hasPendingWrites()
|
|
1424
|
+
// after clearWrites() has already emptied it.
|
|
1425
|
+
if (this.commitsInFlight === 1 && !options.continuation && this.hasPendingWrites()) {
|
|
1426
|
+
(this.root ?? this).committingWrites = true;
|
|
1427
|
+
}
|
|
1428
|
+
let resolution;
|
|
1429
|
+
try {
|
|
1430
|
+
resolution = this.performCommit(options);
|
|
1431
|
+
}
|
|
1432
|
+
catch (error) {
|
|
1433
|
+
this.endCommitAttempt();
|
|
1434
|
+
throw error;
|
|
1435
|
+
}
|
|
1436
|
+
if (!resolution?.then) {
|
|
1437
|
+
this.endCommitAttempt();
|
|
1438
|
+
return resolution;
|
|
1439
|
+
}
|
|
1440
|
+
return resolution.then((settled) => {
|
|
1441
|
+
this.endCommitAttempt();
|
|
1442
|
+
return settled;
|
|
1443
|
+
}, (error) => {
|
|
1444
|
+
this.endCommitAttempt();
|
|
1445
|
+
throw error;
|
|
1446
|
+
});
|
|
1447
|
+
}
|
|
1448
|
+
markNativeCommitSubmitted() {
|
|
1449
|
+
this.nativeCommitSubmitted = true;
|
|
1450
|
+
const root = this.root ?? this;
|
|
1451
|
+
root.commitSubmitted = true;
|
|
1452
|
+
if (!root.submittedLink)
|
|
1453
|
+
root.submittedLink = this;
|
|
1454
|
+
else if (root.submittedLink !== this)
|
|
1455
|
+
(root.submittedLinks ??= new Set()).add(this);
|
|
1456
|
+
// LMDB has its own monitor registry and overrides this method to enroll there. Do not put an
|
|
1457
|
+
// LMDB link into the RocksDB monitor as well (stagesWriteOnSave is the engine discriminator).
|
|
1458
|
+
if (this.stagesWriteOnSave) {
|
|
1459
|
+
root.submittedCommitSupervised = true;
|
|
1460
|
+
supervisedWriteRoots.add(root);
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
/** Engine hook for a submission that has reached a native outcome. */
|
|
1464
|
+
nativeCommitAttemptEnded() { }
|
|
1465
|
+
/**
|
|
1466
|
+
* True when a commit attempt is parked on this link that can no longer land: it never submitted, and
|
|
1467
|
+
* the link is poisoned, so performCommit()'s post-await poison check throws before it commits. The
|
|
1468
|
+
* monitor's commit-phase grace exists to let real pre-commit work finish; extending it to an attempt
|
|
1469
|
+
* with no outcome left only pins this link's read snapshot for the rest of the grace.
|
|
1470
|
+
*/
|
|
1471
|
+
commitAttemptDoomedByPoison() {
|
|
1472
|
+
return !this.nativeCommitSubmitted && !this.poisonedMidCommit && Boolean(this.timedOut || this.disconnected);
|
|
1473
|
+
}
|
|
1474
|
+
/**
|
|
1475
|
+
* Admission for a new write on behalf of a request. A request already cancelled poisons the chain
|
|
1476
|
+
* (releasing whatever it staged) and is refused; otherwise the chain subscribes to the cancellation
|
|
1477
|
+
* for as long as it owns writes, so a disconnect releases its intents at once instead of when its
|
|
1478
|
+
* scope or pre-commit work happens to finish. Reads never subscribe.
|
|
1479
|
+
*/
|
|
1480
|
+
admitRequestWrite() {
|
|
1481
|
+
const root = this.root ?? this;
|
|
1482
|
+
const signal = root.requestSignal;
|
|
1483
|
+
if (!signal)
|
|
1484
|
+
return;
|
|
1485
|
+
if (signal.aborted) {
|
|
1486
|
+
this.abortDueToDisconnect();
|
|
1487
|
+
throw requestAbortedError();
|
|
1488
|
+
}
|
|
1489
|
+
if (root.requestAbortListener)
|
|
1490
|
+
return;
|
|
1491
|
+
const listener = () => {
|
|
1492
|
+
root.requestAbortListener = undefined;
|
|
1493
|
+
try {
|
|
1494
|
+
// isCommittingWrites() is not redundant: commit() marks itself CLOSED and clears its staged
|
|
1495
|
+
// writes before the native commit settles, so for that window the chain reads as idle.
|
|
1496
|
+
if ((root.open === exports.TRANSACTION_STATE.OPEN && root.hasPendingWrites()) || root.isCommittingWrites())
|
|
1497
|
+
root.abortDueToDisconnect();
|
|
1498
|
+
}
|
|
1499
|
+
catch (error) {
|
|
1500
|
+
harperLogger.debug?.('aborting transaction on client disconnect', error);
|
|
1501
|
+
}
|
|
1502
|
+
};
|
|
1503
|
+
root.requestAbortListener = listener;
|
|
1504
|
+
signal.addEventListener('abort', listener, ONCE);
|
|
1505
|
+
}
|
|
1506
|
+
releaseRequestAbortListener() {
|
|
1507
|
+
const listener = this.requestAbortListener;
|
|
1508
|
+
if (!listener)
|
|
1509
|
+
return;
|
|
1510
|
+
this.requestAbortListener = undefined;
|
|
1511
|
+
this.requestSignal.removeEventListener('abort', listener);
|
|
1512
|
+
}
|
|
1513
|
+
/** Protected work has no proven resume path, so the monitor may observe but never poison it. */
|
|
1514
|
+
isProtectedCommit() {
|
|
1515
|
+
const root = this.root ?? this;
|
|
1516
|
+
let attached = false;
|
|
1517
|
+
for (let txn = root; txn; txn = txn.next) {
|
|
1518
|
+
if (txn.sourceApply || txn.isReplay)
|
|
1519
|
+
return true;
|
|
1520
|
+
if (txn === this)
|
|
1521
|
+
attached = true;
|
|
1522
|
+
}
|
|
1523
|
+
return !attached && Boolean(this.sourceApply || this.isReplay);
|
|
1524
|
+
}
|
|
1525
|
+
/**
|
|
1526
|
+
* Reclaim only read resources that are separate from already-submitted writes. Never run this on a
|
|
1527
|
+
* pre-submit link: its native read handle can still carry the write batch the commit needs.
|
|
1528
|
+
*/
|
|
1529
|
+
releaseStalledCommitReadResources() {
|
|
1530
|
+
const release = (txn) => {
|
|
1531
|
+
if (!txn.nativeCommitSubmitted || txn.stalledCommitResourcesReleased)
|
|
1532
|
+
return;
|
|
1533
|
+
txn.stalledCommitResourcesReleased = true;
|
|
1534
|
+
if (txn.closeOwnedReadIterators() > 0)
|
|
1535
|
+
harperLogger.warn?.(`Read iterators held a submitted transaction's snapshot after its native commit stalled; releasing it, from table: ${txn.db?.name}`);
|
|
1536
|
+
if (txn.transaction || txn.readTxn)
|
|
1537
|
+
txn.releaseReadTxn();
|
|
1538
|
+
};
|
|
1539
|
+
const root = this.root ?? this;
|
|
1540
|
+
let attached = false;
|
|
1541
|
+
for (let txn = root; txn; txn = txn.next) {
|
|
1542
|
+
release(txn);
|
|
1543
|
+
if (txn === this)
|
|
1544
|
+
attached = true;
|
|
1545
|
+
}
|
|
1546
|
+
if (!attached)
|
|
1547
|
+
release(this);
|
|
1548
|
+
}
|
|
1549
|
+
endCommitAttempt() {
|
|
1550
|
+
this.commitsInFlight--;
|
|
1551
|
+
if (!this.commitsInFlight) {
|
|
1552
|
+
this.nativeCommitSubmitted = false;
|
|
1553
|
+
this.nativeCommitAttemptEnded();
|
|
1554
|
+
}
|
|
1555
|
+
// A link closes its OWN iterators the moment its own attempt settles, without waiting for the
|
|
1556
|
+
// chain: LMDBTransaction's commit detaches `next` before awaiting the child, so a scope abandoned
|
|
1557
|
+
// in that window never captured this link and the chain-wide cleanup below cannot reach it. Safe
|
|
1558
|
+
// here and not earlier because this link's attempt is the one that just reached its outcome.
|
|
1559
|
+
if ((this.root ?? this).scopeAbandoned)
|
|
1560
|
+
this.closeOwnedReadIterators();
|
|
1561
|
+
if (this.isChainCommitting())
|
|
1562
|
+
return;
|
|
1563
|
+
// The window has closed: a commit entered from here on is a fresh one, and the flags that landed
|
|
1564
|
+
// during the finished attempt must reject it like any other poisoned commit.
|
|
1565
|
+
const root = this.root ?? this;
|
|
1566
|
+
root.committingWrites = false;
|
|
1567
|
+
root.commitSubmitted = false;
|
|
1568
|
+
root.submittedCommitSupervised = false;
|
|
1569
|
+
root.deferredPoisonDeadline = undefined;
|
|
1570
|
+
root.unsubmittedCommitDeadline = undefined;
|
|
1571
|
+
root.stalledCommitLogged = false;
|
|
1572
|
+
const submittedLink = root.submittedLink;
|
|
1573
|
+
root.submittedLink = undefined;
|
|
1574
|
+
const submittedLinks = root.submittedLinks;
|
|
1575
|
+
root.submittedLinks = undefined;
|
|
1576
|
+
if (submittedLink)
|
|
1577
|
+
clearAttemptState(submittedLink);
|
|
1578
|
+
if (submittedLinks)
|
|
1579
|
+
for (const txn of submittedLinks)
|
|
1580
|
+
clearAttemptState(txn);
|
|
1581
|
+
let stillWriteSupervised = false;
|
|
1582
|
+
for (let txn = root; txn; txn = txn.next) {
|
|
1583
|
+
clearAttemptState(txn);
|
|
1584
|
+
if (txn.writeSupervised)
|
|
1585
|
+
stillWriteSupervised = true;
|
|
1586
|
+
}
|
|
1587
|
+
if (!stillWriteSupervised)
|
|
1588
|
+
supervisedWriteRoots.delete(root);
|
|
1589
|
+
if (!root.hasPendingWrites())
|
|
1590
|
+
root.releaseRequestAbortListener();
|
|
1591
|
+
if (root.scopeAbandoned) {
|
|
1592
|
+
root.scopeAbandoned = false;
|
|
1593
|
+
// The links captured when the scope was abandoned, not the current `next` chain: a successful
|
|
1594
|
+
// multi-store commit clears `next` before this runs, which would otherwise leave a former
|
|
1595
|
+
// child's iterators owning a snapshot nothing can reach.
|
|
1596
|
+
const abandoned = root.abandonedLinks;
|
|
1597
|
+
root.abandonedLinks = undefined;
|
|
1598
|
+
if (abandoned)
|
|
1599
|
+
for (const link of abandoned)
|
|
1600
|
+
link.closeOwnedReadIterators();
|
|
1601
|
+
else
|
|
1602
|
+
root.closeOwnedReadIterators();
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
/**
|
|
1606
|
+
* The scope that owned this transaction ended while a commit attempt was still in flight, so the
|
|
1607
|
+
* wrapper could not abort it. Surrender scope ownership now, or the attempt's own
|
|
1608
|
+
* rotateAfterMidScopeCommit reopens an instance with no wrapper left to commit or abort it and the
|
|
1609
|
+
* next transaction() on that context joins it and never commits. Iterator cleanup is deferred to
|
|
1610
|
+
* endCommitAttempt(), since closing them mid-attempt can pull the handle the commit is still
|
|
1611
|
+
* staging through.
|
|
1612
|
+
*/
|
|
1613
|
+
abandonScope() {
|
|
1614
|
+
const root = this.root ?? this;
|
|
1615
|
+
root.scopeAbandoned = true;
|
|
1616
|
+
const links = [];
|
|
1617
|
+
for (let txn = this; txn; txn = txn.next) {
|
|
1618
|
+
links.push(txn);
|
|
1619
|
+
txn.endScopeOwnership();
|
|
1620
|
+
// Ending ownership alone only stops the attempt's own rotation; until it reaches its native
|
|
1621
|
+
// outcome the instance is still OPEN, and a hung `before` hook makes that window arbitrarily
|
|
1622
|
+
// long. Submitted attempts are monitor-deferred; a pre-submit attempt remains safely abortable.
|
|
1623
|
+
// Not on its own sufficient:
|
|
1624
|
+
// LMDBTransaction's own commit reassigns `open`, so the release below is what actually keeps
|
|
1625
|
+
// the next write on this context off an instance with no wrapper behind it.
|
|
1626
|
+
txn.open = exports.TRANSACTION_STATE.CLOSED;
|
|
1627
|
+
}
|
|
1628
|
+
root.abandonedLinks = links;
|
|
1629
|
+
// By abort()'s own rule, so a poisoned chain stays joined and throws while an ordinary one lets
|
|
1630
|
+
// the next write start fresh.
|
|
1631
|
+
this.releaseContext(!this.timedOut && !this.disconnected);
|
|
1632
|
+
}
|
|
1633
|
+
/**
|
|
1634
|
+
* Whether this chain has a write-bearing commit attempt in flight. `hasPendingWrites()` cannot
|
|
1635
|
+
* answer that: commit() marks itself CLOSED and clears its staged writes well before the native
|
|
1636
|
+
* commit settles, so between those points a write-bearing transaction looks read-only and idle to
|
|
1637
|
+
* anything reading its public state — including transaction.ts's disconnect listener.
|
|
1638
|
+
*/
|
|
1639
|
+
isCommittingWrites() {
|
|
1640
|
+
return Boolean((this.root ?? this).committingWrites) && this.isChainCommitting();
|
|
1641
|
+
}
|
|
1642
|
+
performCommit(options = {}) {
|
|
1271
1643
|
// reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain —
|
|
1272
1644
|
// but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction
|
|
1273
1645
|
let transaction = options.transaction ?? this.transaction;
|
|
@@ -1295,447 +1667,455 @@ class DatabaseTransaction {
|
|
|
1295
1667
|
if (completions.length > 0) {
|
|
1296
1668
|
this.setCommitPhase(true);
|
|
1297
1669
|
}
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
}
|
|
1314
|
-
// The save loop above can be what opened this transaction's native handle — save() attaches
|
|
1315
|
-
// one when it had none, which is every ImmediateTransaction commit since its getReadTxn
|
|
1316
|
-
// opens none — leaving the local captured before the loop empty while that handle holds
|
|
1317
|
-
// every staged write, for the detach below to drop uncommitted (issue #2288). Only when
|
|
1318
|
-
// the local is empty: a truthy one is what the loop staged into, and the retained-handle
|
|
1319
|
-
// and replay branches below deliberately commit a handle other than this.transaction.
|
|
1320
|
-
if (!transaction)
|
|
1321
|
-
transaction = this.transaction;
|
|
1322
|
-
if (!options.transaction && this.writes.some((write) => write))
|
|
1323
|
-
this.renewChainForNativeCommit();
|
|
1324
|
-
this.open = exports.TRANSACTION_STATE.CLOSED;
|
|
1325
|
-
// RocksTransaction.commit() resolves with RETRY_NOW_VALUE (a number) under
|
|
1326
|
-
// coordinatedRetry, or void on a normal commit/abort.
|
|
1327
|
-
let commitResolution;
|
|
1328
|
-
// Consume this commit's own read reference — exactly once per read handle: retry
|
|
1329
|
-
// recursions, immediate-commit re-entries, and a second top-level commit() (wrapper
|
|
1330
|
-
// commit after an explicit in-handler commit) must not steal a reference owned by an
|
|
1331
|
-
// outstanding iterator (doneReadTxn() would then never release the native handle).
|
|
1332
|
-
if (!this.baseReadRefConsumed) {
|
|
1333
|
-
this.baseReadRefConsumed = true;
|
|
1334
|
-
this.readTxnsUsed--;
|
|
1335
|
-
}
|
|
1336
|
-
if (this.readTxnsUsed > 0) {
|
|
1337
|
-
// Outstanding iterators still stream through this.transaction — their native iterators
|
|
1338
|
-
// live inside it (GetIterator wraps its write batch + snapshot), so committing or
|
|
1339
|
-
// aborting the handle now would invalidate them mid-stream. Leave it open for the
|
|
1340
|
-
// iterators (doneReadTxn() aborts it when the last one finishes) and commit the writes
|
|
1341
|
-
// NOW by replaying them onto a fresh transaction, the same shape as the ERR_TRY_AGAIN
|
|
1342
|
-
// replay below: entries reload through the new transaction and re-resolve against
|
|
1343
|
-
// current state, and conflicts surface through the normal retry ladder. Deferring the
|
|
1344
|
-
// native commit to doneReadTxn() instead (the old LINGERING state) meant anything that
|
|
1345
|
-
// kept the last iterator from finishing cleanly — a hung stream, the long-transaction
|
|
1346
|
-
// monitor's timeout abort — dropped writes the caller had already been told committed.
|
|
1347
|
-
this.writes = this.writes.filter((write) => write); // filter out removed entries
|
|
1348
|
-
if (this.writes.length > 0) {
|
|
1349
|
-
if (!options.transaction) {
|
|
1350
|
-
if (!replayedWritesWarned) {
|
|
1351
|
-
replayedWritesWarned = true;
|
|
1352
|
-
harperLogger.warn?.(`Committing while read iterators are still open: ${this.writes.length} staged write(s) must be re-staged and committed on a second transaction, doubling their write work` +
|
|
1353
|
-
(this.startedFrom ? `, from ${this.startedFrom.resourceName}.${this.startedFrom.method}` : '') +
|
|
1354
|
-
`. Fully consume (or close) iterators before committing to avoid this. Logged once per process.`);
|
|
1355
|
-
}
|
|
1356
|
-
// Deliberately NOT marked isRetry and NOT carrying over the original's onCommit:
|
|
1357
|
-
// audit/txn-log entries batch natively on the transaction they were staged into and
|
|
1358
|
-
// are only durably written by that transaction's commit attempt (an abort discards
|
|
1359
|
-
// them — unlike the ERR_TRY_AGAIN replay below, where the original's FAILED commit
|
|
1360
|
-
// attempt already wrote its log batch). The original handle here never attempts a
|
|
1361
|
-
// commit, so the replayed stagings must re-append their entries into the replay
|
|
1362
|
-
// transaction's own batch — which also installs the replay's own commit hook.
|
|
1363
|
-
const replayTransaction = new rocksdb_js_1.Transaction((this.writes[0].store.store ?? this.db.store), { coordinatedRetry: true });
|
|
1364
|
-
if (this.timestamp)
|
|
1365
|
-
replayTransaction.setTimestamp(this.timestamp);
|
|
1366
|
-
this.retries++; // a replay round: commit handlers re-base on the reloaded entries
|
|
1367
|
-
try {
|
|
1368
|
-
for (const operation of this.writes) {
|
|
1369
|
-
this.save(operation, replayTransaction, true, options);
|
|
1370
|
-
}
|
|
1371
|
-
}
|
|
1372
|
-
catch (error) {
|
|
1373
|
-
abortNativeTransaction(replayTransaction, 'aborting a replay transaction whose re-save threw');
|
|
1374
|
-
this.abort();
|
|
1375
|
-
throw error;
|
|
1376
|
-
}
|
|
1377
|
-
transaction = replayTransaction;
|
|
1378
|
-
}
|
|
1379
|
-
// with options.transaction set this is a retry round — the save loop above already
|
|
1380
|
-
// re-staged the writes into it
|
|
1381
|
-
this.assertDatabaseCommitAllowed(transaction);
|
|
1382
|
-
commitResolution = transaction.commit();
|
|
1383
|
-
recordCommitLatency(commitResolution, performance.now());
|
|
1384
|
-
// Write-queue-depth accounting for this replay commit happens uniformly below, via
|
|
1385
|
-
// trackOutstandingCommit(commitResolution) — see that function's comment. Omitting
|
|
1386
|
-
// dedicated accounting here (as a prior version of this replay path did) used to leave
|
|
1387
|
-
// write-transaction-queue-depth, the one metric that can observe a commit that never
|
|
1388
|
-
// settles (harper#2001), reading zero for exactly this path.
|
|
1670
|
+
let commitResult;
|
|
1671
|
+
try {
|
|
1672
|
+
commitResult = (0, when_ts_1.when)(completions.length > 0 ? Promise.all(completions) : null, () => {
|
|
1673
|
+
if (completions.length > 0)
|
|
1674
|
+
this.setCommitPhase(false);
|
|
1675
|
+
// The transaction can be aborted underneath us while we are parked in the await above — by the
|
|
1676
|
+
// monitor once the commit phase outlives its grace, or through the multi-store poison chain.
|
|
1677
|
+
// abort() cleared the write set and released the handle, so resuming would commit nothing and
|
|
1678
|
+
// resolve as SUCCESS: the caller is told its write landed when it was dropped, and a write
|
|
1679
|
+
// carrying a blob is left holding an instance whose file was unlinked (issue #2062).
|
|
1680
|
+
if (!this.poisonedMidCommit) {
|
|
1681
|
+
if (this.timedOut)
|
|
1682
|
+
throw transactionOpenTooLongError();
|
|
1683
|
+
if (this.disconnected)
|
|
1684
|
+
throw requestAbortedError();
|
|
1389
1685
|
}
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
// round re-enters this branch on the same retained handle. Fenced like the other
|
|
1396
|
-
// post-submit steps here: the replay commit is already in flight, so a throw must
|
|
1397
|
-
// not skip onCommit/the chain-store commit below. Optional: rocksdb-js < 2.7
|
|
1398
|
-
// lacks the method.
|
|
1399
|
-
if (!this.writesAbandoned) {
|
|
1400
|
-
this.writesAbandoned = true;
|
|
1401
|
-
try {
|
|
1402
|
-
this.transaction?.abandonWrites?.();
|
|
1403
|
-
}
|
|
1404
|
-
catch (error) {
|
|
1405
|
-
harperLogger.warn?.('Failed to release write intents on a retained read transaction', error);
|
|
1406
|
-
}
|
|
1686
|
+
if (stagedWrites > 0 && this.writes.length === 0 && this.open === exports.TRANSACTION_STATE.CLOSED)
|
|
1687
|
+
throw new hdbError_ts_1.ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
|
|
1688
|
+
if (this.writes.length > this.validated) {
|
|
1689
|
+
// check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
|
|
1690
|
+
return this.commit({ ...options, continuation: true });
|
|
1407
1691
|
}
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
//
|
|
1411
|
-
|
|
1412
|
-
|
|
1692
|
+
// The save loop above can be what opened this transaction's native handle — save() attaches
|
|
1693
|
+
// one when it had none, which is every ImmediateTransaction commit since its getReadTxn
|
|
1694
|
+
// opens none — leaving the local captured before the loop empty while that handle holds
|
|
1695
|
+
// every staged write, for the detach below to drop uncommitted (issue #2288). Only when
|
|
1696
|
+
// the local is empty: a truthy one is what the loop staged into, and the retained-handle
|
|
1697
|
+
// and replay branches below deliberately commit a handle other than this.transaction.
|
|
1698
|
+
if (!transaction)
|
|
1699
|
+
transaction = this.transaction;
|
|
1700
|
+
if (!options.transaction && this.writes.some((write) => write))
|
|
1701
|
+
this.renewChainForNativeCommit();
|
|
1702
|
+
this.open = exports.TRANSACTION_STATE.CLOSED;
|
|
1703
|
+
// RocksTransaction.commit() resolves with RETRY_NOW_VALUE (a number) under
|
|
1704
|
+
// coordinatedRetry, or void on a normal commit/abort.
|
|
1705
|
+
let commitResolution;
|
|
1706
|
+
// Consume this commit's own read reference — exactly once per read handle: retry
|
|
1707
|
+
// recursions, immediate-commit re-entries, and a second top-level commit() (wrapper
|
|
1708
|
+
// commit after an explicit in-handler commit) must not steal a reference owned by an
|
|
1709
|
+
// outstanding iterator (doneReadTxn() would then never release the native handle).
|
|
1710
|
+
if (!this.baseReadRefConsumed) {
|
|
1711
|
+
this.baseReadRefConsumed = true;
|
|
1712
|
+
this.readTxnsUsed--;
|
|
1713
|
+
}
|
|
1714
|
+
if (this.readTxnsUsed > 0) {
|
|
1715
|
+
// Outstanding iterators still stream through this.transaction — their native iterators
|
|
1716
|
+
// live inside it (GetIterator wraps its write batch + snapshot), so committing or
|
|
1717
|
+
// aborting the handle now would invalidate them mid-stream. Leave it open for the
|
|
1718
|
+
// iterators (doneReadTxn() aborts it when the last one finishes) and commit the writes
|
|
1719
|
+
// NOW by replaying them onto a fresh transaction, the same shape as the ERR_TRY_AGAIN
|
|
1720
|
+
// replay below: entries reload through the new transaction and re-resolve against
|
|
1721
|
+
// current state, and conflicts surface through the normal retry ladder. Deferring the
|
|
1722
|
+
// native commit to doneReadTxn() instead (the old LINGERING state) meant anything that
|
|
1723
|
+
// kept the last iterator from finishing cleanly — a hung stream, the long-transaction
|
|
1724
|
+
// monitor's timeout abort — dropped writes the caller had already been told committed.
|
|
1413
1725
|
this.writes = this.writes.filter((write) => write); // filter out removed entries
|
|
1414
1726
|
if (this.writes.length > 0) {
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
// does not, whether or not the caller also unlocked.
|
|
1422
|
-
for (let i = 0; this.hasLeaseProtectedWrite && i < this.writes.length; i++) {
|
|
1423
|
-
const lapsed = this.writes[i].lockHandle;
|
|
1424
|
-
if (!lapsed?.isLeaseExpired())
|
|
1425
|
-
continue;
|
|
1426
|
-
try {
|
|
1427
|
-
transaction.abort();
|
|
1727
|
+
if (!options.transaction) {
|
|
1728
|
+
if (!replayedWritesWarned) {
|
|
1729
|
+
replayedWritesWarned = true;
|
|
1730
|
+
harperLogger.warn?.(`Committing while read iterators are still open: ${this.writes.length} staged write(s) must be re-staged and committed on a second transaction, doubling their write work` +
|
|
1731
|
+
(this.startedFrom ? `, from ${this.startedFrom.resourceName}.${this.startedFrom.method}` : '') +
|
|
1732
|
+
`. Fully consume (or close) iterators before committing to avoid this. Logged once per process.`);
|
|
1428
1733
|
}
|
|
1429
|
-
|
|
1430
|
-
//
|
|
1431
|
-
//
|
|
1432
|
-
//
|
|
1433
|
-
//
|
|
1734
|
+
// Deliberately NOT marked isRetry and NOT carrying over the original's onCommit:
|
|
1735
|
+
// audit/txn-log entries batch natively on the transaction they were staged into and
|
|
1736
|
+
// are only durably written by that transaction's commit attempt (an abort discards
|
|
1737
|
+
// them — unlike the ERR_TRY_AGAIN replay below, where the original's FAILED commit
|
|
1738
|
+
// attempt already wrote its log batch). The original handle here never attempts a
|
|
1739
|
+
// commit, so the replayed stagings must re-append their entries into the replay
|
|
1740
|
+
// transaction's own batch — which also installs the replay's own commit hook.
|
|
1741
|
+
const replayTransaction = new rocksdb_js_1.Transaction((this.writes[0].store.store ?? this.db.store), { coordinatedRetry: true });
|
|
1742
|
+
if (this.timestamp)
|
|
1743
|
+
replayTransaction.setTimestamp(this.timestamp);
|
|
1744
|
+
this.retries++; // a replay round: commit handlers re-base on the reloaded entries
|
|
1434
1745
|
try {
|
|
1435
|
-
this.
|
|
1746
|
+
for (const operation of this.writes) {
|
|
1747
|
+
this.save(operation, replayTransaction, true, options);
|
|
1748
|
+
}
|
|
1436
1749
|
}
|
|
1437
1750
|
catch (error) {
|
|
1438
|
-
|
|
1751
|
+
abortNativeTransaction(replayTransaction, 'aborting a replay transaction whose re-save threw');
|
|
1752
|
+
this.abort();
|
|
1753
|
+
throw error;
|
|
1439
1754
|
}
|
|
1440
|
-
|
|
1755
|
+
transaction = replayTransaction;
|
|
1441
1756
|
}
|
|
1442
|
-
//
|
|
1443
|
-
//
|
|
1444
|
-
// sentinel (a number) is why commitResolution is typed
|
|
1445
|
-
// Promise<number | void>; it is handled in the resolve callback below.
|
|
1757
|
+
// with options.transaction set this is a retry round — the save loop above already
|
|
1758
|
+
// re-staged the writes into it
|
|
1446
1759
|
this.assertDatabaseCommitAllowed(transaction);
|
|
1760
|
+
this.markNativeCommitSubmitted();
|
|
1447
1761
|
commitResolution = transaction.commit();
|
|
1448
|
-
// Record how long this commit stays outstanding (submit → settle) as a distribution
|
|
1449
|
-
// metric. This is the same clock the overload check uses (trackOutstandingCommit
|
|
1450
|
-
// stamps each attempt at submit), so a rising p99/p999 is the leading indicator for the
|
|
1451
|
-
// "Outstanding write transactions have too long of queue" (503) rejection. A transient-
|
|
1452
|
-
// conflict retry rejects this promise and issues a fresh commit(), which is tracked as
|
|
1453
|
-
// its own attempt, so recording per attempt matches the overload semantics.
|
|
1454
|
-
// commitResolution's declared type (Promise<number | void> | void) doesn't narrow to
|
|
1455
|
-
// Promise<void> here because the widening union defeats flow analysis on the prior
|
|
1456
|
-
// cast assignment; re-assert it — this branch's commit() result is always a Promise.
|
|
1457
1762
|
recordCommitLatency(commitResolution, performance.now());
|
|
1458
|
-
// Write-queue-depth accounting for this commit happens uniformly below, via
|
|
1459
|
-
// trackOutstandingCommit(commitResolution) — see that function's comment.
|
|
1460
|
-
//
|
|
1461
|
-
//
|
|
1462
|
-
|
|
1463
|
-
else {
|
|
1464
|
-
try {
|
|
1465
|
-
commitResolution = transaction.abort();
|
|
1466
|
-
}
|
|
1467
|
-
catch {
|
|
1468
|
-
// The transaction has uncommitted writes that were already cleared from
|
|
1469
|
-
// this.writes by a concurrent immediate-commit path (e.g. writes made with
|
|
1470
|
-
// an explicitly-reused closed transaction). Those writes are handled by the
|
|
1471
|
-
// concurrent commit, so there is nothing left to do here.
|
|
1472
|
-
}
|
|
1763
|
+
// Write-queue-depth accounting for this replay commit happens uniformly below, via
|
|
1764
|
+
// trackOutstandingCommit(commitResolution) — see that function's comment. Omitting
|
|
1765
|
+
// dedicated accounting here (as a prior version of this replay path did) used to leave
|
|
1766
|
+
// write-transaction-queue-depth, the one metric that can observe a commit that never
|
|
1767
|
+
// settles (harper#2001), reading zero for exactly this path.
|
|
1473
1768
|
}
|
|
1769
|
+
// No commit will ever run on the retained handle — the replay above owns these
|
|
1770
|
+
// writes — so this is the only place its write intents can be released. Left in
|
|
1771
|
+
// place, other writers' coordinated-retry commits park on them until the last
|
|
1772
|
+
// iterator finishes (harper#2001). Reads through the handle, including
|
|
1773
|
+
// read-your-own-writes, keep working. Once only: a coordinated-retry or backoff
|
|
1774
|
+
// round re-enters this branch on the same retained handle. Fenced like the other
|
|
1775
|
+
// post-submit steps here: the replay commit is already in flight, so a throw must
|
|
1776
|
+
// not skip onCommit/the chain-store commit below. Optional: rocksdb-js < 2.7
|
|
1777
|
+
// lacks the method.
|
|
1778
|
+
this.releaseRetainedWriteIntents();
|
|
1474
1779
|
}
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
harperLogger.warn?.(`Source-applied transaction ${transaction.id} still in conflict after ${this.retries} coordinated retries; continuing to retry`);
|
|
1780
|
+
else {
|
|
1781
|
+
// no more reads need to be performed, just commit/abort based if there are any writes
|
|
1782
|
+
this.detachOwnedTransaction(); // any further operations operate immediately
|
|
1783
|
+
if (transaction) {
|
|
1784
|
+
this.writes = this.writes.filter((write) => write); // filter out removed entries
|
|
1785
|
+
if (this.writes.length > 0) {
|
|
1786
|
+
// Commit retries can construct fresh ranges on this live handle after read ownership ends.
|
|
1787
|
+
readTransactionOwners.delete(transaction);
|
|
1788
|
+
// Re-fence before submitting: the loop above skips operations already marked saved, so
|
|
1789
|
+
// save()'s own check cannot see a holder that stalled between staging and commit. The
|
|
1790
|
+
// retry/replay path re-saves every operation and is fenced there instead. Lease expiry
|
|
1791
|
+
// only: an unlock() inside the lease leaves the staged write valid, an elapsed lease
|
|
1792
|
+
// does not, whether or not the caller also unlocked.
|
|
1793
|
+
for (let i = 0; this.hasLeaseProtectedWrite && i < this.writes.length; i++) {
|
|
1794
|
+
const lapsed = this.writes[i].lockHandle;
|
|
1795
|
+
if (!lapsed?.isLeaseExpired())
|
|
1796
|
+
continue;
|
|
1797
|
+
try {
|
|
1798
|
+
transaction.abort();
|
|
1799
|
+
}
|
|
1800
|
+
catch { }
|
|
1801
|
+
// Every other terminal exit from commit() runs the logical cleanup too. Throwing
|
|
1802
|
+
// straight out would strand the OTHER locks this transaction holds, its staged
|
|
1803
|
+
// blobs, and the context's back-reference to a CLOSED transaction — and
|
|
1804
|
+
// transaction()'s onComplete has no rejection path to run it later.
|
|
1805
|
+
try {
|
|
1806
|
+
this.abort();
|
|
1807
|
+
}
|
|
1808
|
+
catch (error) {
|
|
1809
|
+
harperLogger.debug?.('cleaning up a transaction whose record lock lapsed', error);
|
|
1810
|
+
}
|
|
1811
|
+
throw (0, recordLock_ts_1.lockNotHeldError)(lapsed);
|
|
1508
1812
|
}
|
|
1813
|
+
// The transaction was created with coordinatedRetry:true (see
|
|
1814
|
+
// getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
|
|
1815
|
+
// sentinel (a number) is why commitResolution is typed
|
|
1816
|
+
// Promise<number | void>; it is handled in the resolve callback below.
|
|
1817
|
+
this.assertDatabaseCommitAllowed(transaction);
|
|
1818
|
+
this.markNativeCommitSubmitted();
|
|
1819
|
+
commitResolution = transaction.commit();
|
|
1820
|
+
// Record how long this commit stays outstanding (submit → settle) as a distribution
|
|
1821
|
+
// metric. This is the same clock the overload check uses (trackOutstandingCommit
|
|
1822
|
+
// stamps each attempt at submit), so a rising p99/p999 is the leading indicator for the
|
|
1823
|
+
// "Outstanding write transactions have too long of queue" (503) rejection. A transient-
|
|
1824
|
+
// conflict retry rejects this promise and issues a fresh commit(), which is tracked as
|
|
1825
|
+
// its own attempt, so recording per attempt matches the overload semantics.
|
|
1826
|
+
// commitResolution's declared type (Promise<number | void> | void) doesn't narrow to
|
|
1827
|
+
// Promise<void> here because the widening union defeats flow analysis on the prior
|
|
1828
|
+
// cast assignment; re-assert it — this branch's commit() result is always a Promise.
|
|
1829
|
+
recordCommitLatency(commitResolution, performance.now());
|
|
1830
|
+
// Write-queue-depth accounting for this commit happens uniformly below, via
|
|
1831
|
+
// trackOutstandingCommit(commitResolution) — see that function's comment. A
|
|
1832
|
+
// transient-conflict retry rejects this promise and issues a fresh commit()
|
|
1833
|
+
// (re-entering here), which trackOutstandingCommit tracks as its own attempt.
|
|
1509
1834
|
}
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
catch (error) {
|
|
1521
|
-
harperLogger.warn?.('onCommit handler failed after commit', error);
|
|
1522
|
-
}
|
|
1523
|
-
if (this.next) {
|
|
1524
|
-
// never forward options.transaction (a retry/replay round's HEAD-store handle) to
|
|
1525
|
-
// the next store — it must commit its own writes through its own transaction
|
|
1526
|
-
let nextCommit;
|
|
1527
|
-
try {
|
|
1528
|
-
nextCommit = this.next.commit(options.transaction ? { ...options, transaction: undefined } : options);
|
|
1529
|
-
}
|
|
1530
|
-
catch (error) {
|
|
1531
|
-
// This store has landed; its bookkeeping below must run before the failure surfaces.
|
|
1532
|
-
nextCommit = Promise.reject(error);
|
|
1533
|
-
nextCommit.catch(() => { }); // still rejects Promise.all, even if bookkeeping throws first
|
|
1534
|
-
}
|
|
1535
|
-
completions.push(nextCommit);
|
|
1536
|
-
}
|
|
1537
|
-
if (options?.flush) {
|
|
1538
|
-
completions.push(this.writes[0].store.flushed);
|
|
1539
|
-
}
|
|
1540
|
-
if (this.replicatedConfirmation) {
|
|
1541
|
-
// if we want to wait for replication confirmation, we need to track the transaction times
|
|
1542
|
-
// and when replication notifications come in, we count the number of confirms until we reach the desired number
|
|
1543
|
-
const databaseName = this.writes[0].store.rootStore.databaseName;
|
|
1544
|
-
const lastEntry = this.lastConfirmableEntry();
|
|
1545
|
-
if (confirmReplication && lastEntry) {
|
|
1546
|
-
completions.push(confirmReplication(databaseName, lastEntry.version, this.replicatedConfirmation));
|
|
1835
|
+
else {
|
|
1836
|
+
try {
|
|
1837
|
+
commitResolution = transaction.abort();
|
|
1838
|
+
}
|
|
1839
|
+
catch {
|
|
1840
|
+
// The transaction has uncommitted writes that were already cleared from
|
|
1841
|
+
// this.writes by a concurrent immediate-commit path (e.g. writes made with
|
|
1842
|
+
// an explicitly-reused closed transaction). Those writes are handled by the
|
|
1843
|
+
// concurrent commit, so there is nothing left to do here.
|
|
1844
|
+
}
|
|
1547
1845
|
}
|
|
1548
1846
|
}
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
//
|
|
1552
|
-
//
|
|
1553
|
-
//
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
if (
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
return {
|
|
1574
|
-
txnTime,
|
|
1575
|
-
};
|
|
1576
|
-
}, (error) => {
|
|
1577
|
-
// As on the synchronous path: a completion that failed (a chained store's commit,
|
|
1578
|
-
// a replication confirmation) leaves this commit partly landed, so ownership goes
|
|
1579
|
-
// with it rather than letting a later commit rotate on top.
|
|
1580
|
-
this.endScopeOwnership();
|
|
1581
|
-
throw error;
|
|
1582
|
-
});
|
|
1583
|
-
}, (error) => {
|
|
1584
|
-
// Coordinated transactions surface conflicts as RETRY_NOW (handled in the
|
|
1585
|
-
// resolve branch above) and never reach here with ERR_BUSY. But not every
|
|
1586
|
-
// write transaction is coordinated — a write that reaches save() with no
|
|
1587
|
-
// prior getReadTxn() (immediate/publish/invalidate writes) is created
|
|
1588
|
-
// without coordinatedRetry and still rejects with ERR_BUSY on conflict.
|
|
1589
|
-
// Keep the backoff retry as the fallback for those paths.
|
|
1590
|
-
//
|
|
1591
|
-
// ERR_BUSY: optimistic-transaction write conflict. ERR_TRY_AGAIN: RocksDB kTryAgain —
|
|
1592
|
-
// the transaction's snapshot sequence fell outside the memtable conflict-check window
|
|
1593
|
-
// (max_write_buffer_size_to_maintain), which happens under bulk-ingest bursts such as a
|
|
1594
|
-
// migration full-table copy. Both are transient and retryable. Before ERR_TRY_AGAIN was
|
|
1595
|
-
// retried here, the rejection propagated out of the unawaited onCommit() handler as an
|
|
1596
|
-
// unhandled rejection and the write was silently dropped — records lost mid-copy (#308).
|
|
1597
|
-
if (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') {
|
|
1598
|
-
// if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
|
|
1599
|
-
// for future transactions
|
|
1600
|
-
this.retries++;
|
|
1601
|
-
harperLogger.debug?.('retrying', transaction.id, this.retries);
|
|
1602
|
-
// ERR_BUSY and ERR_TRY_AGAIN are both retried by recommitting the SAME native
|
|
1603
|
-
// transaction. ERR_BUSY recovers because the save loop re-writes each key, re-tracking
|
|
1604
|
-
// it at the current sequence. ERR_TRY_AGAIN — a snapshot stranded outside the memtable
|
|
1605
|
-
// conflict-check window after a bulk-ingest flush — used to fail forever on recommit
|
|
1606
|
-
// because the native layer left the stranded snapshot in place; rocksdb-js now resets
|
|
1607
|
-
// the transaction onto a fresh snapshot on the failed TryAgain commit, exactly as it
|
|
1608
|
-
// always did for IsBusy, so the re-run's save loop re-resolves against current state and
|
|
1609
|
-
// converges. Keeping the same transaction means its committedPosition survives the reset
|
|
1610
|
-
// (WAL write-once, rocksdb-js#668) and its onCommit hook stays attached, so the
|
|
1611
|
-
// already-staged change-feed entry publishes only when the retry really commits — no
|
|
1612
|
-
// premature publish, and no fresh-transaction replay that would drop the entry.
|
|
1613
|
-
// Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
|
|
1614
|
-
transaction.isRetry = true;
|
|
1615
|
-
// Before the backoff gate below: the budget can already be spent on the first
|
|
1616
|
-
// retry when an earlier store in this chain consumed it.
|
|
1617
|
-
const pastBudget = this.elapsedPastCommitBudget();
|
|
1618
|
-
if (pastBudget)
|
|
1619
|
-
this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
1620
|
-
if (this.retries > 2) {
|
|
1621
|
-
// Transactions applying data from a canonical source of truth (replication peer or
|
|
1622
|
-
// external caching source) must never drop a write on a transient conflict: there is no
|
|
1623
|
-
// re-subscribe / sequence-id-resume path, so a dropped write would leave this node
|
|
1624
|
-
// permanently diverged (harper-pro#348). Such transactions retry without a cap; the source
|
|
1625
|
-
// apply loop serializes commits (backpressure), so contention clears rather than
|
|
1626
|
-
// compounding. Request-path transactions keep the MAX_RETRIES cap and surface a loud error.
|
|
1627
|
-
const neverDropOnConflict = this.sourceApply;
|
|
1847
|
+
}
|
|
1848
|
+
if (commitResolution) {
|
|
1849
|
+
// Read the table off the write itself, not this.db, which is whichever table first
|
|
1850
|
+
// claimed this per-database transaction in txnForContext and so can name the wrong
|
|
1851
|
+
// table when a transaction spans more than one table in the same database.
|
|
1852
|
+
trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
|
|
1853
|
+
// Every retry round and every chained store re-enters here and must inherit the chain
|
|
1854
|
+
// root's clock rather than restart it, so only the first submission stamps.
|
|
1855
|
+
const chainRoot = this.root ?? this;
|
|
1856
|
+
if (chainRoot.commitStartedAt == null)
|
|
1857
|
+
chainRoot.commitStartedAt = performance.now();
|
|
1858
|
+
const completions = [];
|
|
1859
|
+
const commitOutcome = commitResolution.then((commitResult) => {
|
|
1860
|
+
if (commitResult === RETRY_NOW_VALUE) {
|
|
1861
|
+
this.retries++;
|
|
1862
|
+
harperLogger.debug?.('coordinated retry', transaction.id, this.retries);
|
|
1863
|
+
// Mark this specific native transaction as a retry so RocksTransactionLogStore
|
|
1864
|
+
// skips re-writing its already-staged txn-log entries (#2).
|
|
1865
|
+
transaction.isRetry = true;
|
|
1866
|
+
const pastBudget = this.elapsedPastCommitBudget();
|
|
1867
|
+
if (pastBudget)
|
|
1868
|
+
this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
1869
|
+
// Mirror the ERR_BUSY cap/warn policy: non-sourceApply transactions abort
|
|
1870
|
+
// at MAX_RETRIES; sourceApply transactions keep retrying with periodic warn.
|
|
1628
1871
|
if (this.retries > MAX_RETRIES) {
|
|
1629
|
-
if (!
|
|
1872
|
+
if (!this.sourceApply) {
|
|
1630
1873
|
// giving up: poison and abort the whole linked chain so no link leaks its native
|
|
1631
|
-
// handle / read snapshot until GC.
|
|
1874
|
+
// handle / read snapshot — or any unpublished transaction-log position — until GC.
|
|
1632
1875
|
this.abortChainAfterRetries(transaction);
|
|
1633
|
-
throw new hdbError_ts_1.ServerError(`After ${MAX_RETRIES} retries, unable to commit transaction, transaction is in conflict with ongoing writes`);
|
|
1876
|
+
throw new hdbError_ts_1.ServerError(`After ${MAX_RETRIES} coordinated retries, unable to commit transaction, transaction is in conflict with ongoing writes`);
|
|
1634
1877
|
}
|
|
1635
|
-
// Uncapped retry can otherwise stall silently (debug logging is off in production);
|
|
1636
|
-
// surface periodic visibility into a stalled source-apply commit.
|
|
1637
1878
|
if (this.retries % MAX_RETRIES === 0) {
|
|
1638
|
-
harperLogger.warn?.(`Source-applied transaction ${transaction.id} still in conflict after ${this.retries} retries; continuing to retry`);
|
|
1879
|
+
harperLogger.warn?.(`Source-applied transaction ${transaction.id} still in conflict after ${this.retries} coordinated retries; continuing to retry`);
|
|
1639
1880
|
}
|
|
1640
1881
|
}
|
|
1641
|
-
|
|
1642
|
-
return (0, promises_1.setTimeout)(Math.min(this.retries * this.retries, MAX_RETRY_DELAY_MS)).then(() => this.commit({ ...options, transaction }));
|
|
1882
|
+
return this.commit({ ...options, transaction });
|
|
1643
1883
|
}
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
// terminal (non-conflict) failure: release the native handle so it doesn't leak;
|
|
1648
|
-
// usually already released by the failed commit itself, abort for the unexpected
|
|
1649
|
-
// case (same defensive pattern as the retry-exhaustion give-up above)
|
|
1884
|
+
// onCommit may be async (e.g. RocksTransactionLogStore emits 'aftercommit'). Surface a
|
|
1885
|
+
// rejection — or a synchronous throw — via logging rather than failing the commit, since
|
|
1886
|
+
// the write is already durable.
|
|
1650
1887
|
try {
|
|
1651
|
-
transaction.
|
|
1888
|
+
const onCommitResult = transaction.onCommit?.();
|
|
1889
|
+
if (onCommitResult?.then)
|
|
1890
|
+
onCommitResult.catch((error) => harperLogger.warn?.('onCommit handler failed after commit', error));
|
|
1652
1891
|
}
|
|
1653
|
-
catch (
|
|
1654
|
-
harperLogger.
|
|
1892
|
+
catch (error) {
|
|
1893
|
+
harperLogger.warn?.('onCommit handler failed after commit', error);
|
|
1655
1894
|
}
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1895
|
+
if (this.next) {
|
|
1896
|
+
// never forward options.transaction (a retry/replay round's HEAD-store handle) to
|
|
1897
|
+
// the next store — it must commit its own writes through its own transaction
|
|
1898
|
+
let nextCommit;
|
|
1899
|
+
try {
|
|
1900
|
+
nextCommit = this.next.commit({ ...options, transaction: undefined, continuation: true });
|
|
1901
|
+
}
|
|
1902
|
+
catch (error) {
|
|
1903
|
+
// This store has landed; its bookkeeping below must run before the failure surfaces.
|
|
1904
|
+
nextCommit = Promise.reject(error);
|
|
1905
|
+
nextCommit.catch(() => { }); // still rejects Promise.all, even if bookkeeping throws first
|
|
1906
|
+
}
|
|
1907
|
+
completions.push(nextCommit);
|
|
1908
|
+
}
|
|
1909
|
+
if (options?.flush) {
|
|
1910
|
+
const store = this.writes[0]?.store;
|
|
1911
|
+
if (store)
|
|
1912
|
+
completions.push(store.flushed);
|
|
1913
|
+
}
|
|
1914
|
+
if (this.replicatedConfirmation) {
|
|
1915
|
+
// if we want to wait for replication confirmation, we need to track the transaction times
|
|
1916
|
+
// and when replication notifications come in, we count the number of confirms until we reach the desired number
|
|
1917
|
+
const databaseName = this.writes[0]?.store.rootStore.databaseName;
|
|
1918
|
+
const lastEntry = this.lastConfirmableEntry();
|
|
1919
|
+
if (confirmReplication && databaseName && lastEntry) {
|
|
1920
|
+
completions.push(confirmReplication(databaseName, lastEntry.version, this.replicatedConfirmation));
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
// commit succeeded; clean up files for any writes whose commit-handler took an early-return,
|
|
1924
|
+
// or whose stored record a later write to the same key replaced without an audit entry
|
|
1925
|
+
// keeping its blobs reachable (checked against the final committed record, so a blob the
|
|
1926
|
+
// later write retained survives). Deferred until here so a retry that *would* have
|
|
1927
|
+
// referenced the blob can flip skipped/superseded back to false first.
|
|
1928
|
+
for (const write of this.writes) {
|
|
1929
|
+
if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
|
|
1930
|
+
(0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
|
|
1931
|
+
}
|
|
1932
|
+
if (this.recordLocks)
|
|
1933
|
+
this.noteCommittedLockVersions();
|
|
1934
|
+
// now reset transactions tracking; this transaction be reused and committed again
|
|
1935
|
+
this.retries = 0; // reset per-native-transaction retry counter so a reused DatabaseTransaction's next batch starts fresh
|
|
1936
|
+
this.clearWrites();
|
|
1664
1937
|
this.releaseRecordLocks();
|
|
1665
|
-
|
|
1938
|
+
if (options.doneWriting)
|
|
1939
|
+
this.endScopeOwnership();
|
|
1940
|
+
this.releaseContext(!!options.doneWriting);
|
|
1941
|
+
let txnTime = this.timestamp;
|
|
1942
|
+
this.timestamp = 0; // reset the timestamp as well
|
|
1943
|
+
return Promise.all(completions).then(() => {
|
|
1944
|
+
// Only once the chained store's commit has settled, as on the synchronous path: a
|
|
1945
|
+
// partially failed mid-scope commit must not leave the scope resumable.
|
|
1946
|
+
this.completeMidScopeCommit(options);
|
|
1947
|
+
return {
|
|
1948
|
+
txnTime,
|
|
1949
|
+
};
|
|
1950
|
+
}, (error) => {
|
|
1951
|
+
// As on the synchronous path: a completion that failed (a chained store's commit,
|
|
1952
|
+
// a replication confirmation) leaves this commit partly landed, so ownership goes
|
|
1953
|
+
// with it rather than letting a later commit rotate on top.
|
|
1954
|
+
this.endScopeOwnership();
|
|
1955
|
+
throw error;
|
|
1956
|
+
});
|
|
1957
|
+
}, (error) => {
|
|
1958
|
+
// Coordinated transactions surface conflicts as RETRY_NOW (handled in the
|
|
1959
|
+
// resolve branch above) and never reach here with ERR_BUSY. But not every
|
|
1960
|
+
// write transaction is coordinated — a write that reaches save() with no
|
|
1961
|
+
// prior getReadTxn() (immediate/publish/invalidate writes) is created
|
|
1962
|
+
// without coordinatedRetry and still rejects with ERR_BUSY on conflict.
|
|
1963
|
+
// Keep the backoff retry as the fallback for those paths.
|
|
1964
|
+
//
|
|
1965
|
+
// ERR_BUSY: optimistic-transaction write conflict. ERR_TRY_AGAIN: RocksDB kTryAgain —
|
|
1966
|
+
// the transaction's snapshot sequence fell outside the memtable conflict-check window
|
|
1967
|
+
// (max_write_buffer_size_to_maintain), which happens under bulk-ingest bursts such as a
|
|
1968
|
+
// migration full-table copy. Both are transient and retryable. Before ERR_TRY_AGAIN was
|
|
1969
|
+
// retried here, the rejection propagated out of the unawaited onCommit() handler as an
|
|
1970
|
+
// unhandled rejection and the write was silently dropped — records lost mid-copy (#308).
|
|
1971
|
+
if (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') {
|
|
1972
|
+
// if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
|
|
1973
|
+
// for future transactions
|
|
1974
|
+
this.retries++;
|
|
1975
|
+
harperLogger.debug?.('retrying', transaction.id, this.retries);
|
|
1976
|
+
// ERR_BUSY and ERR_TRY_AGAIN are both retried by recommitting the SAME native
|
|
1977
|
+
// transaction. ERR_BUSY recovers because the save loop re-writes each key, re-tracking
|
|
1978
|
+
// it at the current sequence. ERR_TRY_AGAIN — a snapshot stranded outside the memtable
|
|
1979
|
+
// conflict-check window after a bulk-ingest flush — used to fail forever on recommit
|
|
1980
|
+
// because the native layer left the stranded snapshot in place; rocksdb-js now resets
|
|
1981
|
+
// the transaction onto a fresh snapshot on the failed TryAgain commit, exactly as it
|
|
1982
|
+
// always did for IsBusy, so the re-run's save loop re-resolves against current state and
|
|
1983
|
+
// converges. Keeping the same transaction means its committedPosition survives the reset
|
|
1984
|
+
// (WAL write-once, rocksdb-js#668) and its onCommit hook stays attached, so the
|
|
1985
|
+
// already-staged change-feed entry publishes only when the retry really commits — no
|
|
1986
|
+
// premature publish, and no fresh-transaction replay that would drop the entry.
|
|
1987
|
+
// Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
|
|
1988
|
+
transaction.isRetry = true;
|
|
1989
|
+
// Before the backoff gate below: the budget can already be spent on the first
|
|
1990
|
+
// retry when an earlier store in this chain consumed it.
|
|
1991
|
+
const pastBudget = this.elapsedPastCommitBudget();
|
|
1992
|
+
if (pastBudget)
|
|
1993
|
+
this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
1994
|
+
if (this.retries > 2) {
|
|
1995
|
+
// Transactions applying data from a canonical source of truth (replication peer or
|
|
1996
|
+
// external caching source) must never drop a write on a transient conflict: there is no
|
|
1997
|
+
// re-subscribe / sequence-id-resume path, so a dropped write would leave this node
|
|
1998
|
+
// permanently diverged (harper-pro#348). Such transactions retry without a cap; the source
|
|
1999
|
+
// apply loop serializes commits (backpressure), so contention clears rather than
|
|
2000
|
+
// compounding. Request-path transactions keep the MAX_RETRIES cap and surface a loud error.
|
|
2001
|
+
const neverDropOnConflict = this.sourceApply;
|
|
2002
|
+
if (this.retries > MAX_RETRIES) {
|
|
2003
|
+
if (!neverDropOnConflict) {
|
|
2004
|
+
// giving up: poison and abort the whole linked chain so no link leaks its native
|
|
2005
|
+
// handle / read snapshot until GC.
|
|
2006
|
+
this.abortChainAfterRetries(transaction);
|
|
2007
|
+
throw new hdbError_ts_1.ServerError(`After ${MAX_RETRIES} retries, unable to commit transaction, transaction is in conflict with ongoing writes`);
|
|
2008
|
+
}
|
|
2009
|
+
// Uncapped retry can otherwise stall silently (debug logging is off in production);
|
|
2010
|
+
// surface periodic visibility into a stalled source-apply commit.
|
|
2011
|
+
if (this.retries % MAX_RETRIES === 0) {
|
|
2012
|
+
harperLogger.warn?.(`Source-applied transaction ${transaction.id} still in conflict after ${this.retries} retries; continuing to retry`);
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
// start delaying, back off to try to space out transactions and avoid excessive conflicts
|
|
2016
|
+
return (0, promises_1.setTimeout)(Math.min(this.retries * this.retries, MAX_RETRY_DELAY_MS)).then(() => this.commit({ ...options, transaction }));
|
|
2017
|
+
}
|
|
2018
|
+
return this.commit({ ...options, transaction }); // try again
|
|
2019
|
+
}
|
|
2020
|
+
else {
|
|
2021
|
+
// terminal (non-conflict) failure: release the native handle so it doesn't leak;
|
|
2022
|
+
// usually already released by the failed commit itself, abort for the unexpected
|
|
2023
|
+
// case (same defensive pattern as the retry-exhaustion give-up above)
|
|
2024
|
+
try {
|
|
2025
|
+
transaction.abort();
|
|
2026
|
+
}
|
|
2027
|
+
catch (abortError) {
|
|
2028
|
+
harperLogger.debug?.('aborting transaction after failed commit', abortError);
|
|
2029
|
+
}
|
|
2030
|
+
// A terminal failure is just as final as a success — release the context's
|
|
2031
|
+
// back-reference while handling the native transaction. performCommit's outer
|
|
2032
|
+
// rejection handler then aborts the wrapper and its linked chain.
|
|
2033
|
+
// A failed commit must never be followed by a resumed segment: this generation is
|
|
2034
|
+
// finished and its durability is unknown, so ownership goes with it.
|
|
2035
|
+
this.endScopeOwnership();
|
|
2036
|
+
this.releaseContext(!!options.doneWriting);
|
|
2037
|
+
this.releaseRecordLocks();
|
|
2038
|
+
this.timestamp = 0;
|
|
2039
|
+
throw error;
|
|
2040
|
+
}
|
|
2041
|
+
});
|
|
2042
|
+
// `commitOutcome` settles when the LOGICAL commit ends — its success branch awaits the
|
|
2043
|
+
// chained stores' own commits — so releasing here covers every terminal exit (success,
|
|
2044
|
+
// retry exhaustion, abandonment, terminal failure) in one place rather than five.
|
|
2045
|
+
// Released through the RETURNED promise rather than a second subscriber on
|
|
2046
|
+
// `commitOutcome`: a subscriber would mark a dropped commit rejection as handled and
|
|
2047
|
+
// silence the unhandled-rejection that surfaces it.
|
|
2048
|
+
return commitOutcome.then((resolution) => {
|
|
2049
|
+
chainRoot.commitStartedAt = undefined;
|
|
2050
|
+
return resolution;
|
|
2051
|
+
}, (error) => {
|
|
2052
|
+
chainRoot.commitStartedAt = undefined;
|
|
1666
2053
|
throw error;
|
|
1667
|
-
}
|
|
1668
|
-
});
|
|
1669
|
-
// `commitOutcome` settles when the LOGICAL commit ends — its success branch awaits the
|
|
1670
|
-
// chained stores' own commits — so releasing here covers every terminal exit (success,
|
|
1671
|
-
// retry exhaustion, abandonment, terminal failure) in one place rather than five.
|
|
1672
|
-
// Released through the RETURNED promise rather than a second subscriber on
|
|
1673
|
-
// `commitOutcome`: a subscriber would mark a dropped commit rejection as handled and
|
|
1674
|
-
// silence the unhandled-rejection that surfaces it.
|
|
1675
|
-
return commitOutcome.then((resolution) => {
|
|
1676
|
-
chainRoot.commitStartedAt = undefined;
|
|
1677
|
-
return resolution;
|
|
1678
|
-
}, (error) => {
|
|
1679
|
-
chainRoot.commitStartedAt = undefined;
|
|
1680
|
-
throw error;
|
|
1681
|
-
});
|
|
1682
|
-
}
|
|
1683
|
-
for (const write of this.writes) {
|
|
1684
|
-
if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
|
|
1685
|
-
(0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
|
|
1686
|
-
}
|
|
1687
|
-
if (this.recordLocks)
|
|
1688
|
-
this.noteCommittedLockVersions();
|
|
1689
|
-
this.clearWrites();
|
|
1690
|
-
this.releaseRecordLocks();
|
|
1691
|
-
if (options.doneWriting)
|
|
1692
|
-
this.endScopeOwnership();
|
|
1693
|
-
this.releaseContext(!!options.doneWriting);
|
|
1694
|
-
const txnResolution = {
|
|
1695
|
-
txnTime: this.timestamp,
|
|
1696
|
-
};
|
|
1697
|
-
this.timestamp = 0; // reset like the async path (~1279) so stale lock stamps don't persist
|
|
1698
|
-
if (this.next) {
|
|
1699
|
-
// now run any other transactions
|
|
1700
|
-
options.timestamp = txnResolution.txnTime;
|
|
1701
|
-
// as above: the next store must not inherit this store's explicit native transaction
|
|
1702
|
-
let nextResolution;
|
|
1703
|
-
try {
|
|
1704
|
-
nextResolution = this.next?.commit(options.transaction ? { ...options, transaction: undefined } : options);
|
|
2054
|
+
});
|
|
1705
2055
|
}
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
// half-landed multi-store commit.
|
|
1710
|
-
this.endScopeOwnership();
|
|
1711
|
-
throw error;
|
|
2056
|
+
for (const write of this.writes) {
|
|
2057
|
+
if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
|
|
2058
|
+
(0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
|
|
1712
2059
|
}
|
|
1713
|
-
if (
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
2060
|
+
if (this.recordLocks)
|
|
2061
|
+
this.noteCommittedLockVersions();
|
|
2062
|
+
this.clearWrites();
|
|
2063
|
+
this.releaseRecordLocks();
|
|
2064
|
+
if (options.doneWriting)
|
|
2065
|
+
this.endScopeOwnership();
|
|
2066
|
+
this.releaseContext(!!options.doneWriting);
|
|
2067
|
+
const txnResolution = {
|
|
2068
|
+
txnTime: this.timestamp,
|
|
2069
|
+
};
|
|
2070
|
+
this.timestamp = 0; // reset like the async path (~1279) so stale lock stamps don't persist
|
|
2071
|
+
if (this.next) {
|
|
2072
|
+
// now run any other transactions
|
|
2073
|
+
options.timestamp = txnResolution.txnTime;
|
|
2074
|
+
// as above: the next store must not inherit this store's explicit native transaction
|
|
2075
|
+
let nextResolution;
|
|
2076
|
+
try {
|
|
2077
|
+
nextResolution = this.next?.commit({ ...options, transaction: undefined, continuation: true });
|
|
2078
|
+
}
|
|
2079
|
+
catch (error) {
|
|
2080
|
+
// A synchronous throw reaches neither rejection handler below, and the head has already
|
|
2081
|
+
// committed — surrender ownership here too, or the scope stays resumable on top of a
|
|
2082
|
+
// half-landed multi-store commit.
|
|
1727
2083
|
this.endScopeOwnership();
|
|
1728
2084
|
throw error;
|
|
1729
|
-
}
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
2085
|
+
}
|
|
2086
|
+
if (nextResolution?.then)
|
|
2087
|
+
return nextResolution?.then((nextResolution) => {
|
|
2088
|
+
// Only once the chained store's own commit has SETTLED: rotating first would leave the
|
|
2089
|
+
// scope resumable after a partially failed mid-scope commit.
|
|
2090
|
+
this.completeMidScopeCommit(options);
|
|
2091
|
+
return {
|
|
2092
|
+
txnTime: txnResolution.txnTime,
|
|
2093
|
+
next: nextResolution,
|
|
2094
|
+
};
|
|
2095
|
+
}, (error) => {
|
|
2096
|
+
// A chained store's commit failed, so this multi-store commit half-landed. Surrender
|
|
2097
|
+
// ownership as the head's own failure branch does: a handler that catches this and
|
|
2098
|
+
// commits again must not rotate on top of it, and must not have the failed link
|
|
2099
|
+
// dropped from the chain before its abort can clean up its blobs.
|
|
2100
|
+
this.endScopeOwnership();
|
|
2101
|
+
throw error;
|
|
2102
|
+
});
|
|
2103
|
+
txnResolution.next = nextResolution;
|
|
2104
|
+
}
|
|
2105
|
+
this.completeMidScopeCommit(options);
|
|
2106
|
+
return txnResolution;
|
|
2107
|
+
});
|
|
2108
|
+
}
|
|
2109
|
+
catch (error) {
|
|
1735
2110
|
this.setCommitPhase(false);
|
|
1736
|
-
this.
|
|
1737
|
-
|
|
1738
|
-
|
|
2111
|
+
this.abortAfterCommitError(error);
|
|
2112
|
+
}
|
|
2113
|
+
if (commitResult?.then)
|
|
2114
|
+
return commitResult.catch((error) => {
|
|
2115
|
+
this.setCommitPhase(false);
|
|
2116
|
+
this.abortAfterCommitError(error);
|
|
2117
|
+
});
|
|
2118
|
+
return commitResult;
|
|
1739
2119
|
}
|
|
1740
2120
|
/**
|
|
1741
2121
|
* A successful commit that is NOT the scope's final one leaves the scope still running and still
|
|
@@ -1767,7 +2147,12 @@ class DatabaseTransaction {
|
|
|
1767
2147
|
}
|
|
1768
2148
|
/** See completeMidScopeCommit, which is the only caller and carries the reasoning. */
|
|
1769
2149
|
rotateAfterMidScopeCommit(options) {
|
|
1770
|
-
if (options.doneWriting ||
|
|
2150
|
+
if (options.doneWriting ||
|
|
2151
|
+
this.timedOut ||
|
|
2152
|
+
this.disconnected ||
|
|
2153
|
+
this.postSubmitPoisoned ||
|
|
2154
|
+
this.transaction ||
|
|
2155
|
+
!this.#scopeOwned)
|
|
1771
2156
|
return;
|
|
1772
2157
|
this.open = exports.TRANSACTION_STATE.OPEN;
|
|
1773
2158
|
this.snapshotFree = true;
|
|
@@ -1789,40 +2174,74 @@ class DatabaseTransaction {
|
|
|
1789
2174
|
}
|
|
1790
2175
|
throw new hdbError_ts_1.DatabaseClosingError(rootStore?.databaseName ?? rootStore?.path ?? 'unknown', retryable);
|
|
1791
2176
|
}
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
this.
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
2177
|
+
abortAfterCommitError(error) {
|
|
2178
|
+
try {
|
|
2179
|
+
// Not "always retain": abort(true) keeps the native handle only while read iterators still
|
|
2180
|
+
// own it. resources/transaction.ts's own post-error abort passes the same argument, so the
|
|
2181
|
+
// two layers cannot contradict each other one frame apart.
|
|
2182
|
+
this.abort(true);
|
|
2183
|
+
}
|
|
2184
|
+
catch (abortError) {
|
|
2185
|
+
harperLogger.debug?.('aborting transaction after a failed commit', abortError);
|
|
2186
|
+
}
|
|
2187
|
+
throw error;
|
|
2188
|
+
}
|
|
2189
|
+
abort(retainReadTransaction = false, cascade = true) {
|
|
2190
|
+
const next = cascade ? this.next : undefined;
|
|
2191
|
+
if (cascade && !retainReadTransaction)
|
|
2192
|
+
this.next = null;
|
|
1801
2193
|
try {
|
|
2194
|
+
const hasOpenReadIterator = retainReadTransaction &&
|
|
2195
|
+
this.transaction &&
|
|
2196
|
+
(this.readTxnsUsed > 1 || (this.baseReadRefConsumed && this.readTxnsUsed > 0));
|
|
2197
|
+
if (hasOpenReadIterator) {
|
|
2198
|
+
this.releaseRetainedWriteIntents();
|
|
2199
|
+
if (!this.baseReadRefConsumed) {
|
|
2200
|
+
this.doneReadTxn();
|
|
2201
|
+
this.baseReadRefConsumed = true;
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
else {
|
|
2205
|
+
while (this.readTxnsUsed > 0)
|
|
2206
|
+
this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
|
|
2207
|
+
}
|
|
2208
|
+
// A write-only transaction never took a read reference (getReadTxn was never called), so the
|
|
2209
|
+
// loop above releases nothing even though save() created a native handle. Keep an iterator's
|
|
2210
|
+
// handle alive, but release every other remaining handle through the shared cleanup path.
|
|
2211
|
+
if (this.transaction && !hasOpenReadIterator)
|
|
2212
|
+
this.releaseReadTxn();
|
|
2213
|
+
this.open = exports.TRANSACTION_STATE.CLOSED;
|
|
2214
|
+
this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
|
|
2215
|
+
this.drainCompletions();
|
|
1802
2216
|
for (const write of this.writes) {
|
|
1803
2217
|
if (write?.savedBlobs)
|
|
1804
2218
|
(0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
|
|
1805
2219
|
}
|
|
1806
2220
|
}
|
|
1807
2221
|
finally {
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
2222
|
+
try {
|
|
2223
|
+
this.open = exports.TRANSACTION_STATE.CLOSED;
|
|
2224
|
+
this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
|
|
2225
|
+
this.clearWrites();
|
|
2226
|
+
this.releaseRecordLocks();
|
|
2227
|
+
// A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
|
|
2228
|
+
// Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
|
|
2229
|
+
// starting a fresh one) so the rest of the logical operation fails atomically via the
|
|
2230
|
+
// poison check in addWrite()/commit(), rather than silently landing a later write on a
|
|
2231
|
+
// brand-new transaction after an earlier one was rolled back (#1411). Releasing here would
|
|
2232
|
+
// make that check see `undefined?.timedOut` and take the "start fresh" branch instead.
|
|
2233
|
+
this.releaseContext(!this.timedOut && !this.disconnected);
|
|
2234
|
+
if (!this.root)
|
|
2235
|
+
this.releaseRequestAbortListener();
|
|
2236
|
+
}
|
|
2237
|
+
finally {
|
|
2238
|
+
if (next && !(next.nativeCommitSubmitted && next.commitsInFlight)) {
|
|
2239
|
+
try {
|
|
2240
|
+
next.abort(retainReadTransaction);
|
|
2241
|
+
}
|
|
2242
|
+
catch (error) {
|
|
2243
|
+
harperLogger.debug?.('cleaning up a chained transaction during abort', error);
|
|
2244
|
+
}
|
|
1826
2245
|
}
|
|
1827
2246
|
}
|
|
1828
2247
|
}
|
|
@@ -1933,35 +2352,107 @@ class DatabaseTransaction {
|
|
|
1933
2352
|
*/
|
|
1934
2353
|
hasPendingWrites() {
|
|
1935
2354
|
for (let txn = this; txn; txn = txn.next) {
|
|
1936
|
-
|
|
1937
|
-
|
|
2355
|
+
for (let i = 0; i < txn.writes.length; i++) {
|
|
2356
|
+
if (txn.writes[i])
|
|
2357
|
+
return true;
|
|
2358
|
+
}
|
|
1938
2359
|
}
|
|
1939
2360
|
return false;
|
|
1940
2361
|
}
|
|
1941
2362
|
/**
|
|
1942
|
-
* Abort and poison this transaction
|
|
1943
|
-
* commit
|
|
1944
|
-
*
|
|
1945
|
-
*
|
|
1946
|
-
*
|
|
1947
|
-
*
|
|
2363
|
+
* Abort and poison this transaction (and its multi-store `next` chain) for `reason`, throwing on any
|
|
2364
|
+
* further addWrite/commit (transactionOpenTooLongError / requestAbortedError) rather than the monitor
|
|
2365
|
+
* force-committing a partial write set on the application's behalf (issue #1407) or a chain link
|
|
2366
|
+
* silently committing on behalf of a request that was supposed to have been cut off (harper#2001).
|
|
2367
|
+
* Poisons every link first, then aborts each — so a throw from one link's abort() can't leave a later
|
|
2368
|
+
* link un-poisoned and eligible for a commit cascade to force-commit it or leak its native handle.
|
|
2369
|
+
*
|
|
2370
|
+
* A link whose native commit was submitted is past the point of no destructive return: poison it for
|
|
2371
|
+
* everything that comes after, but let that attempt reach its outcome rather than racing cleanup
|
|
2372
|
+
* against unknown durability. Accepted consequence: a submitted write can land after the client is gone.
|
|
2373
|
+
*
|
|
2374
|
+
* For a disconnect that point covers the whole chain: its unsubmitted links ride the cascade already
|
|
2375
|
+
* under way, so the logical commit lands whole instead of splitting across stores (#1407). The monitor
|
|
2376
|
+
* aborts them independently instead, because it only gets here for pre-commit work that outlived its grace.
|
|
1948
2377
|
*/
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
2378
|
+
abortAndPoison(reason) {
|
|
2379
|
+
const root = this.root ?? this;
|
|
2380
|
+
const submittedCommit = root.commitSubmitted && this.isChainCommitting();
|
|
2381
|
+
const spareWholeChain = submittedCommit && reason === 'disconnected';
|
|
2382
|
+
const links = new Set();
|
|
2383
|
+
const collect = (start) => {
|
|
2384
|
+
for (let txn = start; txn; txn = txn.next)
|
|
2385
|
+
links.add(txn);
|
|
2386
|
+
};
|
|
2387
|
+
collect(root);
|
|
2388
|
+
collect(this);
|
|
2389
|
+
collect(root.submittedLink);
|
|
2390
|
+
if (root.submittedLinks)
|
|
2391
|
+
for (const txn of root.submittedLinks)
|
|
2392
|
+
collect(txn);
|
|
2393
|
+
for (const txn of links) {
|
|
2394
|
+
txn[reason] = true;
|
|
2395
|
+
if (spareWholeChain || (submittedCommit && txn.nativeCommitSubmitted))
|
|
2396
|
+
txn.poisonedMidCommit = true;
|
|
2397
|
+
else {
|
|
2398
|
+
txn.poisonedMidCommit = false;
|
|
2399
|
+
txn.open = exports.TRANSACTION_STATE.CLOSED;
|
|
2400
|
+
}
|
|
1955
2401
|
}
|
|
1956
|
-
|
|
2402
|
+
if (spareWholeChain)
|
|
2403
|
+
return;
|
|
2404
|
+
for (const txn of links) {
|
|
2405
|
+
if (submittedCommit && txn.nativeCommitSubmitted)
|
|
2406
|
+
continue;
|
|
1957
2407
|
try {
|
|
1958
|
-
|
|
2408
|
+
// An iterator's native cursor is owned by this transaction. Drop the staged writes now, but
|
|
2409
|
+
// leave its handle for doneReadTxn() so the iterator can finish without using a freed handle.
|
|
2410
|
+
// Abort only this link: a later link may already have submitted work whose outcome is unknown.
|
|
2411
|
+
txn.abort(true, false);
|
|
1959
2412
|
}
|
|
1960
2413
|
catch (error) {
|
|
1961
|
-
harperLogger.debug?.(`Error aborting
|
|
2414
|
+
harperLogger.debug?.(`Error aborting ${reason} transaction in chain`, error);
|
|
1962
2415
|
}
|
|
1963
2416
|
}
|
|
1964
2417
|
}
|
|
2418
|
+
/** The long-transaction monitor calls this when a write-bearing transaction stays open too long. */
|
|
2419
|
+
abortDueToTimeout() {
|
|
2420
|
+
this.abortAndPoison('timedOut');
|
|
2421
|
+
}
|
|
2422
|
+
/** A request this chain writes for was cancelled (admitRequestWrite, harper#2001). */
|
|
2423
|
+
abortDueToDisconnect() {
|
|
2424
|
+
this.abortAndPoison('disconnected');
|
|
2425
|
+
}
|
|
2426
|
+
/**
|
|
2427
|
+
* Poison fresh ordinary work after a submitted commit stalls, without aborting any unknown native
|
|
2428
|
+
* outcome. Start at the root so every still-attached link receives the poison; also cover a link an
|
|
2429
|
+
* LMDB commit detached before its child settled.
|
|
2430
|
+
*/
|
|
2431
|
+
poisonAfterStalledSubmittedCommit() {
|
|
2432
|
+
const root = this.root ?? this;
|
|
2433
|
+
const links = new Set();
|
|
2434
|
+
const collect = (start) => {
|
|
2435
|
+
for (let txn = start; txn; txn = txn.next)
|
|
2436
|
+
links.add(txn);
|
|
2437
|
+
};
|
|
2438
|
+
collect(root);
|
|
2439
|
+
collect(this);
|
|
2440
|
+
collect(root.submittedLink);
|
|
2441
|
+
if (root.submittedLinks)
|
|
2442
|
+
for (const txn of root.submittedLinks)
|
|
2443
|
+
collect(txn);
|
|
2444
|
+
for (const txn of links) {
|
|
2445
|
+
txn.postSubmitPoisoned = true;
|
|
2446
|
+
// Not on a link an earlier poison already aborted: abort() cleared its write set, so excusing
|
|
2447
|
+
// its continuation from the poison check would let it resume onto nothing and report success.
|
|
2448
|
+
if (!txn.commitAttemptDoomedByPoison())
|
|
2449
|
+
txn.poisonedMidCommit = true;
|
|
2450
|
+
if (txn.nativeCommitSubmitted) {
|
|
2451
|
+
txn.timedOut = true;
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
root.timedOut = true;
|
|
2455
|
+
}
|
|
1965
2456
|
directCommitSync() {
|
|
1966
2457
|
const transaction = this.transaction;
|
|
1967
2458
|
if (transaction)
|
|
@@ -2020,6 +2511,7 @@ class ImmediateTransaction extends DatabaseTransaction {
|
|
|
2020
2511
|
super.save(operation, args[1], true);
|
|
2021
2512
|
}
|
|
2022
2513
|
else {
|
|
2514
|
+
this.admitRequestWrite(); // a deferred save reaches here without passing through addWrite()
|
|
2023
2515
|
this.isCommitting = true;
|
|
2024
2516
|
// A synchronous throw from commit() (e.g. a 409 from an expired lock handle) would
|
|
2025
2517
|
// otherwise leave isCommitting latched at true, causing every subsequent save() in this
|
|
@@ -2209,6 +2701,94 @@ function reportIfLongLived(txn, thresholdMs, now, reportBudget) {
|
|
|
2209
2701
|
reportLongLivedLink(link, thresholdMs, now, reportBudget);
|
|
2210
2702
|
}
|
|
2211
2703
|
}
|
|
2704
|
+
/** Diagnostic grace after a native submission outlives the ordinary transaction timeout. */
|
|
2705
|
+
const MAX_DEFERRED_POISON_TICKS = 2;
|
|
2706
|
+
const STALLED_COMMIT_LOG_MIN_INTERVAL_MS = 1000;
|
|
2707
|
+
let lastStalledCommitWarningAt = -Infinity;
|
|
2708
|
+
let lastStalledCommitErrorAt = -Infinity;
|
|
2709
|
+
function submittedCommitStillInFlight(root) {
|
|
2710
|
+
if (root.submittedLink?.nativeCommitSubmitted && root.submittedLink.commitsInFlight)
|
|
2711
|
+
return true;
|
|
2712
|
+
if (root.submittedLinks) {
|
|
2713
|
+
for (const txn of root.submittedLinks) {
|
|
2714
|
+
if (txn.nativeCommitSubmitted && txn.commitsInFlight)
|
|
2715
|
+
return true;
|
|
2716
|
+
}
|
|
2717
|
+
}
|
|
2718
|
+
return false;
|
|
2719
|
+
}
|
|
2720
|
+
/**
|
|
2721
|
+
* True when this tick must leave `txn` alone because some link has submitted writes whose native
|
|
2722
|
+
* outcome is not known yet. Merely entering commit() is not enough: a wholly pre-submit transaction
|
|
2723
|
+
* remains subject to shouldSpareCommitPhase() and its ordinary bounded grace.
|
|
2724
|
+
*/
|
|
2725
|
+
function deferForCommitInFlight(txn, url, expiration = txnExpiration) {
|
|
2726
|
+
const root = txn.root ?? txn;
|
|
2727
|
+
if (!txn.isChainCommitting() || !root.commitSubmitted)
|
|
2728
|
+
return false;
|
|
2729
|
+
// A sibling's submitted commit is protected by the links that own it. This one is poisoned and never
|
|
2730
|
+
// submitted, so deferring it only holds its retained snapshot for the diagnostic and commit-phase
|
|
2731
|
+
// graces on behalf of an attempt that is going to throw.
|
|
2732
|
+
if (txn.commitAttemptDoomedByPoison())
|
|
2733
|
+
return false;
|
|
2734
|
+
// performance.now(), not Date.now(): a clock correction must not extend a stalled commit's hold on its
|
|
2735
|
+
// diagnostic grace, nor report a legitimately slow one early.
|
|
2736
|
+
const now = performance.now();
|
|
2737
|
+
const deadline = (root.deferredPoisonDeadline ??= now + MAX_DEFERRED_POISON_TICKS * expiration);
|
|
2738
|
+
let hasUnsubmittedCommit = false;
|
|
2739
|
+
let unsubmittedCommitGraceExpired = false;
|
|
2740
|
+
if (now >= deadline) {
|
|
2741
|
+
// Only reclaim after the commit has genuinely stalled. Submitted writes no longer depend on these
|
|
2742
|
+
// separate snapshots; pre-submit handles can still carry writes needed by the attempt.
|
|
2743
|
+
txn.releaseStalledCommitReadResources();
|
|
2744
|
+
if (!txn.isProtectedCommit()) {
|
|
2745
|
+
if (!root.postSubmitPoisoned)
|
|
2746
|
+
txn.poisonAfterStalledSubmittedCommit();
|
|
2747
|
+
const links = new Set();
|
|
2748
|
+
const collect = (start) => {
|
|
2749
|
+
for (let link = start; link; link = link.next)
|
|
2750
|
+
links.add(link);
|
|
2751
|
+
};
|
|
2752
|
+
collect(root);
|
|
2753
|
+
collect(txn);
|
|
2754
|
+
collect(root.submittedLink);
|
|
2755
|
+
if (root.submittedLinks)
|
|
2756
|
+
for (const link of root.submittedLinks)
|
|
2757
|
+
collect(link);
|
|
2758
|
+
for (const link of links) {
|
|
2759
|
+
if (link.committing && !link.nativeCommitSubmitted) {
|
|
2760
|
+
hasUnsubmittedCommit = true;
|
|
2761
|
+
break;
|
|
2762
|
+
}
|
|
2763
|
+
}
|
|
2764
|
+
if (hasUnsubmittedCommit) {
|
|
2765
|
+
const unsubmittedDeadline = (root.unsubmittedCommitDeadline ??= now + exports.COMMIT_PHASE_GRACE * expiration);
|
|
2766
|
+
if (now >= unsubmittedDeadline) {
|
|
2767
|
+
unsubmittedCommitGraceExpired = true;
|
|
2768
|
+
txn.abortDueToTimeout();
|
|
2769
|
+
}
|
|
2770
|
+
}
|
|
2771
|
+
else
|
|
2772
|
+
root.unsubmittedCommitDeadline = undefined;
|
|
2773
|
+
}
|
|
2774
|
+
if (!txn.nativeCommitSubmitted &&
|
|
2775
|
+
txn.open === exports.TRANSACTION_STATE.CLOSED &&
|
|
2776
|
+
!(txn === root && submittedCommitStillInFlight(root)))
|
|
2777
|
+
return hasUnsubmittedCommit && !unsubmittedCommitGraceExpired;
|
|
2778
|
+
if (!root.stalledCommitLogged && now - lastStalledCommitErrorAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS) {
|
|
2779
|
+
lastStalledCommitErrorAt = now;
|
|
2780
|
+
root.stalledCommitLogged = true;
|
|
2781
|
+
harperLogger.error(`A native commit has not settled after exceeding the open-transaction limit by a further ${MAX_DEFERRED_POISON_TICKS * expiration}ms. Its outcome is unknown, so Harper will not abort or delete its writes; fresh ordinary work ` +
|
|
2782
|
+
`on this transaction is rejected until it settles, and the worker may require a restart, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
|
|
2783
|
+
}
|
|
2784
|
+
}
|
|
2785
|
+
else if (now - lastStalledCommitWarningAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS) {
|
|
2786
|
+
lastStalledCommitWarningAt = now;
|
|
2787
|
+
harperLogger.warn?.(`Transaction exceeded the open-transaction limit after submitting a native commit; preserving its unknown outcome, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
|
|
2788
|
+
}
|
|
2789
|
+
txn.timeout = Math.max(expiration, txn.timeoutBudget ?? 0);
|
|
2790
|
+
return true;
|
|
2791
|
+
}
|
|
2212
2792
|
function startMonitoringTxns() {
|
|
2213
2793
|
timer = setInterval(function () {
|
|
2214
2794
|
monitorTick++;
|
|
@@ -2241,6 +2821,15 @@ function startMonitoringTxns() {
|
|
|
2241
2821
|
txn.writeTimeout -= txnExpiration;
|
|
2242
2822
|
if (txn.timeout <= 0) {
|
|
2243
2823
|
const url = txn.getContext()?.url;
|
|
2824
|
+
if (deferForCommitInFlight(txn, url, txnExpiration))
|
|
2825
|
+
return;
|
|
2826
|
+
if (txn.open === exports.TRANSACTION_STATE.CLOSED &&
|
|
2827
|
+
!txn.commitAttemptDoomedByPoison() &&
|
|
2828
|
+
shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
|
|
2829
|
+
harperLogger.warn?.(`Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work; letting it complete, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
|
|
2830
|
+
txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
|
|
2831
|
+
return;
|
|
2832
|
+
}
|
|
2244
2833
|
if (txn.open === exports.TRANSACTION_STATE.CLOSED) {
|
|
2245
2834
|
if (!txn.transaction) {
|
|
2246
2835
|
// Nothing left to supervise, and this is the registry's only unconditional exit:
|
|
@@ -2250,14 +2839,16 @@ function startMonitoringTxns() {
|
|
|
2250
2839
|
txn.dropWriteSupervision();
|
|
2251
2840
|
return;
|
|
2252
2841
|
}
|
|
2253
|
-
//
|
|
2254
|
-
// replay commit (see the outstanding-iterators branch in commit()) and
|
|
2255
|
-
//
|
|
2256
|
-
//
|
|
2257
|
-
//
|
|
2258
|
-
//
|
|
2259
|
-
|
|
2260
|
-
txn.
|
|
2842
|
+
// Staged writes are not the monitor's to abort here: on the committed path they ride an
|
|
2843
|
+
// in-flight replay commit (see the outstanding-iterators branch in commit()) and dropping
|
|
2844
|
+
// them would re-introduce the silent write-loss-after-ack this structure prevents; on the
|
|
2845
|
+
// poisoned path abort(true) already discarded them. Only the snapshot's lifetime is left.
|
|
2846
|
+
// Close the owning iterators first so the handle comes back through doneReadTxn() — a
|
|
2847
|
+
// transaction poisoned outside transaction() has no callback settlement to do it.
|
|
2848
|
+
const closedIterators = txn.closeOwnedReadIterators();
|
|
2849
|
+
harperLogger.warn?.(`Read iterators held a ${txn.timedOut || txn.disconnected ? 'poisoned' : 'committed'} transaction's snapshot past the open-transaction limit; ${txn.transaction ? 'releasing it' : `closed ${closedIterators} abandoned iterator(s)`}, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
|
|
2850
|
+
if (txn.transaction)
|
|
2851
|
+
txn.releaseReadTxn();
|
|
2261
2852
|
}
|
|
2262
2853
|
else if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
|
|
2263
2854
|
// Parked in commit()'s pre-commit await — a `before`/`beforeIntermediate` hook, in practice a
|