harper 5.3.2 → 5.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cliOperations.ts +3 -3
- package/bin/copyDb.ts +4 -3
- package/components/DESIGN.md +12 -0
- package/components/packageComponent.ts +42 -17
- package/config/DESIGN.md +4 -0
- package/config/configUtils.ts +8 -8
- package/config/parseConfigFile.ts +11 -7
- package/dist/bin/cliOperations.js +3 -3
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/copyDb.js +3 -3
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/components/packageComponent.d.ts +9 -7
- package/dist/components/packageComponent.js +41 -16
- package/dist/components/packageComponent.js.map +1 -1
- package/dist/config/configUtils.js +4 -3
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/parseConfigFile.d.ts +1 -0
- package/dist/config/parseConfigFile.js +7 -3
- package/dist/config/parseConfigFile.js.map +1 -1
- package/dist/resources/DatabaseTransaction.d.ts +80 -11
- package/dist/resources/DatabaseTransaction.js +1100 -281
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +1 -1
- package/dist/resources/LMDBTransaction.js +4 -3
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/Resource.js +40 -14
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.js +1 -0
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.js +97 -34
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/write.js +34 -16
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/databases.d.ts +5 -0
- package/dist/resources/databases.js +38 -4
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/recordLock.d.ts +9 -4
- package/dist/resources/recordLock.js +79 -19
- package/dist/resources/recordLock.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +2 -0
- package/dist/server/DurableSubscriptionsSession.js +24 -24
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/mqtt.js +59 -29
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/serverHelpers/NodeAdapterResponse.js +18 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.js +8 -2
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/sharedMessageEncoding.js +3 -2
- package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -1
- package/dist/utility/npmUtilities.js +18 -7
- package/dist/utility/npmUtilities.js.map +1 -1
- package/package.json +1 -1
- package/resources/DESIGN.md +46 -10
- package/resources/DatabaseTransaction.ts +1268 -279
- package/resources/LMDBTransaction.ts +4 -3
- package/resources/Resource.ts +39 -14
- package/resources/RocksTransactionLogStore.ts +1 -0
- package/resources/Table.ts +120 -41
- package/resources/analytics/DESIGN.md +17 -0
- package/resources/analytics/write.ts +28 -15
- package/resources/databases.ts +40 -4
- package/resources/record-locks.md +34 -17
- package/resources/recordLock.ts +89 -22
- package/server/DESIGN.md +9 -1
- package/server/DurableSubscriptionsSession.ts +22 -23
- package/server/mqtt.ts +54 -29
- package/server/serverHelpers/NodeAdapterResponse.ts +19 -1
- package/server/serverHelpers/contentTypes.ts +8 -1
- package/server/serverHelpers/sharedMessageEncoding.ts +3 -2
- package/studio/web/assets/Chat-DjZNWwwA.js +2100 -0
- package/studio/web/assets/FloatingChat-BWfKK_RX.js +23 -0
- package/studio/web/assets/SearchDialog-CdlNxWef.js +1 -0
- package/studio/web/assets/{abnfDiagram-O67JEVCF-raXE027J.js → abnfDiagram-YUKMZFIV-DPR2lFRk.js} +1 -1
- package/studio/web/assets/{apiToken-DNrfij0E.js → apiToken-DyI8IwU3.js} +1 -1
- package/studio/web/assets/applications-DgGopKJq.js +296 -0
- package/studio/web/assets/architecture-WOLXFQ4H-DtJHIK0_.js +1 -0
- package/studio/web/assets/{architectureDiagram-NJMV4G6O-C8_Eet7w.js → architectureDiagram-47P4ROYG-MDymp2dh.js} +3 -3
- package/studio/web/assets/authStore-DE1HFvVP.js +3 -0
- package/studio/web/assets/{blockDiagram-BEXU5L5S-DR6S5-1D.js → blockDiagram-E7TT5TSB-O5qRgZmd.js} +3 -3
- package/studio/web/assets/button-C37w80Uk.js +21 -0
- package/studio/web/assets/{c4Diagram-YGBWAQC7-BAfZ1G01.js → c4Diagram-CRA5TL53-BdPORGHo.js} +2 -2
- package/studio/web/assets/channel-yCStoq0_.js +1 -0
- package/studio/web/assets/{chevron-up-CFxbMGv2.js → chevron-up-DozwrIz-.js} +1 -1
- package/studio/web/assets/chunk-2BW5OAIV-DJ-iXJAw.js +10 -0
- package/studio/web/assets/{chunk-ZIGJFQKS-Cs8fUn7y.js → chunk-3YJQHVM4-BOgacqS1.js} +2 -2
- package/studio/web/assets/{chunk-7PRAP22T-KyqX6x3f.js → chunk-4EA7E6EY-C-7xqUBl.js} +1 -1
- package/studio/web/assets/{chunk-GWA4HPMP-DsejB6gA.js → chunk-7TKQ45FW-C3XA1bIN.js} +1 -1
- package/studio/web/assets/{chunk-GTNCS2PH-DGHLXyow.js → chunk-AW2ZBBNX-BdvgD1UW.js} +3 -3
- package/studio/web/assets/{chunk-5DYCD2WN-BceBfIHT.js → chunk-DBDB3WZW-CKKxIc6F.js} +1 -1
- package/studio/web/assets/{chunk-MBY4JIJT-BwSGweQP.js → chunk-E2ZNV5FY-bVefNHvz.js} +3 -3
- package/studio/web/assets/chunk-EU5HNXII-BHv2kWp4.js +213 -0
- package/studio/web/assets/chunk-FVRAUYC3-BoL_NO6b.js +2 -0
- package/studio/web/assets/{chunk-742MDFTN-CIkt8ijt.js → chunk-HJ2JQQFS-N_98oMRS.js} +1 -1
- package/studio/web/assets/{chunk-NETBCI7D-C8UnOw_f.js → chunk-HTAEGDNF-BQ1fSjPs.js} +1 -1
- package/studio/web/assets/{chunk-WEXAMYUT-BAIhTwyh.js → chunk-KQW6MTUR-qP0QiQAg.js} +1 -1
- package/studio/web/assets/chunk-NGNAAXSQ-DqoPQk1a.js +136 -0
- package/studio/web/assets/{chunk-UA2S7LBM-BHpEdH8D.js → chunk-NTY3LDVX-CSBFjlJg.js} +1 -1
- package/studio/web/assets/chunk-VPRB5NB3-BVHOe9Q-.js +129 -0
- package/studio/web/assets/{chunk-7INBJB4K-BpeAXyOi.js → chunk-XC4XBNZT-46uQwtF4.js} +10 -10
- package/studio/web/assets/{classDiagram-v2-NBCMYWYE-DmDPkt2u.js → classDiagram-v2-K4WV4PDN-Da0ps1Ui.js} +3 -3
- package/studio/web/assets/cn-DpF1PxBT.js +1 -0
- package/studio/web/assets/{createLucideIcon-fO5dL7D5.js → createLucideIcon-DLWRoWN-.js} +1 -1
- package/studio/web/assets/{cssMode-DBucrQdT.js → cssMode-4ddO86ey.js} +1 -1
- package/studio/web/assets/cynefin-EF2NZ3EQ-jQ2eCUaU.js +1 -0
- package/studio/web/assets/cynefinDiagram-3GCD6N5R-TNi5iS0E.js +62 -0
- package/studio/web/assets/dagre-W4DXFKR2-BnLHmA44.js +4 -0
- package/studio/web/assets/{diagram-22UHCM2B-DJCvcaWP.js → diagram-2UJZ2QOL-r_jehhZR.js} +7 -7
- package/studio/web/assets/{diagram-3UASUU5V-Du6VKWln.js → diagram-OVF4WLC6-BSUXGVh8.js} +11 -11
- package/studio/web/assets/{diagram-CDSNMT55-l5Ga0u-W.js → diagram-PFPMY2P6-BrVedycw.js} +3 -3
- package/studio/web/assets/{diagram-MLGK6HIB-CNQY47LA.js → diagram-UMYRVEAY-CBIaakBB.js} +1 -1
- package/studio/web/assets/diagram-YEKJPTXX-DFmmDXCl.js +41 -0
- package/studio/web/assets/diagram-ZIFT7M5P-CQfOWxOi.js +3 -0
- package/studio/web/assets/dist-9lpzt2Hn.js +11 -0
- package/studio/web/assets/{ebnfDiagram-ZINNZB2B-5OnkTpfi.js → ebnfDiagram-VR2GEFS7-FXA3Xe6e.js} +1 -1
- package/studio/web/assets/{editor-CHpQCuBq.js → editor-DATuu7_v.js} +1 -1
- package/studio/web/assets/elk-IJKZMXRS-CC4gfVRU.js +27 -0
- package/studio/web/assets/{erDiagram-OPXOYQCR-CQlCX84D.js → erDiagram-O2IAPWRE-C-BkDTPl.js} +2 -2
- package/studio/web/assets/errorText-8xV8WZBt.js +1 -0
- package/studio/web/assets/eventmodeling-K75KTNOO-D0dJ_PbR.js +1 -0
- package/studio/web/assets/flowDiagram-OXPTDLAJ-C1LEc_Hf.js +1 -0
- package/studio/web/assets/{ganttDiagram-FUAMR5RP-bloX3Voy.js → ganttDiagram-R7TSEDQI-85F44BIo.js} +4 -4
- package/studio/web/assets/{getAnalytics-C0FHjtdQ.js → getAnalytics-B1xpd4kG.js} +1 -1
- package/studio/web/assets/gitGraph-VSP46ZUC-BBmqrmjK.js +1 -0
- package/studio/web/assets/gitGraphDiagram-XJZIOB7I-CFY6iMwg.js +106 -0
- package/studio/web/assets/graphlib-DS17s2tU.js +1 -0
- package/studio/web/assets/{html-B1iA8q38.js → html-c4OKnEF1.js} +1 -1
- package/studio/web/assets/{htmlMode-CrL3YO5Y.js → htmlMode-yXjqUsW9.js} +1 -1
- package/studio/web/assets/index-CDnn0-PN.css +1 -0
- package/studio/web/assets/index-CIK9Nhgo.js +833 -0
- package/studio/web/assets/index.lazy-DQpKL_QQ.js +1 -0
- package/studio/web/assets/info-OHQRW6UA-Bs2aEco7.js +1 -0
- package/studio/web/assets/{infoDiagram-VRGFBTTK-B0J1132p.js → infoDiagram-5W2HQ5XZ-d6yG1yfZ.js} +1 -1
- package/studio/web/assets/invalidateEntityQueries-BkWXiqjz.js +1 -0
- package/studio/web/assets/{ishikawaDiagram-OU5B5YK6-BNk0tuQx.js → ishikawaDiagram-K3B6WC7H-CSYz-WRA.js} +2 -2
- package/studio/web/assets/{javascript-CXgQMkoa.js → javascript-CP1WPkLu.js} +1 -1
- package/studio/web/assets/{journeyDiagram-ZHPQQLJL-Be79RMQM.js → journeyDiagram-COXZFDF6-BDj6Wqcj.js} +3 -3
- package/studio/web/assets/{jsonMode-DycqRKOv.js → jsonMode-BWsIHqbb.js} +1 -1
- package/studio/web/assets/{kanban-definition-PNTS6WVX-4uuoQP9W.js → kanban-definition-P3RFRI5V-C2Bjfbp-.js} +6 -6
- package/studio/web/assets/{languageServices-HYHV1wDJ.js → languageServices-Ca72n4Ql.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-CH_6SYiA.js → lspLanguageFeatures-BQdsSDeh.js} +1 -1
- package/studio/web/assets/{mermaid-parser.core-V1UYMeXV.js → mermaid-parser.core-Ch_T61iL.js} +6 -6
- package/studio/web/assets/mermaid.core-BOAgKzo5.js +44 -0
- package/studio/web/assets/{mindmap-definition-NLK3R4M7-C70Ctm0J.js → mindmap-definition-LZFPQGKD-DzOapzMM.js} +3 -3
- package/studio/web/assets/notifications-BYgO5-yk.js +1 -0
- package/studio/web/assets/notifications-DsGk-TnW.js +1 -0
- package/studio/web/assets/operations-Dy69AlTr.js +4 -0
- package/studio/web/assets/packet-JDAUHWVQ-CGHB_EIE.js +1 -0
- package/studio/web/assets/{pegDiagram-GJSIUBJH-d6z1cc7W.js → pegDiagram-BGZESJAR-ADzOIKrz.js} +1 -1
- package/studio/web/assets/pie-XZMESJXO-Bijz5Y2o.js +1 -0
- package/studio/web/assets/pieDiagram-CAPJLFHJ-VM7fvbwO.js +39 -0
- package/studio/web/assets/profile-DzQvoYBF.js +1 -0
- package/studio/web/assets/{quadrantDiagram-O4NWA36T-CIF5Qkou.js → quadrantDiagram-GDMTTRAM-Bmza84_j.js} +3 -3
- package/studio/web/assets/radar-ABXABTNO-IaC8pB63.js +1 -0
- package/studio/web/assets/railroad-I3PHUGI6-CCrSUqeq.js +1 -0
- package/studio/web/assets/railroad-abnf-LNEFI6M7-BhobVp-t.js +1 -0
- package/studio/web/assets/railroad-ebnf-DWYD2UWJ-OHhGUTEh.js +1 -0
- package/studio/web/assets/railroad-peg-JR7BVNR7-FupC3Jcn.js +1 -0
- package/studio/web/assets/{railroadDiagram-XR7U4H2S-y3Jv2U6N.js → railroadDiagram-SM67HX2B-CgFGOVI5.js} +1 -1
- package/studio/web/assets/regions-Sv_IN7XA.js +1 -0
- package/studio/web/assets/{register-DCN4QUii.js → register-AM2_79FO.js} +3 -3
- package/studio/web/assets/request-BJj8dbdU.js +2 -0
- package/studio/web/assets/{requirementDiagram-PLB6GJNP-ChINZEmu.js → requirementDiagram-X7JNWC4A-BHoQgkoT.js} +4 -4
- package/studio/web/assets/restartGate-D_e93lFE.js +1 -0
- package/studio/web/assets/restartUnconfirmed-BqcDZ__Y.js +1 -0
- package/studio/web/assets/sankeyDiagram-UM26HJRW-8tFu_oUe.js +40 -0
- package/studio/web/assets/sequenceDiagram-ZO4K6R2Y-jryb1rwT.js +169 -0
- package/studio/web/assets/setComponentFile-BCnAnxjR.js +1 -0
- package/studio/web/assets/setup-Ca4RpPzm.js +2 -0
- package/studio/web/assets/{stateDiagram-v2-GCMORJYK-Bs8RIb9K.js → stateDiagram-v2-TBUQTH76-CnWE5yFn.js} +2 -2
- package/studio/web/assets/status-D05eZ5pQ.js +61 -0
- package/studio/web/assets/{swimlanes-2SLR337P-B52xrf6t.js → swimlanes-N4OXWK64-CBlFzDJn.js} +1 -1
- package/studio/web/assets/swimlanesDiagram-K3J5GTZL-DvFLtUPi.js +8 -0
- package/studio/web/assets/{timeline-definition-EJHVYXUP-DJSITpQX.js → timeline-definition-YOQAKGHF-CFWqHrqI.js} +4 -4
- package/studio/web/assets/{toggleHighContrast-CfAp6Sxb.js → toggleHighContrast-JRkLfyfb.js} +1 -1
- package/studio/web/assets/treeView-D4JQ5SDB-CpV9zuVb.js +1 -0
- package/studio/web/assets/treemap-SAJKECNS-Bc94Tqiq.js +1 -0
- package/studio/web/assets/{tsMode-HZR0bfq4.js → tsMode-C_-eSRVR.js} +1 -1
- package/studio/web/assets/{typescript-UMJSfZyF.js → typescript-BYXigM_n.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-B379CS-h.js → useEntityRestURL-HHmjsRm9.js} +1 -1
- package/studio/web/assets/usecaseDiagram-VIAY4XPW-Brf2xP3S.js +328 -0
- package/studio/web/assets/{vendor-core-6LhfJlRC.js → vendor-core-PejxhUWQ.js} +1 -1
- package/studio/web/assets/vendor-datadog-GjET_Cbn.js +6 -0
- package/studio/web/assets/{vendor-react-BOliS0BL.js → vendor-react-DSIx92Ld.js} +1 -1
- package/studio/web/assets/vendor-tanstack-C5Y-nrd2.js +2 -0
- package/studio/web/assets/vendor-ui-BBWHOEoQ.js +41 -0
- package/studio/web/assets/{vennDiagram-UO4OBE2U-DZTsDA4k.js → vennDiagram-BLWOH2XV-XxOyJPq_.js} +3 -3
- package/studio/web/assets/wardley-7MLQ67FV-DRW0zuE4.js +1 -0
- package/studio/web/assets/{wardleyDiagram-VNRHLVJA-qu_f7Tvl.js → wardleyDiagram-YMQ3BMBF-B3WskJ9W.js} +3 -3
- package/studio/web/assets/wasAReleasedBeforeB-D5grAaGM.js +1 -0
- package/studio/web/assets/{workers-Bn_0_L1R.js → workers-DY0ka4Y7.js} +1 -1
- package/studio/web/assets/{xml-CG2N-1r1.js → xml-CdGINpmN.js} +1 -1
- package/studio/web/assets/xychartDiagram-TAQBALBS-D5NV4Env.js +7 -0
- package/studio/web/assets/{yaml-68pda6Av.js → yaml-DYeF07_w.js} +1 -1
- package/studio/web/index.html +17 -15
- package/utility/npmUtilities.ts +17 -7
- package/studio/web/assets/Chat-CrxPD4ph.js +0 -2294
- package/studio/web/assets/FloatingChat-EXzoUQ-9.js +0 -23
- package/studio/web/assets/SearchDialog-7qRL8Yqx.js +0 -1
- package/studio/web/assets/_basePickBy-QJXUw_pr.js +0 -1
- package/studio/web/assets/_baseUniq-DR4vVjyC.js +0 -1
- package/studio/web/assets/alertDialog-BbWD9Xxa.js +0 -1
- package/studio/web/assets/applications-CdqjiSQe.js +0 -296
- package/studio/web/assets/architecture-7GRP2DOG-DDmz3hpI.js +0 -1
- package/studio/web/assets/arrow-up-D2g_57vT.js +0 -1
- package/studio/web/assets/authStore-CJCaADw9.js +0 -3
- package/studio/web/assets/button-DWwfgGWI.js +0 -21
- package/studio/web/assets/channel-D0AkPjJ8.js +0 -1
- package/studio/web/assets/chunk-3FUC2YCW-DuRp-j19.js +0 -2
- package/studio/web/assets/chunk-7M6MHVWA-h6WlaTzO.js +0 -213
- package/studio/web/assets/chunk-FOHPRMQF-DzxwRta7.js +0 -161
- package/studio/web/assets/chunk-O7XYJQB3-CBS3nZYV.js +0 -128
- package/studio/web/assets/chunk-Z7XXMR3K-DSHbImdf.js +0 -10
- package/studio/web/assets/cn-6T8VranJ.js +0 -1
- package/studio/web/assets/cynefin-OW5HDTMX-DIvbXLSj.js +0 -1
- package/studio/web/assets/cynefinDiagram-VND7K2PF-BdtaXiCE.js +0 -62
- package/studio/web/assets/dagre-6A5THRUB-C7RUWHyz.js +0 -4
- package/studio/web/assets/diagram-ATOU4E4O-dxRfeG4B.js +0 -3
- package/studio/web/assets/diagram-MPIPVDR6-7prveD1B.js +0 -41
- package/studio/web/assets/dist-BrRuMUzt.js +0 -11
- package/studio/web/assets/download-Cn6tPG4L.js +0 -1
- package/studio/web/assets/elk-276RUBZZ-DxDQPjQT.js +0 -27
- package/studio/web/assets/errorText-NpBmvk9I.js +0 -1
- package/studio/web/assets/eventmodeling-NTZA5JFV-DONQf5aG.js +0 -1
- package/studio/web/assets/flowDiagram-KWPJA3E3-BoZ8_QR1.js +0 -1
- package/studio/web/assets/gitGraph-4MIJSDKK-BrXWmJt2.js +0 -1
- package/studio/web/assets/gitGraphDiagram-X574FWY7-C2KnnZeY.js +0 -106
- package/studio/web/assets/graphlib-C9Tl7QrK.js +0 -1
- package/studio/web/assets/index-QLZw6xrU.css +0 -1
- package/studio/web/assets/index-qCLyhyah.js +0 -828
- package/studio/web/assets/index.lazy-DCx-DxIV.js +0 -2
- package/studio/web/assets/info-A6RAGUB7-BhzBHe4U.js +0 -1
- package/studio/web/assets/mermaid.core-1PX6R49U.js +0 -44
- package/studio/web/assets/notifications-BSObVIY1.js +0 -1
- package/studio/web/assets/notifications-DFLXk68b.js +0 -1
- package/studio/web/assets/packet-AYTQ26CC-DWbXHx5g.js +0 -1
- package/studio/web/assets/pie-WAS4IAKB-DjoTm3yr.js +0 -1
- package/studio/web/assets/pieDiagram-5QR66LMP-Dz3RHpDA.js +0 -39
- package/studio/web/assets/profile-TBjzOMIy.js +0 -1
- package/studio/web/assets/radar-RG4KPBEZ-CVBgjCOj.js +0 -1
- package/studio/web/assets/railroad-74A4TZTK-BdhXw10Q.js +0 -1
- package/studio/web/assets/railroad-abnf-HS5TGJTU-BcoQPN39.js +0 -1
- package/studio/web/assets/railroad-ebnf-LZEXJU2U-B1Lke2JN.js +0 -1
- package/studio/web/assets/railroad-peg-WCYAUIDC-D-oJFkiV.js +0 -1
- package/studio/web/assets/regions-C4an4H7e.js +0 -1
- package/studio/web/assets/restartGate-nHAFLDh-.js +0 -1
- package/studio/web/assets/sankeyDiagram-IPEJSGJF-D9AyLce2.js +0 -40
- package/studio/web/assets/sequenceDiagram-PO4LG4MO-hfk4aIEm.js +0 -169
- package/studio/web/assets/setComponentFile-BrqOM1Zm.js +0 -1
- package/studio/web/assets/setup-0v2MCm-E.js +0 -2
- package/studio/web/assets/status-DdezofUu.js +0 -61
- package/studio/web/assets/swimlanesDiagram-TC7HE7FX-DfN7BAP5.js +0 -8
- package/studio/web/assets/tabs-BCTQzoQK.js +0 -1
- package/studio/web/assets/treeView-Q6P3EWNA-C-epdxRo.js +0 -1
- package/studio/web/assets/treemap-WGGIJYW6-DOVyp_zw.js +0 -1
- package/studio/web/assets/usecaseDiagram-POWQR4AR-B6252cbq.js +0 -328
- package/studio/web/assets/vendor-datadog-ozKf-pts.js +0 -6
- package/studio/web/assets/vendor-tanstack-Bntfka-M.js +0 -2
- package/studio/web/assets/vendor-ui-9EYYDgoN.js +0 -41
- package/studio/web/assets/wardley-WFR3VGLG-Cfqoe6z4.js +0 -1
- package/studio/web/assets/xychartDiagram-PMCCYNJV-UrOKRsmR.js +0 -7
|
@@ -60,7 +60,7 @@ const MAX_RETRIES = 40;
|
|
|
60
60
|
export const COMMIT_PHASE_GRACE = 10;
|
|
61
61
|
// Cap the per-retry backoff so replication-applied transactions, which retry conflicts without a
|
|
62
62
|
// cap (see the commit rejection handler), don't grow the delay unbounded.
|
|
63
|
-
|
|
63
|
+
let MAX_RETRY_DELAY_MS = 1000;
|
|
64
64
|
// Every native write commit currently outstanding on this worker thread, oldest first.
|
|
65
65
|
// checkOverloaded() sheds new application writes once the OLDEST of these has been outstanding
|
|
66
66
|
// past MAX_OUTSTANDING_TXN_DURATION, so a commit whose promise never settles (harper#2001) is
|
|
@@ -101,12 +101,20 @@ const OVERLOAD_LOG_MIN_INTERVAL_MS = 1000;
|
|
|
101
101
|
// wedge and would starve `abandon`, which names a different transaction and is the only server-side
|
|
102
102
|
// record of why a request was failed. Mutates only when it grants, so a caller with its own
|
|
103
103
|
// suppression as well (checkOverloaded's per-node `logged`) must test that first.
|
|
104
|
-
const lastStuckCommitLogAt = { shed: -Infinity, abandon: -Infinity };
|
|
105
|
-
function allowStuckCommitLog(site: 'shed' | 'abandon', now: number): boolean {
|
|
104
|
+
const lastStuckCommitLogAt = { shed: -Infinity, abandon: -Infinity, overdue: -Infinity };
|
|
105
|
+
function allowStuckCommitLog(site: 'shed' | 'abandon' | 'overdue', now: number): boolean {
|
|
106
106
|
if (now - lastStuckCommitLogAt[site] <= OVERLOAD_LOG_MIN_INTERVAL_MS) return false;
|
|
107
107
|
lastStuckCommitLogAt[site] = now;
|
|
108
108
|
return true;
|
|
109
109
|
}
|
|
110
|
+
/** Test seam: clears the stuck- and stalled-commit log cooldowns so consecutive tests can assert on log emission without sleeping. */
|
|
111
|
+
export function resetStuckCommitLogCooldownForTests(): void {
|
|
112
|
+
lastStuckCommitLogAt.shed = -Infinity;
|
|
113
|
+
lastStuckCommitLogAt.abandon = -Infinity;
|
|
114
|
+
lastStuckCommitLogAt.overdue = -Infinity;
|
|
115
|
+
lastStalledCommitWarningAt = -Infinity;
|
|
116
|
+
lastStalledCommitErrorAt = -Infinity;
|
|
117
|
+
}
|
|
110
118
|
|
|
111
119
|
// Which database/table, which native transaction, and which request a stuck commit belongs to —
|
|
112
120
|
// without it a wedge gives no indication of what to investigate (harper#2001).
|
|
@@ -323,12 +331,27 @@ let replayedWritesWarned = false;
|
|
|
323
331
|
* committed or aborted, and every caller is a cleanup path whose own callers have no handler — a
|
|
324
332
|
* throw there would abandon the rest of the cleanup.
|
|
325
333
|
*/
|
|
326
|
-
function abortNativeTransaction(
|
|
334
|
+
function abortNativeTransaction(
|
|
335
|
+
transaction: RocksTransaction | null | undefined,
|
|
336
|
+
context: string,
|
|
337
|
+
store?: any,
|
|
338
|
+
startedFrom?: { resourceName: string; method: string }
|
|
339
|
+
): void {
|
|
327
340
|
if (transaction == null) return;
|
|
341
|
+
const txn = transaction as RocksTransactionWithRetry;
|
|
342
|
+
if (txn.abortedByWrapper || txn.committed || txn.commitInFlight) return;
|
|
343
|
+
txn.abortedByWrapper = true;
|
|
328
344
|
try {
|
|
329
345
|
transaction.abort();
|
|
330
346
|
} catch (error) {
|
|
331
|
-
|
|
347
|
+
if ((error as any)?.code === 'ERR_TRANSACTION_ABANDONED') {
|
|
348
|
+
harperLogger.error?.(
|
|
349
|
+
`Aborted a transaction with a durable log entry (${context}), ${describeCommitIdentity(store, startedFrom, transaction)}`,
|
|
350
|
+
error
|
|
351
|
+
);
|
|
352
|
+
} else {
|
|
353
|
+
harperLogger.debug?.(context, error);
|
|
354
|
+
}
|
|
332
355
|
}
|
|
333
356
|
}
|
|
334
357
|
|
|
@@ -546,7 +569,13 @@ export type TransactionWrite = {
|
|
|
546
569
|
entry?: Partial<Entry>;
|
|
547
570
|
before?: () => void | Promise<void>;
|
|
548
571
|
beforeIntermediate?: () => void | Promise<void>;
|
|
549
|
-
commit?: (
|
|
572
|
+
commit?: (
|
|
573
|
+
txnTime: number,
|
|
574
|
+
existingEntry: Partial<Entry>,
|
|
575
|
+
retry: boolean,
|
|
576
|
+
transaction: any,
|
|
577
|
+
committedBy?: DatabaseTransaction
|
|
578
|
+
) => MaybePromise<void>;
|
|
550
579
|
// Once a write has been taken over, the transaction committing it is not the one that staged it, and
|
|
551
580
|
// overload accounting, the replay marker and a no-op write's removal all belong to the committer.
|
|
552
581
|
validate?: (txnTime: number, committedBy: DatabaseTransaction) => void;
|
|
@@ -563,18 +592,21 @@ export type TransactionWrite = {
|
|
|
563
592
|
result?: any;
|
|
564
593
|
// blobs that were pre-saved as part of this write; used to clean up files if the commit is skipped or aborted
|
|
565
594
|
savedBlobs?: Blob[];
|
|
566
|
-
//
|
|
567
|
-
//
|
|
595
|
+
// this round staged neither a record nor an audit entry. An earlier durable audit entry may
|
|
596
|
+
// still own savedBlobs. Reset at the top of each commit-handler invocation.
|
|
568
597
|
skipped?: boolean;
|
|
569
598
|
// sticky: a non-isRetry staging of this write appended its audit entry (set in save(); the retry
|
|
570
599
|
// dedup guards in the commit handler read it to ignore the write's own orphaned entry)
|
|
571
600
|
appendedAuditEntry?: boolean;
|
|
601
|
+
// the native handle whose batch holds that audit entry (an iterator replay discards the original's batch)
|
|
602
|
+
appendedTo?: RocksTransaction;
|
|
572
603
|
// the transaction holding this write in its `writes` (set in addWrite). A deferred write's save() is
|
|
573
604
|
// only its trigger, so it can be triggered after the context has moved on to another transaction;
|
|
574
605
|
// this is who commits it when the transaction current at that point is not a scope (#2292).
|
|
575
606
|
stagedIn?: DatabaseTransaction;
|
|
576
607
|
// the preceding write to the same store and key in this transaction, if any (linked in addWrite)
|
|
577
608
|
priorWrite?: TransactionWrite;
|
|
609
|
+
keyId?: unknown;
|
|
578
610
|
// set only by a write that BOTH reads priorStagedWrite() and publishes stagedEntry; addWrite orders
|
|
579
611
|
// those against earlier same-key writes. A write that does one or neither would be ordered against a
|
|
580
612
|
// basis it cannot consume.
|
|
@@ -589,15 +621,15 @@ export type TransactionWrite = {
|
|
|
589
621
|
// stored, so blobs this write saved are only reachable through its audit entry. Reset per
|
|
590
622
|
// commit-handler round, like stagedEntry.
|
|
591
623
|
superseded?: boolean;
|
|
592
|
-
//
|
|
593
|
-
//
|
|
624
|
+
// Audit pruning owns blobs referenced by an appended audit entry once its handle commits or
|
|
625
|
+
// has a durable log; a staged entry in an unsubmitted handle owns none.
|
|
594
626
|
blobsAuditReferenced?: boolean;
|
|
595
627
|
// Lock handle set by Table._writeUpdate when the write is staged through a held record lock;
|
|
596
628
|
// used in DatabaseTransaction.save() to assign and track that handle's write versions.
|
|
597
629
|
lockHandle?: RecordLockHandle;
|
|
598
|
-
// Per-operation holder version, set once on first save() and reused on retry so
|
|
599
|
-
//
|
|
600
|
-
//
|
|
630
|
+
// Per-operation holder version, set once on first save() and reused on retry so each lock write
|
|
631
|
+
// keeps its own candidate version across retries instead of falling back to stagedTxnTime or an
|
|
632
|
+
// explicit this.timestamp shared on the transaction.
|
|
601
633
|
lockStamp?: number;
|
|
602
634
|
// Version staged by a write that actually changed the record this round. It advances any
|
|
603
635
|
// same-key held handle's floor only after the native transaction commits successfully.
|
|
@@ -614,6 +646,11 @@ export type TransactionWrite = {
|
|
|
614
646
|
// the commit derives stored state (folds, index diffs, residency) from its base entry, so
|
|
615
647
|
// save() must reload that base through the committing transaction's snapshot
|
|
616
648
|
reloadCommitBase?: boolean;
|
|
649
|
+
stagedOn?: RocksTransaction;
|
|
650
|
+
// the handle and retry round this write last ran its commit handler in: a retry re-saves writes in order,
|
|
651
|
+
// so a sibling's saved/skipped flags describe the current snapshot only once it has staged in this round
|
|
652
|
+
committedOn?: RocksTransaction;
|
|
653
|
+
committedRound?: number;
|
|
617
654
|
baseReadTxn?: any;
|
|
618
655
|
writeGeneration?: WriteGeneration;
|
|
619
656
|
instanceClosed?: boolean;
|
|
@@ -676,7 +713,39 @@ function clearAttemptState(txn: DatabaseTransaction): void {
|
|
|
676
713
|
txn.stalledCommitResourcesReleased = false;
|
|
677
714
|
}
|
|
678
715
|
|
|
679
|
-
|
|
716
|
+
// Returned by takeCompletions() when nothing is staged; only its length is read before Promise.all.
|
|
717
|
+
const NO_COMPLETIONS: Promise<void>[] = Object.freeze([]) as unknown as Promise<void>[];
|
|
718
|
+
|
|
719
|
+
type RocksTransactionWithRetry = RocksTransaction & {
|
|
720
|
+
isRetry?: boolean;
|
|
721
|
+
retries?: number;
|
|
722
|
+
hasStagedLog?: boolean;
|
|
723
|
+
stagedTxnTime?: number;
|
|
724
|
+
commitStartedAt?: number;
|
|
725
|
+
commitOverdueLogged?: boolean;
|
|
726
|
+
lockHolds?: Set<RecordLockHandle>;
|
|
727
|
+
holdBudgetSpent?: boolean;
|
|
728
|
+
abortedByWrapper?: boolean;
|
|
729
|
+
commitInFlight?: boolean;
|
|
730
|
+
committed?: boolean;
|
|
731
|
+
immediateCommit?: boolean;
|
|
732
|
+
};
|
|
733
|
+
|
|
734
|
+
// Crash-recovery replay sets `isRetry` before `save()`, so `hasStagedLog` is never set there.
|
|
735
|
+
function hasDurableLog(transaction: RocksTransaction | null | undefined, error?: { hasLog?: boolean }): boolean {
|
|
736
|
+
const txn = transaction as RocksTransactionWithRetry | null | undefined;
|
|
737
|
+
return error?.hasLog === true || Boolean(txn?.isRetry && txn?.hasStagedLog);
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
function releaseTransactionLockHolds(transaction: RocksTransaction | null | undefined): void {
|
|
741
|
+
const txn = transaction as RocksTransactionWithRetry | null | undefined;
|
|
742
|
+
const holds = txn?.lockHolds;
|
|
743
|
+
if (!holds) return;
|
|
744
|
+
txn.lockHolds = undefined;
|
|
745
|
+
for (const handle of holds) {
|
|
746
|
+
handle.releaseCommitHold?.();
|
|
747
|
+
}
|
|
748
|
+
}
|
|
680
749
|
|
|
681
750
|
export class DatabaseTransaction implements Transaction {
|
|
682
751
|
#context: Context;
|
|
@@ -694,11 +763,16 @@ export class DatabaseTransaction implements Transaction {
|
|
|
694
763
|
// the last staged write per store and key, used to chain repeat writes to the same key (linkWrite)
|
|
695
764
|
declare writesByKey?: Map<any, Map<unknown, TransactionWrite>>;
|
|
696
765
|
completions: Promise<void>[] = []; // the set of outstanding async operations to complete
|
|
766
|
+
declare private completionHandles?: (RocksTransaction | undefined)[];
|
|
697
767
|
db: RootDatabaseKind;
|
|
698
768
|
transaction: RocksTransactionWithRetry;
|
|
699
769
|
readTxn: ReadTransaction;
|
|
700
770
|
readTxnRefCount: number;
|
|
701
771
|
readTxnsUsed: number;
|
|
772
|
+
// Bumped each time this instance acquires a read handle, so a reference can be returned to the
|
|
773
|
+
// acquisition it came from (doneReadTxn()). Not the handle's identity: lmdb-js hands back the same
|
|
774
|
+
// read Txn object for a new snapshot.
|
|
775
|
+
readTxnGeneration = 0;
|
|
702
776
|
timeout: number;
|
|
703
777
|
// Write recency, tracked separately from `timeout`: set only by addWrite, never by a read. `timeout`
|
|
704
778
|
// is re-armed by reads too (on a link with no pending writes of its own, see the fast path in
|
|
@@ -731,6 +805,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
731
805
|
// unrelated request on the thread. Cleared when the logical commit settles, so a reused
|
|
732
806
|
// transaction's next batch starts on a fresh budget.
|
|
733
807
|
declare commitStartedAt?: number;
|
|
808
|
+
declare commitOverdueLogged?: boolean;
|
|
734
809
|
// Whether this link is why its chain root is write-supervised (see endWriteSupervision).
|
|
735
810
|
declare writeSupervised?: boolean;
|
|
736
811
|
declare stale: boolean;
|
|
@@ -821,6 +896,11 @@ export class DatabaseTransaction implements Transaction {
|
|
|
821
896
|
declare ownedReadIterators?: Set<OwnedReadIterator>;
|
|
822
897
|
// Depth of commit() attempts on this link that have not reached a native outcome.
|
|
823
898
|
commitsInFlight = 0;
|
|
899
|
+
declare private activeCommitHandle?: RocksTransaction;
|
|
900
|
+
declare private activeCommitHandles?: Set<RocksTransaction>;
|
|
901
|
+
declare private activeHandleStagedTxnTime?: number;
|
|
902
|
+
declare private activeHandleInFlight?: boolean;
|
|
903
|
+
declare private activeHandleCommitted?: boolean;
|
|
824
904
|
// A poison landed while a commit was already in flight somewhere in the chain. Excuses the head's
|
|
825
905
|
// cascade into `next`, but not a link the monitor poisoned BEFORE any commit started.
|
|
826
906
|
declare poisonedMidCommit?: boolean;
|
|
@@ -829,6 +909,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
829
909
|
// can visit every link of a chain, so a counter would advance once per link per tick.
|
|
830
910
|
declare deferredPoisonDeadline?: number;
|
|
831
911
|
declare unsubmittedCommitDeadline?: number;
|
|
912
|
+
// Retry count of the chain's logged retry at the monitor's last tick, and the tick that decided whether
|
|
913
|
+
// it had moved: a count that stops moving is a native attempt that never settled, and every link the
|
|
914
|
+
// monitor visits in one tick must read the same decision.
|
|
915
|
+
declare loggedRetriesSeen?: number;
|
|
916
|
+
declare loggedRetriesChangedAt?: number;
|
|
917
|
+
declare loggedRetriesTick?: number;
|
|
918
|
+
declare loggedRetryAdvancing?: boolean;
|
|
832
919
|
// Per-link submission state. A commit method can still be parked in pre-commit work, where an abort
|
|
833
920
|
// is safe; once storage work has been submitted, its outcome is unknown and monitor cleanup must not
|
|
834
921
|
// clear writes or blobs that the eventual commit can reference.
|
|
@@ -928,6 +1015,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
928
1015
|
readTransactionOwners.set(transaction, this);
|
|
929
1016
|
this.rangeReadActive = false;
|
|
930
1017
|
this.readTxnsUsed = 1;
|
|
1018
|
+
this.readTxnGeneration++;
|
|
931
1019
|
this.baseReadRefConsumed = false;
|
|
932
1020
|
this.handleOpenedAt = performance.now();
|
|
933
1021
|
}
|
|
@@ -1028,12 +1116,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1028
1116
|
// the array it never got threw (issue #2222). Capturing an Error per read that nothing can dump
|
|
1029
1117
|
// is not worth doing either, so those reads stay untraced.
|
|
1030
1118
|
if (DEBUG_LONG_TXNS && this.stackTraces) this.stackTraces.push(new StartedTransaction());
|
|
1031
|
-
this.readTxnsUsed++;
|
|
1119
|
+
if (readTxn) this.readTxnsUsed++;
|
|
1032
1120
|
return readTxn;
|
|
1033
1121
|
}
|
|
1034
1122
|
|
|
1035
|
-
doneReadTxn() {
|
|
1036
|
-
if (!this.transaction) return;
|
|
1123
|
+
doneReadTxn(generation = this.readTxnGeneration) {
|
|
1124
|
+
if (!this.transaction || generation !== this.readTxnGeneration) return;
|
|
1037
1125
|
if (--this.readTxnsUsed === 0) {
|
|
1038
1126
|
// The native handle was only being held open for the read iterators: any writes staged
|
|
1039
1127
|
// through it were already committed at commit() time by replaying them onto a fresh
|
|
@@ -1072,7 +1160,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1072
1160
|
*/
|
|
1073
1161
|
releaseReadTxn(): void {
|
|
1074
1162
|
const transaction = this.detachOwnedTransaction();
|
|
1075
|
-
abortNativeTransaction(
|
|
1163
|
+
abortNativeTransaction(
|
|
1164
|
+
transaction,
|
|
1165
|
+
'releasing timed-out read transaction',
|
|
1166
|
+
this.writes[0]?.store ?? this.db,
|
|
1167
|
+
this.startedFrom
|
|
1168
|
+
);
|
|
1076
1169
|
this.completeDeferredContextRelease();
|
|
1077
1170
|
}
|
|
1078
1171
|
|
|
@@ -1108,7 +1201,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1108
1201
|
if (operation.key === undefined) return;
|
|
1109
1202
|
let writesForStore = (this.writesByKey ??= new Map()).get(operation.store);
|
|
1110
1203
|
if (!writesForStore) this.writesByKey.set(operation.store, (writesForStore = new Map()));
|
|
1111
|
-
const keyId = writeKeyId(operation.key);
|
|
1204
|
+
const keyId = (operation.keyId = writeKeyId(operation.key));
|
|
1112
1205
|
const priorWrite = writesForStore.get(keyId);
|
|
1113
1206
|
if (priorWrite) operation.priorWrite = priorWrite;
|
|
1114
1207
|
writesForStore.set(keyId, operation);
|
|
@@ -1122,6 +1215,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1122
1215
|
const index = this.writes.indexOf(operation);
|
|
1123
1216
|
if (index > -1) this.writes[index] = null;
|
|
1124
1217
|
this.ownedWrites?.delete(operation);
|
|
1218
|
+
if (operation.stagedIn === this) operation.stagedIn = undefined;
|
|
1125
1219
|
if (operation.key === undefined) return;
|
|
1126
1220
|
const writesForStore = this.writesByKey?.get(operation.store);
|
|
1127
1221
|
if (!writesForStore) return;
|
|
@@ -1200,11 +1294,330 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1200
1294
|
if (recordLocks.size === 0) this.recordLocks = undefined;
|
|
1201
1295
|
}
|
|
1202
1296
|
|
|
1203
|
-
private
|
|
1297
|
+
private budgetCarrier(transaction?: RocksTransaction | null): {
|
|
1298
|
+
commitStartedAt?: number;
|
|
1299
|
+
commitOverdueLogged?: boolean;
|
|
1300
|
+
} {
|
|
1301
|
+
const txn = transaction as RocksTransactionWithRetry | null | undefined;
|
|
1302
|
+
if (txn?.commitStartedAt != null) return txn;
|
|
1303
|
+
if (this.saveCommits && txn && this.hasSiblingCommitInFlight(transaction ?? undefined)) {
|
|
1304
|
+
return txn;
|
|
1305
|
+
}
|
|
1306
|
+
if (txn?.immediateCommit) return txn;
|
|
1307
|
+
return this.saveCommits ? this : (this.root ?? this);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
private backfillActiveHandleWrites(handle: RocksTransaction): void {
|
|
1311
|
+
if ((handle as RocksTransactionWithRetry).immediateCommit) return;
|
|
1312
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1313
|
+
const w = this.writes[i];
|
|
1314
|
+
if (w?.saved && !w.stagedOn) w.stagedOn = handle;
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
private isSingleHandleTarget(txn: RocksTransaction): boolean {
|
|
1319
|
+
return (
|
|
1320
|
+
!(txn as RocksTransactionWithRetry).immediateCommit &&
|
|
1321
|
+
(!this.activeCommitHandles || this.activeCommitHandles.size === 0) &&
|
|
1322
|
+
(!this.activeCommitHandle || this.activeCommitHandle === txn)
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
private getHandleStagedTxnTime(txn?: RocksTransaction | null): number | undefined {
|
|
1327
|
+
if (!txn) return undefined;
|
|
1328
|
+
if (
|
|
1329
|
+
this.isSingleHandleTarget(txn) &&
|
|
1330
|
+
(this.activeCommitHandle === txn || (!this.activeCommitHandle && (!this.transaction || this.transaction === txn)))
|
|
1331
|
+
) {
|
|
1332
|
+
return this.activeHandleStagedTxnTime || (txn as RocksTransactionWithRetry).stagedTxnTime;
|
|
1333
|
+
}
|
|
1334
|
+
return (txn as RocksTransactionWithRetry).stagedTxnTime;
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
private setHandleStagedTxnTime(txn: RocksTransaction, txnTime: number): void {
|
|
1338
|
+
if (
|
|
1339
|
+
this.isSingleHandleTarget(txn) &&
|
|
1340
|
+
(this.activeCommitHandle === txn || (!this.activeCommitHandle && (!this.transaction || this.transaction === txn)))
|
|
1341
|
+
) {
|
|
1342
|
+
if (!this.activeHandleStagedTxnTime) this.activeHandleStagedTxnTime = txnTime;
|
|
1343
|
+
return;
|
|
1344
|
+
}
|
|
1345
|
+
(txn as RocksTransactionWithRetry).stagedTxnTime ??= txnTime;
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
private isHandleCommitInFlight(txn?: RocksTransaction | null): boolean {
|
|
1349
|
+
if (!txn) return false;
|
|
1350
|
+
if (this.activeCommitHandle === txn) {
|
|
1351
|
+
return Boolean(this.activeHandleInFlight || (txn as RocksTransactionWithRetry).commitInFlight);
|
|
1352
|
+
}
|
|
1353
|
+
return Boolean((txn as RocksTransactionWithRetry).commitInFlight);
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
private setHandleCommitInFlight(txn: RocksTransaction, inFlight: boolean): void {
|
|
1357
|
+
if (this.isSingleHandleTarget(txn) && inFlight && !this.activeCommitHandle) {
|
|
1358
|
+
this.activeCommitHandle = txn;
|
|
1359
|
+
}
|
|
1360
|
+
if (this.activeCommitHandle === txn) {
|
|
1361
|
+
this.activeHandleInFlight = inFlight;
|
|
1362
|
+
if (!inFlight) {
|
|
1363
|
+
(txn as RocksTransactionWithRetry).commitInFlight = false;
|
|
1364
|
+
}
|
|
1365
|
+
return;
|
|
1366
|
+
}
|
|
1367
|
+
(txn as RocksTransactionWithRetry).commitInFlight = inFlight;
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
private isHandleCommitted(txn?: RocksTransaction | null): boolean {
|
|
1371
|
+
if (!txn) return false;
|
|
1372
|
+
if (this.activeCommitHandle === txn) {
|
|
1373
|
+
return Boolean(this.activeHandleCommitted || (txn as RocksTransactionWithRetry).committed);
|
|
1374
|
+
}
|
|
1375
|
+
return Boolean((txn as RocksTransactionWithRetry).committed);
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
private setHandleCommitted(txn: RocksTransaction): void {
|
|
1379
|
+
if (this.activeCommitHandle === txn) {
|
|
1380
|
+
this.activeHandleInFlight = false;
|
|
1381
|
+
this.activeHandleCommitted = true;
|
|
1382
|
+
(txn as RocksTransactionWithRetry).commitInFlight = false;
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
(txn as RocksTransactionWithRetry).commitInFlight = false;
|
|
1386
|
+
(txn as RocksTransactionWithRetry).committed = true;
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
private trackActiveCommitHandle(transaction: RocksTransaction): void {
|
|
1390
|
+
if (
|
|
1391
|
+
!(transaction as RocksTransactionWithRetry).immediateCommit &&
|
|
1392
|
+
!this.activeCommitHandle &&
|
|
1393
|
+
(!this.activeCommitHandles || this.activeCommitHandles.size === 0)
|
|
1394
|
+
) {
|
|
1395
|
+
this.activeCommitHandle = transaction;
|
|
1396
|
+
return;
|
|
1397
|
+
}
|
|
1398
|
+
if (this.activeCommitHandle === transaction) return;
|
|
1399
|
+
let handles = this.activeCommitHandles;
|
|
1400
|
+
if (!handles) {
|
|
1401
|
+
handles = this.activeCommitHandles = new Set();
|
|
1402
|
+
}
|
|
1403
|
+
if (this.activeCommitHandle) {
|
|
1404
|
+
const first = this.activeCommitHandle as RocksTransactionWithRetry;
|
|
1405
|
+
if (this.activeHandleStagedTxnTime && !first.stagedTxnTime) {
|
|
1406
|
+
first.stagedTxnTime = this.activeHandleStagedTxnTime;
|
|
1407
|
+
}
|
|
1408
|
+
if (this.activeHandleInFlight) first.commitInFlight = true;
|
|
1409
|
+
if (this.activeHandleCommitted) first.committed = true;
|
|
1410
|
+
this.activeHandleStagedTxnTime = 0;
|
|
1411
|
+
this.activeHandleInFlight = false;
|
|
1412
|
+
this.activeHandleCommitted = false;
|
|
1413
|
+
this.backfillActiveHandleWrites(first);
|
|
1414
|
+
if (this.saveCommits) {
|
|
1415
|
+
if (this.commitStartedAt != null && first.commitStartedAt == null) {
|
|
1416
|
+
first.commitStartedAt = this.commitStartedAt;
|
|
1417
|
+
this.commitStartedAt = undefined;
|
|
1418
|
+
}
|
|
1419
|
+
if (this.commitOverdueLogged) {
|
|
1420
|
+
first.commitOverdueLogged = true;
|
|
1421
|
+
this.commitOverdueLogged = undefined;
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
handles.add(this.activeCommitHandle);
|
|
1425
|
+
this.activeCommitHandle = undefined;
|
|
1426
|
+
}
|
|
1427
|
+
handles.add(transaction);
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
private releaseActiveCommitHandle(transaction: RocksTransaction): void {
|
|
1431
|
+
if (this.activeCommitHandle === transaction) {
|
|
1432
|
+
this.activeCommitHandle = undefined;
|
|
1433
|
+
this.activeHandleStagedTxnTime = 0;
|
|
1434
|
+
this.activeHandleInFlight = false;
|
|
1435
|
+
this.activeHandleCommitted = false;
|
|
1436
|
+
return;
|
|
1437
|
+
}
|
|
1438
|
+
const handles = this.activeCommitHandles;
|
|
1439
|
+
if (!handles) return;
|
|
1440
|
+
handles.delete(transaction);
|
|
1441
|
+
if (handles.size === 1) {
|
|
1442
|
+
for (const remaining of handles) {
|
|
1443
|
+
const rem = remaining as RocksTransactionWithRetry;
|
|
1444
|
+
if (!rem.immediateCommit) {
|
|
1445
|
+
this.activeCommitHandle = remaining;
|
|
1446
|
+
this.activeHandleStagedTxnTime = rem.stagedTxnTime ?? 0;
|
|
1447
|
+
this.activeHandleInFlight = Boolean(rem.commitInFlight);
|
|
1448
|
+
this.activeHandleCommitted = Boolean(rem.committed);
|
|
1449
|
+
handles.clear();
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
private clearActiveCommitHandles(): void {
|
|
1456
|
+
if (this.activeCommitHandle !== undefined || this.activeHandleStagedTxnTime) {
|
|
1457
|
+
this.activeCommitHandle = undefined;
|
|
1458
|
+
this.activeHandleStagedTxnTime = 0;
|
|
1459
|
+
this.activeHandleInFlight = false;
|
|
1460
|
+
this.activeHandleCommitted = false;
|
|
1461
|
+
}
|
|
1462
|
+
this.activeCommitHandles?.clear();
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
private hasActiveCommitHandle(): boolean {
|
|
1466
|
+
return (
|
|
1467
|
+
this.activeCommitHandle !== undefined ||
|
|
1468
|
+
(this.activeCommitHandles !== undefined && this.activeCommitHandles.size > 0)
|
|
1469
|
+
);
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
/**
|
|
1473
|
+
* Retries so far of the handles with a durable transaction-log entry still retrying their commit on
|
|
1474
|
+
* this link, summed over overlapping commits so the total moves whenever any of them does, or -1
|
|
1475
|
+
* when there is none.
|
|
1476
|
+
*/
|
|
1477
|
+
activeDurableLogRetries(): number {
|
|
1478
|
+
let retries = -1;
|
|
1479
|
+
const note = (handle: RocksTransaction | null | undefined) => {
|
|
1480
|
+
const txn = handle as RocksTransactionWithRetry | null | undefined;
|
|
1481
|
+
if (!txn || !hasDurableLog(txn) || this.isHandleCommitted(txn) || txn.abortedByWrapper) return;
|
|
1482
|
+
retries = Math.max(retries, 0) + (txn.retries ?? 0);
|
|
1483
|
+
};
|
|
1484
|
+
note(this.activeCommitHandle);
|
|
1485
|
+
if (this.activeCommitHandles) for (const handle of this.activeCommitHandles) note(handle);
|
|
1486
|
+
note(this.transaction);
|
|
1487
|
+
return retries;
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
private isActiveCommitHandle(transaction: RocksTransaction): boolean {
|
|
1491
|
+
return this.activeCommitHandle === transaction || Boolean(this.activeCommitHandles?.has(transaction));
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
private hasSiblingCommitInFlight(targetTransaction?: RocksTransaction): boolean {
|
|
1495
|
+
const excluded = targetTransaction ?? this.transaction;
|
|
1496
|
+
if (this.activeCommitHandle) return this.activeCommitHandle !== excluded;
|
|
1497
|
+
const active = this.activeCommitHandles;
|
|
1498
|
+
if (!active || active.size === 0) return false;
|
|
1499
|
+
if (!excluded) return true;
|
|
1500
|
+
return active.size > (active.has(excluded) ? 1 : 0);
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
private isWriteOwnedByActiveCommit(
|
|
1504
|
+
write: TransactionWrite | undefined,
|
|
1505
|
+
targetTransaction: RocksTransaction | undefined,
|
|
1506
|
+
hasSiblingInFlight: boolean,
|
|
1507
|
+
fromAbort = false,
|
|
1508
|
+
excludeImmediate = false
|
|
1509
|
+
): boolean {
|
|
1510
|
+
if (!write) return false;
|
|
1511
|
+
if ((targetTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit) {
|
|
1512
|
+
return write.stagedOn !== targetTransaction;
|
|
1513
|
+
}
|
|
1514
|
+
if (write.deferSave && !write.saved && this.saveCommits)
|
|
1515
|
+
return (
|
|
1516
|
+
(!fromAbort || Boolean(targetTransaction) || Boolean((this as { isCommitting?: boolean }).isCommitting)) &&
|
|
1517
|
+
!write.stagedOn &&
|
|
1518
|
+
!write.lockHandle?.released &&
|
|
1519
|
+
!write.lockHandle?.isExpired()
|
|
1520
|
+
);
|
|
1521
|
+
if (!write.saved)
|
|
1522
|
+
return !fromAbort && Boolean(targetTransaction) && this.open === TRANSACTION_STATE.OPEN && !write.stagedOn;
|
|
1523
|
+
const stagedOn = (write.stagedOn ?? this.activeCommitHandle ?? this.transaction) as
|
|
1524
|
+
RocksTransactionWithRetry | undefined;
|
|
1525
|
+
if (!stagedOn || stagedOn === targetTransaction || stagedOn.abortedByWrapper || this.isHandleCommitted(stagedOn))
|
|
1526
|
+
return false;
|
|
1527
|
+
if (excludeImmediate && stagedOn.immediateCommit) return false;
|
|
1528
|
+
if (
|
|
1529
|
+
!fromAbort &&
|
|
1530
|
+
Boolean(targetTransaction) &&
|
|
1531
|
+
this.open === TRANSACTION_STATE.OPEN &&
|
|
1532
|
+
stagedOn === this.transaction
|
|
1533
|
+
)
|
|
1534
|
+
return true;
|
|
1535
|
+
if (
|
|
1536
|
+
hasSiblingInFlight &&
|
|
1537
|
+
(Boolean(targetTransaction) ||
|
|
1538
|
+
(!fromAbort && this.isActiveCommitHandle(stagedOn)) ||
|
|
1539
|
+
(fromAbort &&
|
|
1540
|
+
this.isActiveCommitHandle(stagedOn) &&
|
|
1541
|
+
((!excludeImmediate && Boolean(stagedOn.immediateCommit)) || Boolean(stagedOn.isRetry))))
|
|
1542
|
+
)
|
|
1543
|
+
return true;
|
|
1544
|
+
return Boolean(this.isHandleCommitInFlight(stagedOn) || hasDurableLog(stagedOn));
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
private hasWriteOwnedByActiveCommit(
|
|
1548
|
+
targetTransaction: RocksTransaction | undefined,
|
|
1549
|
+
hasSiblingInFlight: boolean,
|
|
1550
|
+
fromAbort = false,
|
|
1551
|
+
excludeImmediate = false
|
|
1552
|
+
): boolean {
|
|
1553
|
+
if (!this.stagesWriteOnSave) return false;
|
|
1554
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1555
|
+
if (
|
|
1556
|
+
this.isWriteOwnedByActiveCommit(
|
|
1557
|
+
this.writes[i],
|
|
1558
|
+
targetTransaction,
|
|
1559
|
+
hasSiblingInFlight,
|
|
1560
|
+
fromAbort,
|
|
1561
|
+
excludeImmediate
|
|
1562
|
+
)
|
|
1563
|
+
)
|
|
1564
|
+
return true;
|
|
1565
|
+
}
|
|
1566
|
+
return false;
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
private hasSavedWrite(): boolean {
|
|
1570
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1571
|
+
if (this.writes[i]?.saved) return true;
|
|
1572
|
+
}
|
|
1573
|
+
return false;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
private hasSavedWriteForHandle(transaction: RocksTransaction, isImmediateCommit: boolean): boolean {
|
|
1577
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1578
|
+
const write = this.writes[i];
|
|
1579
|
+
if (!write?.saved) continue;
|
|
1580
|
+
if (isImmediateCommit ? write.stagedOn === transaction : !write.stagedOn || write.stagedOn === transaction) {
|
|
1581
|
+
return true;
|
|
1582
|
+
}
|
|
1583
|
+
}
|
|
1584
|
+
return false;
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
private retainTransactionLockHolds(transaction: RocksTransaction | null | undefined): void {
|
|
1588
|
+
if (!transaction || !this.hasLeaseProtectedWrite) return;
|
|
1589
|
+
const txn = transaction as RocksTransactionWithRetry;
|
|
1590
|
+
if (txn.lockHolds || txn.holdBudgetSpent) return;
|
|
1591
|
+
const carrier = this.budgetCarrier(transaction);
|
|
1592
|
+
if (carrier.commitStartedAt == null) carrier.commitStartedAt = performance.now();
|
|
1593
|
+
const budget = this.commitConflictBudget();
|
|
1594
|
+
const holdDeadlineMono = Number.isFinite(budget) ? carrier.commitStartedAt + budget : undefined;
|
|
1595
|
+
let holds: Set<RecordLockHandle> | undefined;
|
|
1596
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1597
|
+
const write = this.writes[i];
|
|
1598
|
+
if (!write?.saved || (write.stagedOn && write.stagedOn !== transaction)) continue;
|
|
1599
|
+
const handle = write.lockHandle;
|
|
1600
|
+
if (!handle?.retainForCommit || (holds && holds.has(handle))) continue;
|
|
1601
|
+
handle.retainForCommit(holdDeadlineMono);
|
|
1602
|
+
(holds ??= new Set()).add(handle);
|
|
1603
|
+
}
|
|
1604
|
+
if (holds) txn.lockHolds = holds;
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
private noteCommittedLockVersions(committedTransaction?: RocksTransaction): void {
|
|
1204
1608
|
const recordLocks = this.recordLocks;
|
|
1205
1609
|
if (!recordLocks) return;
|
|
1610
|
+
const isImmediateCommit = Boolean((committedTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1611
|
+
const filterByStagedOn = Boolean(committedTransaction || this.hasSiblingCommitInFlight(committedTransaction));
|
|
1206
1612
|
for (const write of this.writes) {
|
|
1207
|
-
if (write?.
|
|
1613
|
+
if (!write?.saved) continue;
|
|
1614
|
+
if (
|
|
1615
|
+
isImmediateCommit
|
|
1616
|
+
? write.stagedOn !== committedTransaction
|
|
1617
|
+
: filterByStagedOn && write.stagedOn && write.stagedOn !== committedTransaction
|
|
1618
|
+
)
|
|
1619
|
+
continue;
|
|
1620
|
+
if (write.appliedRecordVersion == null) continue;
|
|
1208
1621
|
const handle = write.lockHandle ?? recordLocks.get(write.store)?.get(writeKeyId(write.key));
|
|
1209
1622
|
if (handle && !handle.released) handle.noteHolderVersion(write.appliedRecordVersion);
|
|
1210
1623
|
}
|
|
@@ -1212,9 +1625,33 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1212
1625
|
|
|
1213
1626
|
/**
|
|
1214
1627
|
* Discard the staged write set (committed or aborted); the per-key chain must go with it so a
|
|
1215
|
-
* reused transaction never bases a write on a previous batch's staged state.
|
|
1628
|
+
* reused transaction never bases a write on a previous batch's staged state. When concurrent
|
|
1629
|
+
* commits share one DatabaseTransaction, only the writes staged on `committedTransaction` are
|
|
1630
|
+
* removed while another native commit is still in flight.
|
|
1216
1631
|
*/
|
|
1217
|
-
clearWrites(): void {
|
|
1632
|
+
clearWrites(committedTransaction?: RocksTransaction, fromAbort = false): void {
|
|
1633
|
+
if (committedTransaction) this.releaseActiveCommitHandle(committedTransaction);
|
|
1634
|
+
const isImmediateCommit = Boolean((committedTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1635
|
+
const hasSiblingInFlight = this.hasSiblingCommitInFlight(committedTransaction);
|
|
1636
|
+
if (
|
|
1637
|
+
this.stagesWriteOnSave &&
|
|
1638
|
+
(isImmediateCommit ||
|
|
1639
|
+
((hasSiblingInFlight || !committedTransaction || this.commitsInFlight > 1 || this.saveCommits) &&
|
|
1640
|
+
this.hasWriteOwnedByActiveCommit(committedTransaction, hasSiblingInFlight, fromAbort)))
|
|
1641
|
+
) {
|
|
1642
|
+
for (const write of [...this.writes]) {
|
|
1643
|
+
if (!write || this.isWriteOwnedByActiveCommit(write, committedTransaction, hasSiblingInFlight, fromAbort))
|
|
1644
|
+
continue;
|
|
1645
|
+
this.detachWrite(write);
|
|
1646
|
+
}
|
|
1647
|
+
this.writes = this.writes.filter((write) => write);
|
|
1648
|
+
this.validated = this.writes.reduce((acc, w, idx) => (w?.saved ? idx + 1 : acc), 0);
|
|
1649
|
+
if (this.hasLeaseProtectedWrite && !this.writes.some((write) => write?.lockHandle)) {
|
|
1650
|
+
this.hasLeaseProtectedWrite = false;
|
|
1651
|
+
}
|
|
1652
|
+
return;
|
|
1653
|
+
}
|
|
1654
|
+
this.clearActiveCommitHandles();
|
|
1218
1655
|
// A deferred write's `stagedIn` must not outlive this transaction's ability to commit it: save() can
|
|
1219
1656
|
// fire after a commit or abort, and routing it back here would revive a write this transaction
|
|
1220
1657
|
// already rolled back — whose blobs abort() has reclaimed. Cleared here, save() resolves the
|
|
@@ -1224,6 +1661,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1224
1661
|
if (write) this.ownedWrites?.delete(write);
|
|
1225
1662
|
}
|
|
1226
1663
|
this.writes = [];
|
|
1664
|
+
this.validated = 0;
|
|
1227
1665
|
this.writesByKey = undefined;
|
|
1228
1666
|
// The commit-fence latch belongs to the discarded batch. A reused transaction that once staged
|
|
1229
1667
|
// a locked write would otherwise re-scan every write of every later unlocked batch. Assigned
|
|
@@ -1315,12 +1753,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1315
1753
|
* no record), and writes with no stored entry at all — a delete leaves a readable tombstone on any
|
|
1316
1754
|
* audited or delete-tracking table, so only a delete on a table with neither is entry-less. Such a
|
|
1317
1755
|
* `put(A); delete(B)` confirms on A's entry, whose version is this transaction's (every write in a
|
|
1318
|
-
* transaction is stamped with one version).
|
|
1756
|
+
* transaction is stamped with one version). `savedOnly` also skips a deferred write not yet saved,
|
|
1757
|
+
* which a commit leaves in place for a later one.
|
|
1319
1758
|
*/
|
|
1320
|
-
lastConfirmableEntry(): Partial<Entry> | undefined {
|
|
1759
|
+
lastConfirmableEntry(savedOnly = false): Partial<Entry> | undefined {
|
|
1321
1760
|
for (let i = this.writes.length - 1; i >= 0; i--) {
|
|
1322
1761
|
const write = this.writes[i];
|
|
1323
|
-
if (!write || write.skipReplicationConfirmation) continue;
|
|
1762
|
+
if (!write || write.skipReplicationConfirmation || (savedOnly && !write.saved)) continue;
|
|
1324
1763
|
const entry = write.store.getEntry(write.key);
|
|
1325
1764
|
if (entry) return entry;
|
|
1326
1765
|
}
|
|
@@ -1332,9 +1771,61 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1332
1771
|
* rejection in that window is an unhandled rejection (fatal under --unhandled-rejections=strict).
|
|
1333
1772
|
* The rejection still surfaces through commit()'s Promise.all.
|
|
1334
1773
|
*/
|
|
1335
|
-
stageCompletion(completion: Promise<void
|
|
1774
|
+
stageCompletion(completion: Promise<void>, transaction?: RocksTransaction) {
|
|
1336
1775
|
completion.then(undefined, () => {});
|
|
1776
|
+
const index = this.completions.length;
|
|
1337
1777
|
this.completions.push(completion);
|
|
1778
|
+
if (
|
|
1779
|
+
transaction &&
|
|
1780
|
+
(this.saveCommits ||
|
|
1781
|
+
Boolean((transaction as RocksTransactionWithRetry).immediateCommit) ||
|
|
1782
|
+
this.hasActiveCommitHandle() ||
|
|
1783
|
+
Boolean(this.completionHandles))
|
|
1784
|
+
) {
|
|
1785
|
+
(this.completionHandles ??= [])[index] = transaction;
|
|
1786
|
+
}
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
private takeCompletions(transaction?: RocksTransaction): Promise<void>[] {
|
|
1790
|
+
if (this.completions.length === 0) return NO_COMPLETIONS;
|
|
1791
|
+
const handles = this.completionHandles;
|
|
1792
|
+
const isImmediate = Boolean((transaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1793
|
+
if (!handles) {
|
|
1794
|
+
if (isImmediate) return NO_COMPLETIONS;
|
|
1795
|
+
const completions = this.completions;
|
|
1796
|
+
this.completions = [];
|
|
1797
|
+
return completions;
|
|
1798
|
+
}
|
|
1799
|
+
let allMatch = true;
|
|
1800
|
+
for (let i = 0; i < this.completions.length; i++) {
|
|
1801
|
+
const handle = handles[i];
|
|
1802
|
+
if (!(handle ? handle === transaction : !isImmediate)) {
|
|
1803
|
+
allMatch = false;
|
|
1804
|
+
break;
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
1807
|
+
if (allMatch) {
|
|
1808
|
+
const completions = this.completions;
|
|
1809
|
+
this.completions = [];
|
|
1810
|
+
this.completionHandles = undefined;
|
|
1811
|
+
return completions;
|
|
1812
|
+
}
|
|
1813
|
+
const taken: Promise<void>[] = [];
|
|
1814
|
+
const remaining: Promise<void>[] = [];
|
|
1815
|
+
const remainingHandles: (RocksTransaction | undefined)[] = [];
|
|
1816
|
+
for (let i = 0; i < this.completions.length; i++) {
|
|
1817
|
+
const completion = this.completions[i];
|
|
1818
|
+
const handle = handles[i];
|
|
1819
|
+
if (handle ? handle === transaction : !isImmediate) {
|
|
1820
|
+
taken.push(completion);
|
|
1821
|
+
} else {
|
|
1822
|
+
remaining.push(completion);
|
|
1823
|
+
remainingHandles.push(handle);
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
this.completions = remaining;
|
|
1827
|
+
this.completionHandles = remainingHandles.length > 0 ? remainingHandles : undefined;
|
|
1828
|
+
return taken;
|
|
1338
1829
|
}
|
|
1339
1830
|
|
|
1340
1831
|
/**
|
|
@@ -1343,11 +1834,38 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1343
1834
|
* Clearing them also keeps a reused transaction's next commit() from rejecting with the previous
|
|
1344
1835
|
* batch's error.
|
|
1345
1836
|
*/
|
|
1346
|
-
drainCompletions(): void {
|
|
1837
|
+
drainCompletions(targetTransaction?: RocksTransaction, preserveActiveWrites = false): void {
|
|
1347
1838
|
if (this.completions.length === 0) return;
|
|
1348
|
-
const
|
|
1349
|
-
|
|
1350
|
-
|
|
1839
|
+
const handles = this.completionHandles;
|
|
1840
|
+
const isImmediateTarget = Boolean((targetTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1841
|
+
let drained: Promise<void>[];
|
|
1842
|
+
if (!preserveActiveWrites) {
|
|
1843
|
+
drained = this.completions;
|
|
1844
|
+
this.completions = [];
|
|
1845
|
+
this.completionHandles = undefined;
|
|
1846
|
+
} else {
|
|
1847
|
+
drained = [];
|
|
1848
|
+
const remaining: Promise<void>[] = [];
|
|
1849
|
+
const remainingHandles: (RocksTransaction | undefined)[] = [];
|
|
1850
|
+
for (let i = 0; i < this.completions.length; i++) {
|
|
1851
|
+
const completion = this.completions[i];
|
|
1852
|
+
const handle = handles?.[i];
|
|
1853
|
+
const keep = isImmediateTarget
|
|
1854
|
+
? handle !== targetTransaction
|
|
1855
|
+
: Boolean(handle) &&
|
|
1856
|
+
handle !== targetTransaction &&
|
|
1857
|
+
(this.isActiveCommitHandle(handle) || hasDurableLog(handle));
|
|
1858
|
+
if (keep) {
|
|
1859
|
+
remaining.push(completion);
|
|
1860
|
+
remainingHandles.push(handle);
|
|
1861
|
+
} else {
|
|
1862
|
+
drained.push(completion);
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
this.completions = remaining;
|
|
1866
|
+
this.completionHandles = remainingHandles.length > 0 ? remainingHandles : undefined;
|
|
1867
|
+
}
|
|
1868
|
+
for (const completion of drained)
|
|
1351
1869
|
completion.then(undefined, (error) =>
|
|
1352
1870
|
harperLogger.warn?.('A staged transaction completion failed after the transaction was aborted', error)
|
|
1353
1871
|
);
|
|
@@ -1419,7 +1937,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1419
1937
|
// (the same rule the pre-submit fence applies below). Without the distinction,
|
|
1420
1938
|
// `lock(); save(); unlock(); commit()` succeeded normally and threw 409 only when a conflict
|
|
1421
1939
|
// retry or an open read iterator forced the replay to re-save it.
|
|
1422
|
-
if (operation.saved ? lockHandle?.isLeaseExpired() : lockHandle?.isExpired()) {
|
|
1940
|
+
if (operation.saved ? lockHandle?.isLeaseExpired() && !hasDurableLog(transaction) : lockHandle?.isExpired()) {
|
|
1423
1941
|
// Through `detachWrite`, which also repairs the per-key chain: dropping it from `writes` alone
|
|
1424
1942
|
// leaves `writesByKey` pointing at a write that was refused, so a caller that catches this 409
|
|
1425
1943
|
// and stages the same key again would take the rejected operation as its merge basis.
|
|
@@ -1442,7 +1960,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1442
1960
|
if (!operation.lockStamp) operation.lockStamp = lockHandle.holderVersionCandidate();
|
|
1443
1961
|
}
|
|
1444
1962
|
}
|
|
1445
|
-
|
|
1963
|
+
const concurrentClosedWrite =
|
|
1964
|
+
Boolean((transaction as RocksTransactionWithRetry | undefined)?.immediateCommit) ||
|
|
1965
|
+
(!transaction && this.open !== TRANSACTION_STATE.OPEN && !this.saveCommits);
|
|
1966
|
+
let txnTime =
|
|
1967
|
+
operation.lockStamp ?? this.getHandleStagedTxnTime(transaction) ?? (concurrentClosedWrite ? 0 : this.timestamp);
|
|
1968
|
+
const mintedTimestamp = !txnTime;
|
|
1446
1969
|
// Only an OPEN transaction accepts new staged writes. After commit, this.transaction may still
|
|
1447
1970
|
// be retained for outstanding read iterators; staging into it would silently discard the write
|
|
1448
1971
|
// when doneReadTxn() aborts the handle, so such writes commit immediately on a fresh
|
|
@@ -1474,19 +1997,39 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1474
1997
|
} else {
|
|
1475
1998
|
// if it is closed, we have to immediately commit, using our immediate transaction
|
|
1476
1999
|
immediateCommit = true;
|
|
2000
|
+
if (!this.saveCommits) (transaction as RocksTransactionWithRetry).immediateCommit = true;
|
|
1477
2001
|
}
|
|
1478
2002
|
if (txnTime) {
|
|
1479
2003
|
transaction.setTimestamp(txnTime);
|
|
1480
2004
|
}
|
|
1481
2005
|
}
|
|
2006
|
+
if (this.activeCommitHandle && this.activeCommitHandle !== transaction) {
|
|
2007
|
+
this.backfillActiveHandleWrites(this.activeCommitHandle);
|
|
2008
|
+
} else if (!this.activeCommitHandle && this.transaction && this.transaction !== transaction) {
|
|
2009
|
+
this.backfillActiveHandleWrites(this.transaction);
|
|
2010
|
+
}
|
|
2011
|
+
const shouldStampStagedOn =
|
|
2012
|
+
this.saveCommits ||
|
|
2013
|
+
immediateCommit ||
|
|
2014
|
+
Boolean((transaction as RocksTransactionWithRetry | undefined)?.immediateCommit) ||
|
|
2015
|
+
operation.stagedOn !== undefined ||
|
|
2016
|
+
(this.activeCommitHandles !== undefined && this.activeCommitHandles.size > 0) ||
|
|
2017
|
+
(this.activeCommitHandle !== undefined && this.activeCommitHandle !== transaction) ||
|
|
2018
|
+
(this.transaction != null && this.transaction !== transaction);
|
|
1482
2019
|
if (this.isReplay) {
|
|
1483
2020
|
// Replayed writes came FROM the transaction log; never re-append them —
|
|
1484
2021
|
// replay iterates that same log, so re-appending prevents convergence
|
|
1485
2022
|
// (boot hangs replaying its own output). Conflict retries stamp isRetry
|
|
1486
2023
|
// at the retry sites in commit(); this is the replay-path equivalent.
|
|
1487
2024
|
(transaction as RocksTransactionWithRetry).isRetry = true;
|
|
2025
|
+
(transaction as RocksTransactionWithRetry).retries = this.retries;
|
|
1488
2026
|
}
|
|
1489
|
-
if (!txnTime)
|
|
2027
|
+
if (!txnTime) {
|
|
2028
|
+
txnTime = transaction.getTimestamp();
|
|
2029
|
+
if (!concurrentClosedWrite) this.timestamp = txnTime;
|
|
2030
|
+
}
|
|
2031
|
+
if (!operation.lockStamp) this.setHandleStagedTxnTime(transaction, txnTime);
|
|
2032
|
+
if (this.saveCommits && mintedTimestamp) this.timestamp = 0;
|
|
1490
2033
|
if (!operation.saved && operation.pendingPriorWrite) {
|
|
1491
2034
|
const pendingWrites = [];
|
|
1492
2035
|
for (let pending = operation.pendingPriorWrite; pending;) {
|
|
@@ -1514,45 +2057,84 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1514
2057
|
// An entry read uncached through this same pinned-snapshot handle is already that base, until a retry resets
|
|
1515
2058
|
// the snapshot or an earlier staged write to the key changes what a read returns; the resource withholds the
|
|
1516
2059
|
// handle once its entry was replaced or evicted, and locked, snapshot-free and replayed writes always reload.
|
|
2060
|
+
const txnRetries = (transaction as RocksTransactionWithRetry | undefined)?.retries ?? 0;
|
|
1517
2061
|
const reusesBaseRead =
|
|
1518
2062
|
operation.baseReadTxn === transaction &&
|
|
1519
|
-
|
|
2063
|
+
txnRetries === 0 &&
|
|
1520
2064
|
!reloadEntry &&
|
|
1521
2065
|
!operation.priorWrite &&
|
|
1522
2066
|
!operation.lockHandle &&
|
|
1523
2067
|
!this.snapshotFree &&
|
|
1524
2068
|
!(transaction as any).snapshotDisabled &&
|
|
1525
2069
|
!this.isReplay;
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
if (
|
|
2070
|
+
const wasSaved = Boolean(operation.saved);
|
|
2071
|
+
let completion: Promise<void>;
|
|
2072
|
+
try {
|
|
2073
|
+
if (!reusesBaseRead && (reloadEntry || operation.entry === undefined || reloadsCommitBase)) {
|
|
2074
|
+
const uncachedRead = !!operation.reloadCommitBase || reloadEntry || this.isReplay;
|
|
2075
|
+
operation.entry = operation.store.getEntry(operation.key, { transaction, uncachedRead });
|
|
2076
|
+
}
|
|
2077
|
+
if (!operation.saved) {
|
|
2078
|
+
// immediately execute in this transaction
|
|
2079
|
+
const validated = validateWrite(operation, writeVersion, this);
|
|
2080
|
+
if ((validated as any) === false) {
|
|
2081
|
+
operation.saved = true;
|
|
2082
|
+
if (shouldStampStagedOn) operation.stagedOn = transaction;
|
|
2083
|
+
operation.commit = () => {}; // noop if we try again
|
|
2084
|
+
closeWriteInstance(operation);
|
|
2085
|
+
if (immediateCommit) {
|
|
2086
|
+
this.detachWrite(operation);
|
|
2087
|
+
if (!this.activeCommitHandle) this.activeHandleStagedTxnTime = 0;
|
|
2088
|
+
abortNativeTransaction(
|
|
2089
|
+
transaction,
|
|
2090
|
+
'aborting an immediate transaction with nothing to write',
|
|
2091
|
+
operation.store,
|
|
2092
|
+
this.startedFrom
|
|
2093
|
+
);
|
|
2094
|
+
}
|
|
2095
|
+
return;
|
|
2096
|
+
}
|
|
1534
2097
|
operation.saved = true;
|
|
1535
|
-
|
|
2098
|
+
let result: Promise<void> = operation.before?.() as Promise<void>;
|
|
2099
|
+
if (result?.then) this.stageCompletion(result, transaction);
|
|
2100
|
+
result = operation.beforeIntermediate?.() as Promise<void>;
|
|
2101
|
+
if (result?.then) this.stageCompletion(result, transaction);
|
|
2102
|
+
}
|
|
2103
|
+
if (shouldStampStagedOn) operation.stagedOn = transaction;
|
|
2104
|
+
if (lockHandle || this.recordLocks) operation.trackRecordVersion = true;
|
|
2105
|
+
if (operation.trackRecordVersion) operation.recordVersionApplied = false;
|
|
2106
|
+
operation.committedOn = transaction;
|
|
2107
|
+
operation.committedRound = txnRetries;
|
|
2108
|
+
try {
|
|
2109
|
+
completion = operation.commit(
|
|
2110
|
+
writeVersion,
|
|
2111
|
+
operation.entry,
|
|
2112
|
+
(wasSaved || this.isReplay) && txnRetries > 0,
|
|
2113
|
+
transaction,
|
|
2114
|
+
this
|
|
2115
|
+
) as Promise<void>;
|
|
2116
|
+
} finally {
|
|
1536
2117
|
closeWriteInstance(operation);
|
|
1537
|
-
return;
|
|
1538
2118
|
}
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
2119
|
+
} catch (error) {
|
|
2120
|
+
// A handle created for an immediate commit is owned by nobody else: a throw before its commit
|
|
2121
|
+
// is submitted must release it rather than leak it.
|
|
2122
|
+
if (immediateCommit) {
|
|
2123
|
+
this.drainCompletions(transaction, true);
|
|
2124
|
+
this.detachWrite(operation);
|
|
2125
|
+
if (!this.activeCommitHandle) this.activeHandleStagedTxnTime = 0;
|
|
2126
|
+
abortNativeTransaction(
|
|
2127
|
+
transaction,
|
|
2128
|
+
'aborting an immediate transaction whose write threw',
|
|
2129
|
+
operation.store,
|
|
2130
|
+
this.startedFrom
|
|
2131
|
+
);
|
|
2132
|
+
}
|
|
2133
|
+
throw error;
|
|
1552
2134
|
}
|
|
1553
2135
|
if (operation.trackRecordVersion)
|
|
1554
2136
|
operation.appliedRecordVersion = operation.recordVersionApplied ? writeVersion : undefined;
|
|
1555
|
-
if (typeof completion?.then === 'function') this.stageCompletion(completion);
|
|
2137
|
+
if (typeof completion?.then === 'function') this.stageCompletion(completion, transaction);
|
|
1556
2138
|
// Sticky record that THIS write staged with its audit entry appended (log entries batch on the
|
|
1557
2139
|
// native transaction and are durably written by its commit attempt — even a failed one — so
|
|
1558
2140
|
// they survive the abort-after-failed-commit of the retry paths). isRetry stagings
|
|
@@ -1561,6 +2143,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1561
2143
|
// retries where a recommit round self-skips before a fresh-transaction replay.
|
|
1562
2144
|
if (!operation.skipped && !(transaction as RocksTransactionWithRetry).isRetry) {
|
|
1563
2145
|
operation.appendedAuditEntry = true;
|
|
2146
|
+
operation.appendedTo = transaction;
|
|
1564
2147
|
}
|
|
1565
2148
|
if (immediateCommit) {
|
|
1566
2149
|
// immediately commit if the harper transaction is closed
|
|
@@ -1757,6 +2340,9 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1757
2340
|
private endCommitAttempt(): void {
|
|
1758
2341
|
this.commitsInFlight--;
|
|
1759
2342
|
if (!this.commitsInFlight) {
|
|
2343
|
+
this.clearActiveCommitHandles();
|
|
2344
|
+
this.commitStartedAt = undefined;
|
|
2345
|
+
if (this.commitOverdueLogged) this.commitOverdueLogged = undefined;
|
|
1760
2346
|
this.nativeCommitSubmitted = false;
|
|
1761
2347
|
this.nativeCommitAttemptEnded();
|
|
1762
2348
|
}
|
|
@@ -1774,7 +2360,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1774
2360
|
root.submittedCommitSupervised = false;
|
|
1775
2361
|
root.deferredPoisonDeadline = undefined;
|
|
1776
2362
|
root.unsubmittedCommitDeadline = undefined;
|
|
2363
|
+
root.loggedRetriesSeen = undefined;
|
|
2364
|
+
root.loggedRetriesChangedAt = undefined;
|
|
2365
|
+
root.loggedRetriesTick = undefined;
|
|
2366
|
+
root.loggedRetryAdvancing = undefined;
|
|
1777
2367
|
root.stalledCommitLogged = false;
|
|
2368
|
+
root.commitStartedAt = undefined;
|
|
2369
|
+
if (root.commitOverdueLogged) root.commitOverdueLogged = undefined;
|
|
1778
2370
|
const submittedLink = root.submittedLink;
|
|
1779
2371
|
root.submittedLink = undefined;
|
|
1780
2372
|
const submittedLinks = root.submittedLinks;
|
|
@@ -1843,31 +2435,61 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1843
2435
|
// reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain —
|
|
1844
2436
|
// but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction
|
|
1845
2437
|
let transaction = options.transaction ?? this.transaction;
|
|
2438
|
+
const retryRound = Boolean((options.transaction as RocksTransactionWithRetry)?.isRetry);
|
|
2439
|
+
const isImmediateCommit = Boolean((transaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1846
2440
|
try {
|
|
2441
|
+
if (retryRound && this.activeCommitHandle && this.activeCommitHandle !== transaction) {
|
|
2442
|
+
this.backfillActiveHandleWrites(this.activeCommitHandle);
|
|
2443
|
+
}
|
|
1847
2444
|
for (let i = 0; i < this.writes.length; i++) {
|
|
1848
2445
|
let operation = this.writes[i];
|
|
1849
|
-
if (
|
|
1850
|
-
|
|
2446
|
+
if (
|
|
2447
|
+
!operation ||
|
|
2448
|
+
(isImmediateCommit
|
|
2449
|
+
? !retryRound || !operation.saved || operation.stagedOn !== transaction
|
|
2450
|
+
: retryRound
|
|
2451
|
+
? !operation.saved || (Boolean(operation.stagedOn) && operation.stagedOn !== transaction)
|
|
2452
|
+
: Boolean(operation.saved) || (this.saveCommits && Boolean(operation.deferSave)))
|
|
2453
|
+
)
|
|
2454
|
+
continue;
|
|
2455
|
+
this.save(operation, transaction, Boolean(operation.saved) || i < this.validated, options);
|
|
2456
|
+
if (!transaction) transaction = this.transaction;
|
|
1851
2457
|
}
|
|
2458
|
+
if (transaction && (this.writes.length > 0 || hasDurableLog(transaction)))
|
|
2459
|
+
this.trackActiveCommitHandle(transaction);
|
|
1852
2460
|
} catch (error) {
|
|
2461
|
+
if (!transaction) transaction = this.transaction ?? undefined;
|
|
1853
2462
|
// abort() releases only this.transaction; a retry round's handle was detached before its
|
|
1854
2463
|
// first submission and would otherwise hold its write intents until GC.
|
|
1855
|
-
if (transaction !== this.transaction)
|
|
1856
|
-
abortNativeTransaction(
|
|
1857
|
-
|
|
2464
|
+
if (transaction && transaction !== this.transaction)
|
|
2465
|
+
abortNativeTransaction(
|
|
2466
|
+
transaction,
|
|
2467
|
+
'aborting a retry transaction whose re-save threw',
|
|
2468
|
+
this.writes[0]?.store ?? this.db,
|
|
2469
|
+
this.startedFrom
|
|
2470
|
+
);
|
|
2471
|
+
this.abort(false, true, transaction);
|
|
1858
2472
|
throw error;
|
|
1859
2473
|
}
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
2474
|
+
if (!isImmediateCommit) {
|
|
2475
|
+
this.validated = this.saveCommits
|
|
2476
|
+
? this.writes.reduce((acc, w, idx) => (w && (w.saved || !w.deferSave) ? idx + 1 : acc), 0)
|
|
2477
|
+
: this.writes.length;
|
|
2478
|
+
}
|
|
2479
|
+
const completions = this.takeCompletions(transaction);
|
|
1863
2480
|
const stagedWrites = this.writes.length;
|
|
1864
|
-
|
|
2481
|
+
let inCommitPhase = false;
|
|
2482
|
+
if (completions.length > 0 && !isImmediateCommit) {
|
|
2483
|
+
inCommitPhase = true;
|
|
1865
2484
|
this.setCommitPhase(true);
|
|
1866
2485
|
}
|
|
1867
2486
|
let commitResult: MaybePromise<CommitResolution>;
|
|
1868
2487
|
try {
|
|
1869
2488
|
commitResult = when(completions.length > 0 ? Promise.all(completions) : null, () => {
|
|
1870
|
-
if (
|
|
2489
|
+
if (inCommitPhase) {
|
|
2490
|
+
inCommitPhase = false;
|
|
2491
|
+
this.setCommitPhase(false);
|
|
2492
|
+
}
|
|
1871
2493
|
// The transaction can be aborted underneath us while we are parked in the await above — by the
|
|
1872
2494
|
// monitor once the commit phase outlives its grace, or through the multi-store poison chain.
|
|
1873
2495
|
// abort() cleared the write set and released the handle, so resuming would commit nothing and
|
|
@@ -1879,7 +2501,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1879
2501
|
}
|
|
1880
2502
|
if (stagedWrites > 0 && this.writes.length === 0 && this.open === TRANSACTION_STATE.CLOSED)
|
|
1881
2503
|
throw new ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
|
|
1882
|
-
if (
|
|
2504
|
+
if (
|
|
2505
|
+
!retryRound &&
|
|
2506
|
+
!isImmediateCommit &&
|
|
2507
|
+
(this.saveCommits
|
|
2508
|
+
? this.writes.reduce((acc, w, idx) => (w && (w.saved || !w.deferSave) ? idx + 1 : acc), 0)
|
|
2509
|
+
: this.writes.length) > this.validated
|
|
2510
|
+
) {
|
|
1883
2511
|
// check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
|
|
1884
2512
|
return this.commit({ ...options, continuation: true });
|
|
1885
2513
|
}
|
|
@@ -1890,8 +2518,8 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1890
2518
|
// the local is empty: a truthy one is what the loop staged into, and the retained-handle
|
|
1891
2519
|
// and replay branches below deliberately commit a handle other than this.transaction.
|
|
1892
2520
|
if (!transaction) transaction = this.transaction;
|
|
1893
|
-
if (!options.transaction && this.
|
|
1894
|
-
this.open = TRANSACTION_STATE.CLOSED;
|
|
2521
|
+
if (!options.transaction && this.hasSavedWrite()) this.renewChainForNativeCommit();
|
|
2522
|
+
if (!isImmediateCommit) this.open = TRANSACTION_STATE.CLOSED;
|
|
1895
2523
|
// RocksTransaction.commit() resolves with RETRY_NOW_VALUE (a number) under
|
|
1896
2524
|
// coordinatedRetry, or void on a normal commit/abort.
|
|
1897
2525
|
let commitResolution: Promise<number | void> | void;
|
|
@@ -1899,11 +2527,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1899
2527
|
// recursions, immediate-commit re-entries, and a second top-level commit() (wrapper
|
|
1900
2528
|
// commit after an explicit in-handler commit) must not steal a reference owned by an
|
|
1901
2529
|
// outstanding iterator (doneReadTxn() would then never release the native handle).
|
|
1902
|
-
if (!this.baseReadRefConsumed) {
|
|
2530
|
+
if (!isImmediateCommit && !this.baseReadRefConsumed) {
|
|
1903
2531
|
this.baseReadRefConsumed = true;
|
|
1904
2532
|
this.readTxnsUsed--;
|
|
1905
2533
|
}
|
|
1906
|
-
|
|
2534
|
+
let hadLeaseProtectedWrite = false;
|
|
2535
|
+
if (!isImmediateCommit && this.readTxnsUsed > 0) {
|
|
1907
2536
|
// Outstanding iterators still stream through this.transaction — their native iterators
|
|
1908
2537
|
// live inside it (GetIterator wraps its write batch + snapshot), so committing or
|
|
1909
2538
|
// aborting the handle now would invalidate them mid-stream. Leave it open for the
|
|
@@ -1915,7 +2544,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1915
2544
|
// kept the last iterator from finishing cleanly — a hung stream, the long-transaction
|
|
1916
2545
|
// monitor's timeout abort — dropped writes the caller had already been told committed.
|
|
1917
2546
|
this.writes = this.writes.filter((write) => write); // filter out removed entries
|
|
1918
|
-
if (this.
|
|
2547
|
+
if (this.hasSavedWrite() || hasDurableLog(transaction)) {
|
|
1919
2548
|
if (!options.transaction) {
|
|
1920
2549
|
if (!replayedWritesWarned) {
|
|
1921
2550
|
replayedWritesWarned = true;
|
|
@@ -1937,23 +2566,53 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1937
2566
|
{ coordinatedRetry: true }
|
|
1938
2567
|
);
|
|
1939
2568
|
if (this.timestamp) replayTransaction.setTimestamp(this.timestamp);
|
|
1940
|
-
|
|
2569
|
+
const retries = ((transaction as RocksTransactionWithRetry | undefined)?.retries ?? this.retries) + 1;
|
|
2570
|
+
(replayTransaction as RocksTransactionWithRetry).retries = retries;
|
|
2571
|
+
this.retries = retries; // a replay round: commit handlers re-base on the reloaded entries
|
|
2572
|
+
const prevStagedTxnTime = this.getHandleStagedTxnTime(transaction);
|
|
2573
|
+
if (transaction) this.releaseActiveCommitHandle(transaction);
|
|
2574
|
+
this.trackActiveCommitHandle(replayTransaction);
|
|
2575
|
+
if (prevStagedTxnTime) this.setHandleStagedTxnTime(replayTransaction, prevStagedTxnTime);
|
|
2576
|
+
const abandoned = transaction;
|
|
2577
|
+
const abandonedLogged = hasDurableLog(abandoned);
|
|
2578
|
+
transaction = replayTransaction;
|
|
1941
2579
|
try {
|
|
1942
2580
|
for (const operation of this.writes) {
|
|
2581
|
+
if (!operation?.saved) continue;
|
|
2582
|
+
// A non-durable abandoned handle left no readable audit entry; a durable one may
|
|
2583
|
+
// have published its log and must retain the retry shield.
|
|
2584
|
+
if (operation.appendedTo === abandoned && !abandonedLogged) {
|
|
2585
|
+
operation.appendedAuditEntry = false;
|
|
2586
|
+
operation.appendedTo = undefined;
|
|
2587
|
+
}
|
|
1943
2588
|
this.save(operation, replayTransaction, true, options);
|
|
1944
2589
|
}
|
|
1945
2590
|
} catch (error) {
|
|
1946
|
-
abortNativeTransaction(
|
|
1947
|
-
|
|
2591
|
+
abortNativeTransaction(
|
|
2592
|
+
replayTransaction,
|
|
2593
|
+
'aborting a replay transaction whose re-save threw',
|
|
2594
|
+
this.writes[0]?.store ?? this.db,
|
|
2595
|
+
this.startedFrom
|
|
2596
|
+
);
|
|
2597
|
+
this.releaseActiveCommitHandle(replayTransaction);
|
|
2598
|
+
this.abort(
|
|
2599
|
+
false,
|
|
2600
|
+
true,
|
|
2601
|
+
this.hasSiblingCommitInFlight(replayTransaction) ? replayTransaction : undefined
|
|
2602
|
+
);
|
|
1948
2603
|
throw error;
|
|
1949
2604
|
}
|
|
1950
|
-
transaction = replayTransaction;
|
|
1951
2605
|
}
|
|
1952
2606
|
// with options.transaction set this is a retry round — the save loop above already
|
|
1953
2607
|
// re-staged the writes into it
|
|
1954
2608
|
this.assertDatabaseCommitAllowed(transaction);
|
|
1955
2609
|
this.markNativeCommitSubmitted();
|
|
1956
2610
|
commitResolution = transaction.commit() as Promise<void>;
|
|
2611
|
+
this.setHandleCommitInFlight(transaction, true);
|
|
2612
|
+
if (this.hasLeaseProtectedWrite) {
|
|
2613
|
+
hadLeaseProtectedWrite = true;
|
|
2614
|
+
this.retainTransactionLockHolds(transaction);
|
|
2615
|
+
}
|
|
1957
2616
|
recordCommitLatency(commitResolution, performance.now());
|
|
1958
2617
|
// Write-queue-depth accounting for this replay commit happens uniformly below, via
|
|
1959
2618
|
// trackOutstandingCommit(commitResolution) — see that function's comment. Omitting
|
|
@@ -1973,10 +2632,10 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1973
2632
|
this.releaseRetainedWriteIntents();
|
|
1974
2633
|
} else {
|
|
1975
2634
|
// no more reads need to be performed, just commit/abort based if there are any writes
|
|
1976
|
-
this.detachOwnedTransaction(); // any further operations operate immediately
|
|
2635
|
+
if (!isImmediateCommit) this.detachOwnedTransaction(); // any further operations operate immediately
|
|
1977
2636
|
if (transaction) {
|
|
1978
2637
|
this.writes = this.writes.filter((write) => write); // filter out removed entries
|
|
1979
|
-
if (this.
|
|
2638
|
+
if (this.hasSavedWriteForHandle(transaction, isImmediateCommit) || hasDurableLog(transaction)) {
|
|
1980
2639
|
// Commit retries can construct fresh ranges on this live handle after read ownership ends.
|
|
1981
2640
|
readTransactionOwners.delete(transaction);
|
|
1982
2641
|
// Re-fence before submitting: the loop above skips operations already marked saved, so
|
|
@@ -1984,22 +2643,35 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1984
2643
|
// retry/replay path re-saves every operation and is fenced there instead. Lease expiry
|
|
1985
2644
|
// only: an unlock() inside the lease leaves the staged write valid, an elapsed lease
|
|
1986
2645
|
// does not, whether or not the caller also unlocked.
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2646
|
+
if (this.hasLeaseProtectedWrite && !hasDurableLog(transaction)) {
|
|
2647
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
2648
|
+
const write = this.writes[i];
|
|
2649
|
+
if (
|
|
2650
|
+
!write?.saved ||
|
|
2651
|
+
(isImmediateCommit
|
|
2652
|
+
? write.stagedOn !== transaction
|
|
2653
|
+
: Boolean(write.stagedOn) && write.stagedOn !== transaction)
|
|
2654
|
+
)
|
|
2655
|
+
continue;
|
|
2656
|
+
const lapsed = write.lockHandle;
|
|
2657
|
+
if (!lapsed?.isLeaseExpired()) continue;
|
|
2658
|
+
abortNativeTransaction(
|
|
2659
|
+
transaction,
|
|
2660
|
+
'aborting a transaction whose record lock lapsed before submission',
|
|
2661
|
+
this.writes[0]?.store ?? this.db,
|
|
2662
|
+
this.startedFrom
|
|
2663
|
+
);
|
|
2664
|
+
// Every other terminal exit from commit() runs the logical cleanup too. Throwing
|
|
2665
|
+
// straight out would strand the OTHER locks this transaction holds, its staged
|
|
2666
|
+
// blobs, and the context's back-reference to a CLOSED transaction — and
|
|
2667
|
+
// transaction()'s onComplete has no rejection path to run it later.
|
|
2668
|
+
try {
|
|
2669
|
+
this.abort(false, true, transaction);
|
|
2670
|
+
} catch (error) {
|
|
2671
|
+
harperLogger.debug?.('cleaning up a transaction whose record lock lapsed', error);
|
|
2672
|
+
}
|
|
2673
|
+
throw lockNotHeldError(lapsed);
|
|
2001
2674
|
}
|
|
2002
|
-
throw lockNotHeldError(lapsed);
|
|
2003
2675
|
}
|
|
2004
2676
|
// The transaction was created with coordinatedRetry:true (see
|
|
2005
2677
|
// getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
|
|
@@ -2008,6 +2680,11 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2008
2680
|
this.assertDatabaseCommitAllowed(transaction);
|
|
2009
2681
|
this.markNativeCommitSubmitted();
|
|
2010
2682
|
commitResolution = transaction.commit();
|
|
2683
|
+
this.setHandleCommitInFlight(transaction, true);
|
|
2684
|
+
if (this.hasLeaseProtectedWrite) {
|
|
2685
|
+
hadLeaseProtectedWrite = true;
|
|
2686
|
+
this.retainTransactionLockHolds(transaction);
|
|
2687
|
+
}
|
|
2011
2688
|
// Record how long this commit stays outstanding (submit → settle) as a distribution
|
|
2012
2689
|
// metric. This is the same clock the overload check uses (trackOutstandingCommit
|
|
2013
2690
|
// stamps each attempt at submit), so a rising p99/p999 is the leading indicator for the
|
|
@@ -2025,7 +2702,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2025
2702
|
} else {
|
|
2026
2703
|
try {
|
|
2027
2704
|
commitResolution = transaction.abort();
|
|
2028
|
-
} catch {
|
|
2705
|
+
} catch (abortError) {
|
|
2706
|
+
if ((abortError as any)?.code === 'ERR_TRANSACTION_ABANDONED') {
|
|
2707
|
+
harperLogger.error?.(
|
|
2708
|
+
`Aborted a transaction with a durable log entry (empty write set), ${describeCommitIdentity(this.db, this.startedFrom, transaction)}`,
|
|
2709
|
+
abortError
|
|
2710
|
+
);
|
|
2711
|
+
}
|
|
2029
2712
|
// The transaction has uncommitted writes that were already cleared from
|
|
2030
2713
|
// this.writes by a concurrent immediate-commit path (e.g. writes made with
|
|
2031
2714
|
// an explicitly-reused closed transaction). Those writes are handled by the
|
|
@@ -2039,41 +2722,56 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2039
2722
|
// Read the table off the write itself, not this.db, which is whichever table first
|
|
2040
2723
|
// claimed this per-database transaction in txnForContext and so can name the wrong
|
|
2041
2724
|
// table when a transaction spans more than one table in the same database.
|
|
2042
|
-
trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
|
|
2725
|
+
trackOutstandingCommit(commitResolution, this.writes[0]?.store ?? this.db, this.startedFrom, transaction);
|
|
2043
2726
|
// Every retry round and every chained store re-enters here and must inherit the chain
|
|
2044
|
-
// root's clock rather than restart it
|
|
2045
|
-
|
|
2046
|
-
|
|
2727
|
+
// root's clock rather than restart it (or the per-handle clock on an unchained
|
|
2728
|
+
// ImmediateTransaction), so only the first submission stamps.
|
|
2729
|
+
const carrier = this.budgetCarrier(transaction);
|
|
2730
|
+
if (carrier.commitStartedAt == null) carrier.commitStartedAt = performance.now();
|
|
2047
2731
|
const completions = [];
|
|
2048
2732
|
const commitOutcome = commitResolution.then(
|
|
2049
2733
|
(commitResult) => {
|
|
2734
|
+
const txnWithRetry = transaction as RocksTransactionWithRetry;
|
|
2050
2735
|
if (commitResult === RETRY_NOW_VALUE) {
|
|
2051
|
-
this.
|
|
2052
|
-
|
|
2736
|
+
this.setHandleCommitInFlight(transaction, false);
|
|
2737
|
+
const retries = (txnWithRetry.retries = (txnWithRetry.retries ?? 0) + 1);
|
|
2738
|
+
if (!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) this.retries = retries;
|
|
2739
|
+
harperLogger.debug?.('coordinated retry', transaction.id, retries);
|
|
2053
2740
|
// Mark this specific native transaction as a retry so RocksTransactionLogStore
|
|
2054
2741
|
// skips re-writing its already-staged txn-log entries (#2).
|
|
2055
|
-
|
|
2056
|
-
const
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2742
|
+
txnWithRetry.isRetry = true;
|
|
2743
|
+
const logged = hasDurableLog(transaction);
|
|
2744
|
+
const pastBudget = this.elapsedPastCommitBudget(transaction);
|
|
2745
|
+
if (pastBudget) {
|
|
2746
|
+
if (logged) this.noteOverdueLoggedCommit(transaction, pastBudget);
|
|
2747
|
+
else this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
2748
|
+
}
|
|
2749
|
+
if (retries > MAX_RETRIES && !this.sourceApply && !logged) {
|
|
2750
|
+
// giving up: poison and abort the whole linked chain so no link leaks its native
|
|
2751
|
+
// handle / read snapshot — or any unpublished transaction-log position — until GC.
|
|
2752
|
+
this.abortChainAfterRetries(transaction);
|
|
2753
|
+
throw new ServerError(
|
|
2754
|
+
`After ${MAX_RETRIES} coordinated retries, unable to commit transaction, transaction is in conflict with ongoing writes`
|
|
2755
|
+
);
|
|
2756
|
+
}
|
|
2757
|
+
if ((this.sourceApply || logged) && retries > 0 && retries % MAX_RETRIES === 0) {
|
|
2758
|
+
harperLogger.warn?.(
|
|
2759
|
+
`${this.sourceApply ? 'Source-applied' : 'Logged'} transaction ${transaction.id} still in conflict after ${retries} coordinated retries; continuing to retry`
|
|
2760
|
+
);
|
|
2761
|
+
}
|
|
2762
|
+
// Randomized so two uncapped logged commits crossing on the same keys stop waking in lockstep.
|
|
2763
|
+
if (logged && retries > 2) {
|
|
2764
|
+
const backoff = delay(Math.ceil(Math.min(retries * 2, MAX_RETRY_DELAY_MS) * Math.random()));
|
|
2765
|
+
trackOutstandingCommit(backoff, this.writes[0]?.store ?? this.db, this.startedFrom, transaction);
|
|
2766
|
+
return backoff.then(() => this.commit({ ...options, transaction }));
|
|
2074
2767
|
}
|
|
2075
2768
|
return this.commit({ ...options, transaction });
|
|
2076
2769
|
}
|
|
2770
|
+
const committedTransaction = transaction;
|
|
2771
|
+
const committedImmediate = Boolean(
|
|
2772
|
+
(committedTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit
|
|
2773
|
+
);
|
|
2774
|
+
this.setHandleCommitted(transaction);
|
|
2077
2775
|
// onCommit may be async (e.g. RocksTransactionLogStore emits 'aftercommit'). Surface a
|
|
2078
2776
|
// rejection — or a synchronous throw — via logging rather than failing the commit, since
|
|
2079
2777
|
// the write is already durable.
|
|
@@ -2084,7 +2782,15 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2084
2782
|
} catch (error) {
|
|
2085
2783
|
harperLogger.warn?.('onCommit handler failed after commit', error);
|
|
2086
2784
|
}
|
|
2087
|
-
if (this.
|
|
2785
|
+
if (this.recordLocks) this.noteCommittedLockVersions(committedTransaction);
|
|
2786
|
+
releaseTransactionLockHolds(transaction);
|
|
2787
|
+
if (
|
|
2788
|
+
committedImmediate
|
|
2789
|
+
? this.open !== TRANSACTION_STATE.OPEN && !this.hasSiblingCommitInFlight(committedTransaction)
|
|
2790
|
+
: !this.saveCommits || !this.hasSiblingCommitInFlight(committedTransaction)
|
|
2791
|
+
)
|
|
2792
|
+
this.releaseRecordLocks();
|
|
2793
|
+
if (this.next && !committedImmediate) {
|
|
2088
2794
|
// never forward options.transaction (a retry/replay round's HEAD-store handle) to
|
|
2089
2795
|
// the next store — it must commit its own writes through its own transaction
|
|
2090
2796
|
let nextCommit: MaybePromise<CommitResolution>;
|
|
@@ -2105,7 +2811,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2105
2811
|
// if we want to wait for replication confirmation, we need to track the transaction times
|
|
2106
2812
|
// and when replication notifications come in, we count the number of confirms until we reach the desired number
|
|
2107
2813
|
const databaseName = this.writes[0]?.store.rootStore.databaseName;
|
|
2108
|
-
const lastEntry = this.lastConfirmableEntry();
|
|
2814
|
+
const lastEntry = this.lastConfirmableEntry(true);
|
|
2109
2815
|
if (confirmReplication && databaseName && lastEntry) {
|
|
2110
2816
|
completions.push(
|
|
2111
2817
|
confirmReplication(databaseName, (lastEntry as any).version, this.replicatedConfirmation)
|
|
@@ -2118,23 +2824,43 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2118
2824
|
// later write retained survives). Deferred until here so a retry that *would* have
|
|
2119
2825
|
// referenced the blob can flip skipped/superseded back to false first.
|
|
2120
2826
|
for (const write of this.writes) {
|
|
2121
|
-
if (write?.
|
|
2827
|
+
if (!write?.saved || !write.savedBlobs) continue;
|
|
2828
|
+
if (
|
|
2829
|
+
committedImmediate
|
|
2830
|
+
? write.stagedOn !== committedTransaction
|
|
2831
|
+
: Boolean(write.stagedOn) && write.stagedOn !== committedTransaction
|
|
2832
|
+
)
|
|
2833
|
+
continue;
|
|
2834
|
+
if (
|
|
2835
|
+
(write.skipped &&
|
|
2836
|
+
!(
|
|
2837
|
+
write.blobsAuditReferenced &&
|
|
2838
|
+
(write.appendedTo === committedTransaction || hasDurableLog(write.appendedTo))
|
|
2839
|
+
)) ||
|
|
2840
|
+
(write.superseded && !write.blobsAuditReferenced)
|
|
2841
|
+
)
|
|
2122
2842
|
cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
|
|
2123
2843
|
}
|
|
2124
|
-
|
|
2844
|
+
const committedStagedTxnTime = this.getHandleStagedTxnTime(committedTransaction);
|
|
2125
2845
|
// now reset transactions tracking; this transaction be reused and committed again
|
|
2126
|
-
this.
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2846
|
+
this.clearWrites(committedTransaction);
|
|
2847
|
+
if (!committedImmediate) {
|
|
2848
|
+
this.retries = 0;
|
|
2849
|
+
if (options.doneWriting) this.endScopeOwnership();
|
|
2850
|
+
this.releaseContext(!!options.doneWriting);
|
|
2851
|
+
}
|
|
2852
|
+
let txnTime = committedStagedTxnTime || this.timestamp || 0;
|
|
2853
|
+
if (
|
|
2854
|
+
(!committedImmediate || this.open !== TRANSACTION_STATE.OPEN) &&
|
|
2855
|
+
this.writes.length === 0 &&
|
|
2856
|
+
!this.hasActiveCommitHandle()
|
|
2857
|
+
)
|
|
2858
|
+
this.timestamp = 0; // reset the timestamp as well
|
|
2133
2859
|
return Promise.all(completions).then(
|
|
2134
2860
|
() => {
|
|
2135
2861
|
// Only once the chained store's commit has settled, as on the synchronous path: a
|
|
2136
2862
|
// partially failed mid-scope commit must not leave the scope resumable.
|
|
2137
|
-
this.completeMidScopeCommit(options);
|
|
2863
|
+
this.completeMidScopeCommit(options, committedTransaction);
|
|
2138
2864
|
return {
|
|
2139
2865
|
txnTime,
|
|
2140
2866
|
};
|
|
@@ -2143,12 +2869,15 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2143
2869
|
// As on the synchronous path: a completion that failed (a chained store's commit,
|
|
2144
2870
|
// a replication confirmation) leaves this commit partly landed, so ownership goes
|
|
2145
2871
|
// with it rather than letting a later commit rotate on top.
|
|
2146
|
-
|
|
2872
|
+
(committedTransaction as RocksTransactionWithRetry).committed = true;
|
|
2873
|
+
if (!committedImmediate) this.endScopeOwnership();
|
|
2147
2874
|
throw error;
|
|
2148
2875
|
}
|
|
2149
2876
|
);
|
|
2150
2877
|
},
|
|
2151
2878
|
(error) => {
|
|
2879
|
+
const txnWithRetry = transaction as RocksTransactionWithRetry;
|
|
2880
|
+
this.setHandleCommitInFlight(transaction, false);
|
|
2152
2881
|
// Coordinated transactions surface conflicts as RETRY_NOW (handled in the
|
|
2153
2882
|
// resolve branch above) and never reach here with ERR_BUSY. But not every
|
|
2154
2883
|
// write transaction is coordinated — a write that reaches save() with no
|
|
@@ -2165,8 +2894,9 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2165
2894
|
if (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') {
|
|
2166
2895
|
// if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
|
|
2167
2896
|
// for future transactions
|
|
2168
|
-
|
|
2169
|
-
|
|
2897
|
+
const retries = (txnWithRetry.retries = (txnWithRetry.retries ?? 0) + 1);
|
|
2898
|
+
if (!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) this.retries = retries;
|
|
2899
|
+
harperLogger.debug?.('retrying', transaction.id, retries);
|
|
2170
2900
|
// ERR_BUSY and ERR_TRY_AGAIN are both retried by recommitting the SAME native
|
|
2171
2901
|
// transaction. ERR_BUSY recovers because the save loop re-writes each key, re-tracking
|
|
2172
2902
|
// it at the current sequence. ERR_TRY_AGAIN — a snapshot stranded outside the memtable
|
|
@@ -2179,95 +2909,155 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2179
2909
|
// already-staged change-feed entry publishes only when the retry really commits — no
|
|
2180
2910
|
// premature publish, and no fresh-transaction replay that would drop the entry.
|
|
2181
2911
|
// Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
|
|
2182
|
-
|
|
2912
|
+
txnWithRetry.isRetry = true;
|
|
2913
|
+
if ((error as any)?.hasLog === true) {
|
|
2914
|
+
txnWithRetry.hasStagedLog = true;
|
|
2915
|
+
}
|
|
2916
|
+
const logged = hasDurableLog(transaction, error as any);
|
|
2183
2917
|
// Before the backoff gate below: the budget can already be spent on the first
|
|
2184
2918
|
// retry when an earlier store in this chain consumed it.
|
|
2185
|
-
const pastBudget = this.elapsedPastCommitBudget();
|
|
2186
|
-
if (pastBudget)
|
|
2187
|
-
|
|
2919
|
+
const pastBudget = this.elapsedPastCommitBudget(transaction);
|
|
2920
|
+
if (pastBudget) {
|
|
2921
|
+
if (logged) this.noteOverdueLoggedCommit(transaction, pastBudget);
|
|
2922
|
+
else this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
2923
|
+
}
|
|
2924
|
+
if (retries > 2) {
|
|
2188
2925
|
// Transactions applying data from a canonical source of truth (replication peer or
|
|
2189
2926
|
// external caching source) must never drop a write on a transient conflict: there is no
|
|
2190
2927
|
// re-subscribe / sequence-id-resume path, so a dropped write would leave this node
|
|
2191
2928
|
// permanently diverged (harper-pro#348). Such transactions retry without a cap; the source
|
|
2192
2929
|
// apply loop serializes commits (backpressure), so contention clears rather than
|
|
2193
|
-
// compounding.
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
}
|
|
2930
|
+
// compounding. Durably logged transactions likewise retry without a cap so a written
|
|
2931
|
+
// transaction-log entry is never abandoned.
|
|
2932
|
+
const neverDropOnConflict = this.sourceApply || logged;
|
|
2933
|
+
if (retries > MAX_RETRIES && !neverDropOnConflict) {
|
|
2934
|
+
// giving up: poison and abort the whole linked chain so no link leaks its native
|
|
2935
|
+
// handle / read snapshot until GC.
|
|
2936
|
+
this.abortChainAfterRetries(transaction);
|
|
2937
|
+
throw new ServerError(
|
|
2938
|
+
`After ${MAX_RETRIES} retries, unable to commit transaction, transaction is in conflict with ongoing writes`
|
|
2939
|
+
);
|
|
2940
|
+
}
|
|
2941
|
+
// Uncapped retry can otherwise stall silently (debug logging is off in production);
|
|
2942
|
+
// surface periodic visibility into a stalled source-apply or logged commit.
|
|
2943
|
+
if (neverDropOnConflict && retries % MAX_RETRIES === 0) {
|
|
2944
|
+
harperLogger.warn?.(
|
|
2945
|
+
`${this.sourceApply ? 'Source-applied' : 'Logged'} transaction ${transaction.id} still in conflict after ${retries} retries; continuing to retry`
|
|
2946
|
+
);
|
|
2211
2947
|
}
|
|
2212
2948
|
// start delaying, back off to try to space out transactions and avoid excessive conflicts
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2949
|
+
const backoff = delay(Math.min(retries * retries, MAX_RETRY_DELAY_MS));
|
|
2950
|
+
if (logged) {
|
|
2951
|
+
// waitForDrain() must outlast the backoff too
|
|
2952
|
+
trackOutstandingCommit(backoff, this.writes[0]?.store ?? this.db, this.startedFrom, transaction);
|
|
2953
|
+
}
|
|
2954
|
+
return backoff.then(() => this.commit({ ...options, transaction }));
|
|
2216
2955
|
}
|
|
2217
2956
|
return this.commit({ ...options, transaction }); // try again
|
|
2218
2957
|
} else {
|
|
2219
2958
|
// terminal (non-conflict) failure: release the native handle so it doesn't leak;
|
|
2220
2959
|
// usually already released by the failed commit itself, abort for the unexpected
|
|
2221
2960
|
// case (same defensive pattern as the retry-exhaustion give-up above)
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
harperLogger.debug?.('aborting transaction after failed commit', abortError);
|
|
2961
|
+
if ((error as any)?.hasLog) {
|
|
2962
|
+
txnWithRetry.isRetry = true;
|
|
2963
|
+
txnWithRetry.hasStagedLog = true;
|
|
2226
2964
|
}
|
|
2965
|
+
abortNativeTransaction(
|
|
2966
|
+
transaction,
|
|
2967
|
+
'aborting transaction after failed commit',
|
|
2968
|
+
this.writes[0]?.store ?? this.db,
|
|
2969
|
+
this.startedFrom
|
|
2970
|
+
);
|
|
2227
2971
|
// A terminal failure is just as final as a success — release the context's
|
|
2228
2972
|
// back-reference while handling the native transaction. performCommit's outer
|
|
2229
2973
|
// rejection handler then aborts the wrapper and its linked chain.
|
|
2230
2974
|
// A failed commit must never be followed by a resumed segment: this generation is
|
|
2231
2975
|
// finished and its durability is unknown, so ownership goes with it.
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2976
|
+
if (
|
|
2977
|
+
!isImmediateCommit &&
|
|
2978
|
+
!this.hasWriteOwnedByActiveCommit(transaction, this.hasSiblingCommitInFlight(transaction), true, true)
|
|
2979
|
+
) {
|
|
2980
|
+
this.endScopeOwnership();
|
|
2981
|
+
this.releaseContext(!!options.doneWriting);
|
|
2982
|
+
}
|
|
2983
|
+
if (!hasDurableLog(transaction, error as any)) {
|
|
2984
|
+
for (const write of this.writes) {
|
|
2985
|
+
if (
|
|
2986
|
+
!write?.saved ||
|
|
2987
|
+
!write.savedBlobs ||
|
|
2988
|
+
(isImmediateCommit
|
|
2989
|
+
? write.stagedOn !== transaction
|
|
2990
|
+
: Boolean(write.stagedOn) && write.stagedOn !== transaction)
|
|
2991
|
+
)
|
|
2992
|
+
continue;
|
|
2993
|
+
cleanupUnusedBlobs(
|
|
2994
|
+
write.savedBlobs,
|
|
2995
|
+
collectRetainedFileIds(write.store.getEntry(write.key)?.value)
|
|
2996
|
+
);
|
|
2997
|
+
}
|
|
2998
|
+
}
|
|
2999
|
+
this.clearWrites(transaction);
|
|
3000
|
+
releaseTransactionLockHolds(transaction);
|
|
3001
|
+
if (
|
|
3002
|
+
(!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) &&
|
|
3003
|
+
!this.hasActiveCommitHandle() &&
|
|
3004
|
+
this.writes.length === 0
|
|
3005
|
+
)
|
|
3006
|
+
this.releaseRecordLocks();
|
|
3007
|
+
if ((!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) && !this.hasActiveCommitHandle())
|
|
3008
|
+
this.timestamp = 0;
|
|
2236
3009
|
throw error;
|
|
2237
3010
|
}
|
|
2238
3011
|
}
|
|
2239
3012
|
);
|
|
2240
|
-
//
|
|
2241
|
-
//
|
|
2242
|
-
//
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
chainRoot.commitStartedAt = undefined;
|
|
2249
|
-
return resolution;
|
|
2250
|
-
},
|
|
2251
|
-
(error) => {
|
|
2252
|
-
chainRoot.commitStartedAt = undefined;
|
|
2253
|
-
throw error;
|
|
2254
|
-
}
|
|
2255
|
-
);
|
|
3013
|
+
// Release any lock holds remaining if `commitOutcome` rejects (for example when a chained
|
|
3014
|
+
// store's commit or a completion fails) through the returned promise rather than a second
|
|
3015
|
+
// subscriber that would mark an unawaited commit rejection as handled.
|
|
3016
|
+
if (!hadLeaseProtectedWrite) return commitOutcome;
|
|
3017
|
+
return commitOutcome.then(undefined, (error) => {
|
|
3018
|
+
releaseTransactionLockHolds(transaction);
|
|
3019
|
+
throw error;
|
|
3020
|
+
});
|
|
2256
3021
|
}
|
|
2257
3022
|
for (const write of this.writes) {
|
|
2258
|
-
if (write?.
|
|
3023
|
+
if (!write?.saved) continue;
|
|
3024
|
+
if (
|
|
3025
|
+
isImmediateCommit
|
|
3026
|
+
? write.stagedOn !== transaction
|
|
3027
|
+
: this.hasSiblingCommitInFlight(transaction) && Boolean(write.stagedOn) && write.stagedOn !== transaction
|
|
3028
|
+
)
|
|
3029
|
+
continue;
|
|
3030
|
+
if (
|
|
3031
|
+
write.savedBlobs &&
|
|
3032
|
+
((write.skipped &&
|
|
3033
|
+
!(write.blobsAuditReferenced && (write.appendedTo === transaction || hasDurableLog(write.appendedTo)))) ||
|
|
3034
|
+
(write.superseded && !write.blobsAuditReferenced))
|
|
3035
|
+
)
|
|
2259
3036
|
cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
|
|
2260
3037
|
}
|
|
2261
|
-
if (this.recordLocks) this.noteCommittedLockVersions();
|
|
2262
|
-
this.
|
|
2263
|
-
this.
|
|
2264
|
-
if (
|
|
2265
|
-
|
|
3038
|
+
if (this.recordLocks) this.noteCommittedLockVersions(transaction);
|
|
3039
|
+
const committedStagedTxnTime = this.getHandleStagedTxnTime(transaction);
|
|
3040
|
+
this.clearWrites(transaction);
|
|
3041
|
+
if (
|
|
3042
|
+
isImmediateCommit
|
|
3043
|
+
? this.open !== TRANSACTION_STATE.OPEN && !this.hasActiveCommitHandle() && this.writes.length === 0
|
|
3044
|
+
: !this.saveCommits || (!this.hasActiveCommitHandle() && this.writes.length === 0)
|
|
3045
|
+
)
|
|
3046
|
+
this.releaseRecordLocks();
|
|
3047
|
+
if (!isImmediateCommit) {
|
|
3048
|
+
if (options.doneWriting) this.endScopeOwnership();
|
|
3049
|
+
this.releaseContext(!!options.doneWriting);
|
|
3050
|
+
}
|
|
2266
3051
|
const txnResolution: CommitResolution = {
|
|
2267
|
-
txnTime: this.timestamp,
|
|
3052
|
+
txnTime: committedStagedTxnTime || this.timestamp,
|
|
2268
3053
|
};
|
|
2269
|
-
|
|
2270
|
-
|
|
3054
|
+
if (
|
|
3055
|
+
(!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) &&
|
|
3056
|
+
this.writes.length === 0 &&
|
|
3057
|
+
!this.hasActiveCommitHandle()
|
|
3058
|
+
)
|
|
3059
|
+
this.timestamp = 0;
|
|
3060
|
+
if (this.next && !isImmediateCommit) {
|
|
2271
3061
|
// now run any other transactions
|
|
2272
3062
|
options.timestamp = txnResolution.txnTime;
|
|
2273
3063
|
// as above: the next store must not inherit this store's explicit native transaction
|
|
@@ -2286,7 +3076,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2286
3076
|
(nextResolution) => {
|
|
2287
3077
|
// Only once the chained store's own commit has SETTLED: rotating first would leave the
|
|
2288
3078
|
// scope resumable after a partially failed mid-scope commit.
|
|
2289
|
-
this.completeMidScopeCommit(options);
|
|
3079
|
+
this.completeMidScopeCommit(options, transaction);
|
|
2290
3080
|
return {
|
|
2291
3081
|
txnTime: txnResolution.txnTime,
|
|
2292
3082
|
next: nextResolution,
|
|
@@ -2303,17 +3093,17 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2303
3093
|
);
|
|
2304
3094
|
txnResolution.next = nextResolution as any;
|
|
2305
3095
|
}
|
|
2306
|
-
this.completeMidScopeCommit(options);
|
|
3096
|
+
this.completeMidScopeCommit(options, transaction);
|
|
2307
3097
|
return txnResolution;
|
|
2308
3098
|
});
|
|
2309
3099
|
} catch (error) {
|
|
2310
|
-
this.setCommitPhase(false);
|
|
2311
|
-
this.abortAfterCommitError(error);
|
|
3100
|
+
if (inCommitPhase) this.setCommitPhase(false);
|
|
3101
|
+
this.abortAfterCommitError(error, transaction);
|
|
2312
3102
|
}
|
|
2313
3103
|
if ((commitResult as Promise<CommitResolution>)?.then)
|
|
2314
3104
|
return (commitResult as Promise<CommitResolution>).catch((error) => {
|
|
2315
|
-
this.setCommitPhase(false);
|
|
2316
|
-
this.abortAfterCommitError(error);
|
|
3105
|
+
if (inCommitPhase) this.setCommitPhase(false);
|
|
3106
|
+
this.abortAfterCommitError(error, transaction);
|
|
2317
3107
|
});
|
|
2318
3108
|
return commitResult;
|
|
2319
3109
|
}
|
|
@@ -2342,8 +3132,10 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2342
3132
|
this.snapshotFree = false;
|
|
2343
3133
|
}
|
|
2344
3134
|
|
|
2345
|
-
private completeMidScopeCommit(options: CommitOptions): void {
|
|
3135
|
+
private completeMidScopeCommit(options: CommitOptions, committedTransaction?: RocksTransaction | null): void {
|
|
3136
|
+
if ((committedTransaction as RocksTransactionWithRetry | null | undefined)?.immediateCommit) return;
|
|
2346
3137
|
this.next = null;
|
|
3138
|
+
if (this.saveCommits && this.hasActiveCommitHandle()) return;
|
|
2347
3139
|
this.rotateAfterMidScopeCommit(options);
|
|
2348
3140
|
}
|
|
2349
3141
|
|
|
@@ -2361,75 +3153,149 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2361
3153
|
this.open = TRANSACTION_STATE.OPEN;
|
|
2362
3154
|
this.snapshotFree = true;
|
|
2363
3155
|
this.writesAbandoned = false;
|
|
3156
|
+
if (this.#context && (this.#context.transaction === this || this.#context.transaction === RELEASED_TRANSACTION)) {
|
|
3157
|
+
this.#context.transaction = this;
|
|
3158
|
+
}
|
|
2364
3159
|
}
|
|
2365
3160
|
|
|
2366
3161
|
private assertDatabaseCommitAllowed(transaction: RocksTransaction): void {
|
|
3162
|
+
if (hasDurableLog(transaction)) return;
|
|
2367
3163
|
const rootStore = this.writes[0]?.store?.rootStore;
|
|
2368
3164
|
if (!databaseCommitsSuspended(rootStore)) return;
|
|
2369
3165
|
const retryable = !this.root && !this.snapshotFree;
|
|
2370
3166
|
if (this.transaction === transaction) this.detachOwnedTransaction();
|
|
2371
|
-
abortNativeTransaction(
|
|
3167
|
+
abortNativeTransaction(
|
|
3168
|
+
transaction,
|
|
3169
|
+
'aborting a transaction while its database is closing',
|
|
3170
|
+
this.writes[0]?.store ?? this.db,
|
|
3171
|
+
this.startedFrom
|
|
3172
|
+
);
|
|
2372
3173
|
try {
|
|
2373
|
-
this.abort();
|
|
3174
|
+
this.abort(false, true, transaction);
|
|
2374
3175
|
} catch (error) {
|
|
2375
3176
|
harperLogger.debug?.('cleaning up a transaction while its database is closing', error);
|
|
2376
3177
|
}
|
|
2377
3178
|
throw new DatabaseClosingError(rootStore?.databaseName ?? rootStore?.path ?? 'unknown', retryable);
|
|
2378
3179
|
}
|
|
2379
3180
|
|
|
2380
|
-
protected abortAfterCommitError(error): never {
|
|
3181
|
+
protected abortAfterCommitError(error, targetTransaction?: RocksTransaction): never {
|
|
2381
3182
|
try {
|
|
3183
|
+
const retryTxn = targetTransaction as RocksTransactionWithRetry | undefined;
|
|
3184
|
+
const retryTxnCommitted = this.isHandleCommitted(retryTxn);
|
|
3185
|
+
const retryTxnInFlight = this.isHandleCommitInFlight(retryTxn);
|
|
3186
|
+
if (retryTxn && retryTxnCommitted) retryTxn.committed = true;
|
|
3187
|
+
if (retryTxn) this.releaseActiveCommitHandle(retryTxn);
|
|
3188
|
+
if (retryTxn && retryTxn !== this.transaction && !retryTxnCommitted && !retryTxnInFlight) {
|
|
3189
|
+
releaseTransactionLockHolds(retryTxn);
|
|
3190
|
+
abortNativeTransaction(
|
|
3191
|
+
retryTxn,
|
|
3192
|
+
'aborting a retry transaction after pre-commit failure',
|
|
3193
|
+
this.writes[0]?.store ?? this.db,
|
|
3194
|
+
this.startedFrom
|
|
3195
|
+
);
|
|
3196
|
+
}
|
|
2382
3197
|
// Not "always retain": abort(true) keeps the native handle only while read iterators still
|
|
2383
3198
|
// own it. resources/transaction.ts's own post-error abort passes the same argument, so the
|
|
2384
3199
|
// two layers cannot contradict each other one frame apart.
|
|
2385
|
-
this.abort(true);
|
|
3200
|
+
this.abort(true, true, targetTransaction);
|
|
2386
3201
|
} catch (abortError) {
|
|
2387
3202
|
harperLogger.debug?.('aborting transaction after a failed commit', abortError);
|
|
2388
3203
|
}
|
|
2389
3204
|
throw error;
|
|
2390
3205
|
}
|
|
2391
|
-
abort(retainReadTransaction = false, cascade = true): void {
|
|
2392
|
-
const
|
|
2393
|
-
|
|
3206
|
+
abort(retainReadTransaction = false, cascade = true, targetTransaction?: RocksTransaction): void {
|
|
3207
|
+
const nativeTxn = targetTransaction ?? this.transaction;
|
|
3208
|
+
const isImmediateTarget = Boolean((targetTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
3209
|
+
const hasSiblingInFlight = this.hasSiblingCommitInFlight(targetTransaction);
|
|
3210
|
+
const preserveActiveWrites =
|
|
3211
|
+
isImmediateTarget || this.hasWriteOwnedByActiveCommit(targetTransaction, hasSiblingInFlight, true);
|
|
3212
|
+
const preserveNonImmediateActiveWrites =
|
|
3213
|
+
isImmediateTarget || this.hasWriteOwnedByActiveCommit(targetTransaction, hasSiblingInFlight, true, true);
|
|
3214
|
+
const cascadeNext = cascade && !preserveNonImmediateActiveWrites;
|
|
3215
|
+
const next = cascadeNext ? this.next : undefined;
|
|
3216
|
+
if (cascadeNext && !retainReadTransaction) this.next = null;
|
|
2394
3217
|
try {
|
|
2395
|
-
const
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
3218
|
+
const ownsCurrentTransaction =
|
|
3219
|
+
!isImmediateTarget && (!hasSiblingInFlight || !targetTransaction || this.transaction === targetTransaction);
|
|
3220
|
+
const currentTxn = this.transaction as RocksTransactionWithRetry | null | undefined;
|
|
3221
|
+
const currentTxnActive =
|
|
3222
|
+
Boolean(currentTxn) &&
|
|
3223
|
+
!currentTxn.abortedByWrapper &&
|
|
3224
|
+
!this.isHandleCommitted(currentTxn) &&
|
|
3225
|
+
(this.isHandleCommitInFlight(currentTxn) ||
|
|
3226
|
+
hasDurableLog(currentTxn) ||
|
|
3227
|
+
(Boolean(targetTransaction) && currentTxn !== targetTransaction && this.isActiveCommitHandle(currentTxn)));
|
|
3228
|
+
if (ownsCurrentTransaction && !currentTxnActive) {
|
|
3229
|
+
if (this.transaction) this.releaseActiveCommitHandle(this.transaction);
|
|
3230
|
+
const hasOpenReadIterator =
|
|
3231
|
+
retainReadTransaction &&
|
|
3232
|
+
this.transaction &&
|
|
3233
|
+
(this.readTxnsUsed > 1 || (this.baseReadRefConsumed && this.readTxnsUsed > 0));
|
|
3234
|
+
if (hasOpenReadIterator) {
|
|
3235
|
+
this.releaseRetainedWriteIntents();
|
|
3236
|
+
if (!this.baseReadRefConsumed) {
|
|
3237
|
+
this.doneReadTxn();
|
|
3238
|
+
this.baseReadRefConsumed = true;
|
|
3239
|
+
}
|
|
3240
|
+
} else {
|
|
3241
|
+
while (this.transaction && this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
|
|
2404
3242
|
}
|
|
2405
|
-
|
|
2406
|
-
|
|
3243
|
+
// A write-only transaction never took a read reference (getReadTxn was never called), so the
|
|
3244
|
+
// loop above releases nothing even though save() created a native handle. Keep an iterator's
|
|
3245
|
+
// handle alive, but release every other remaining handle through the shared cleanup path.
|
|
3246
|
+
if (this.transaction && !hasOpenReadIterator) this.releaseReadTxn();
|
|
2407
3247
|
}
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
if (this.transaction && !hasOpenReadIterator) this.releaseReadTxn();
|
|
2412
|
-
this.open = TRANSACTION_STATE.CLOSED;
|
|
2413
|
-
this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
|
|
2414
|
-
this.drainCompletions();
|
|
3248
|
+
if (!isImmediateTarget) this.open = TRANSACTION_STATE.CLOSED;
|
|
3249
|
+
this.drainCompletions(targetTransaction, preserveActiveWrites);
|
|
3250
|
+
const targetLogged = hasDurableLog(targetTransaction);
|
|
2415
3251
|
for (const write of this.writes) {
|
|
2416
|
-
if (write
|
|
3252
|
+
if (!write) continue;
|
|
3253
|
+
if (preserveActiveWrites && this.isWriteOwnedByActiveCommit(write, targetTransaction, hasSiblingInFlight, true))
|
|
3254
|
+
continue;
|
|
3255
|
+
if (
|
|
3256
|
+
targetLogged &&
|
|
3257
|
+
write.saved &&
|
|
3258
|
+
(isImmediateTarget
|
|
3259
|
+
? write.stagedOn === targetTransaction
|
|
3260
|
+
: !write.stagedOn || write.stagedOn === targetTransaction)
|
|
3261
|
+
)
|
|
3262
|
+
continue;
|
|
3263
|
+
if (write.savedBlobs)
|
|
2417
3264
|
cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
|
|
2418
3265
|
}
|
|
2419
3266
|
} finally {
|
|
2420
3267
|
try {
|
|
2421
|
-
this.open = TRANSACTION_STATE.CLOSED;
|
|
2422
|
-
|
|
2423
|
-
this.
|
|
2424
|
-
this.
|
|
3268
|
+
if (!isImmediateTarget) this.open = TRANSACTION_STATE.CLOSED;
|
|
3269
|
+
const endScopeOnAbort = !isImmediateTarget && (!targetTransaction || !preserveNonImmediateActiveWrites);
|
|
3270
|
+
if (endScopeOnAbort) this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
|
|
3271
|
+
this.clearWrites(targetTransaction, true);
|
|
3272
|
+
if (targetTransaction) {
|
|
3273
|
+
releaseTransactionLockHolds(targetTransaction);
|
|
3274
|
+
}
|
|
3275
|
+
if ((!isImmediateTarget || this.open !== TRANSACTION_STATE.OPEN) && !this.hasActiveCommitHandle()) {
|
|
3276
|
+
this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
|
|
3277
|
+
this.retries = 0;
|
|
3278
|
+
}
|
|
3279
|
+
if (
|
|
3280
|
+
(isImmediateTarget
|
|
3281
|
+
? this.open !== TRANSACTION_STATE.OPEN && !this.hasActiveCommitHandle()
|
|
3282
|
+
: !preserveActiveWrites) &&
|
|
3283
|
+
this.writes.length === 0
|
|
3284
|
+
) {
|
|
3285
|
+
releaseTransactionLockHolds(nativeTxn);
|
|
3286
|
+
if (this.transaction && this.transaction !== nativeTxn) releaseTransactionLockHolds(this.transaction);
|
|
3287
|
+
this.releaseRecordLocks();
|
|
3288
|
+
}
|
|
2425
3289
|
// A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
|
|
2426
3290
|
// Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
|
|
2427
3291
|
// starting a fresh one) so the rest of the logical operation fails atomically via the
|
|
2428
3292
|
// poison check in addWrite()/commit(), rather than silently landing a later write on a
|
|
2429
3293
|
// brand-new transaction after an earlier one was rolled back (#1411). Releasing here would
|
|
2430
3294
|
// make that check see `undefined?.timedOut` and take the "start fresh" branch instead.
|
|
2431
|
-
|
|
2432
|
-
|
|
3295
|
+
if (endScopeOnAbort) {
|
|
3296
|
+
this.releaseContext(!this.timedOut && !this.disconnected);
|
|
3297
|
+
if (!this.root) this.releaseRequestAbortListener();
|
|
3298
|
+
}
|
|
2433
3299
|
} finally {
|
|
2434
3300
|
if (next && !(next.nativeCommitSubmitted && next.commitsInFlight)) {
|
|
2435
3301
|
try {
|
|
@@ -2455,9 +3321,9 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2455
3321
|
* Source-applied writes are exempt for the same reason they are exempt from the attempt cap:
|
|
2456
3322
|
* there is no resubscribe/sequence-resume path, so dropping one permanently diverges this node.
|
|
2457
3323
|
*/
|
|
2458
|
-
private elapsedPastCommitBudget(): number {
|
|
3324
|
+
private elapsedPastCommitBudget(transaction?: RocksTransaction): number {
|
|
2459
3325
|
if (this.sourceApply) return 0;
|
|
2460
|
-
const startedAt =
|
|
3326
|
+
const startedAt = this.budgetCarrier(transaction).commitStartedAt;
|
|
2461
3327
|
if (startedAt == null) return 0;
|
|
2462
3328
|
const elapsed = performance.now() - startedAt;
|
|
2463
3329
|
return elapsed > this.commitConflictBudget() ? elapsed : 0;
|
|
@@ -2478,13 +3344,18 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2478
3344
|
const elapsed = Math.round(elapsedMs);
|
|
2479
3345
|
const budget = this.commitConflictBudget();
|
|
2480
3346
|
const retryable = !this.root && !this.snapshotFree;
|
|
3347
|
+
const retries = (headTransaction as RocksTransactionWithRetry | undefined)?.retries ?? this.retries;
|
|
2481
3348
|
if (allowStuckCommitLog('abandon', performance.now())) {
|
|
2482
|
-
|
|
3349
|
+
let headTransactionId;
|
|
3350
|
+
try {
|
|
3351
|
+
headTransactionId = headTransaction?.id;
|
|
3352
|
+
} catch {}
|
|
3353
|
+
harperLogger.error?.(
|
|
2483
3354
|
`Abandoning a write transaction: its commit has been in write-intent conflict for ${elapsed}ms ` +
|
|
2484
|
-
`(exceeds the ${budget}ms limit) across ${
|
|
3355
|
+
`(exceeds the ${budget}ms limit) across ${retries} retries, ` +
|
|
2485
3356
|
describeCommitIdentity(this.writes[0]?.store, this.startedFrom, headTransaction) +
|
|
2486
3357
|
`.` +
|
|
2487
|
-
describeHolderCandidates(this.writes[0]?.store?.rootStore?.path,
|
|
3358
|
+
describeHolderCandidates(this.writes[0]?.store?.rootStore?.path, headTransactionId) +
|
|
2488
3359
|
` Another transaction holds a conflicting write intent and has not completed; the request is ` +
|
|
2489
3360
|
`failed with a 503${retryable ? '' : ' (not retryable — an earlier store in this transaction already committed)'} ` +
|
|
2490
3361
|
`rather than waiting further.`
|
|
@@ -2492,11 +3363,41 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2492
3363
|
}
|
|
2493
3364
|
this.abortChainAfterRetries(headTransaction);
|
|
2494
3365
|
throw new TransactionCommitConflictTimeoutError(
|
|
2495
|
-
`Commit was in conflict with ongoing writes for ${elapsed}ms, exceeding the ${budget}ms limit; transaction abandoned after ${
|
|
3366
|
+
`Commit was in conflict with ongoing writes for ${elapsed}ms, exceeding the ${budget}ms limit; transaction abandoned after ${retries} retries`,
|
|
2496
3367
|
retryable
|
|
2497
3368
|
);
|
|
2498
3369
|
}
|
|
2499
3370
|
|
|
3371
|
+
/**
|
|
3372
|
+
* A durably logged commit past its conflict budget cannot be aborted; release its deferred
|
|
3373
|
+
* record-lock holds (`holdBudgetSpent` prevents re-retaining them) and log once per chain/cooldown.
|
|
3374
|
+
*/
|
|
3375
|
+
private noteOverdueLoggedCommit(headTransaction: RocksTransaction, elapsedMs: number): void {
|
|
3376
|
+
(headTransaction as RocksTransactionWithRetry).holdBudgetSpent = true;
|
|
3377
|
+
releaseTransactionLockHolds(headTransaction);
|
|
3378
|
+
const carrier = this.budgetCarrier(headTransaction);
|
|
3379
|
+
if (carrier.commitOverdueLogged) return;
|
|
3380
|
+
if (!allowStuckCommitLog('overdue', performance.now())) return;
|
|
3381
|
+
carrier.commitOverdueLogged = true;
|
|
3382
|
+
const elapsed = Math.round(elapsedMs);
|
|
3383
|
+
const budget = this.commitConflictBudget();
|
|
3384
|
+
const retries = (headTransaction as RocksTransactionWithRetry | undefined)?.retries ?? this.retries;
|
|
3385
|
+
let headTransactionId;
|
|
3386
|
+
try {
|
|
3387
|
+
headTransactionId = headTransaction?.id;
|
|
3388
|
+
} catch {}
|
|
3389
|
+
harperLogger.error?.(
|
|
3390
|
+
`A durably logged write transaction is still in write-intent conflict after ${elapsed}ms ` +
|
|
3391
|
+
`(exceeds the ${budget}ms limit) across ${retries} retries, ` +
|
|
3392
|
+
describeCommitIdentity(this.writes[0]?.store, this.startedFrom, headTransaction) +
|
|
3393
|
+
`.` +
|
|
3394
|
+
describeHolderCandidates(this.writes[0]?.store?.rootStore?.path, headTransactionId) +
|
|
3395
|
+
` Because its transaction-log entry is already durable on disk, it cannot be aborted and ` +
|
|
3396
|
+
`will continue retrying until the conflicting write intent releases (pinning the local ` +
|
|
3397
|
+
`transaction-log read watermark for subscriptions and replication while uncommitted).`
|
|
3398
|
+
);
|
|
3399
|
+
}
|
|
3400
|
+
|
|
2500
3401
|
/**
|
|
2501
3402
|
* Give up on a chain of linked transactions after exhausting conflict retries: poison every link
|
|
2502
3403
|
* first, then abort each link's native transaction and release its DatabaseTransaction-level
|
|
@@ -2508,34 +3409,67 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2508
3409
|
* from the replay transaction; retry exhaustion aborts both.
|
|
2509
3410
|
*/
|
|
2510
3411
|
abortChainAfterRetries(headTransaction: RocksTransaction): void {
|
|
3412
|
+
const isImmediateCommit = Boolean((headTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
3413
|
+
const hasSiblingInFlight = this.hasSiblingCommitInFlight(headTransaction);
|
|
3414
|
+
if (
|
|
3415
|
+
isImmediateCommit ||
|
|
3416
|
+
(hasSiblingInFlight && this.hasWriteOwnedByActiveCommit(headTransaction, hasSiblingInFlight, true, true))
|
|
3417
|
+
) {
|
|
3418
|
+
if (!isImmediateCommit && this.transaction === headTransaction) this.detachOwnedTransaction();
|
|
3419
|
+
releaseTransactionLockHolds(headTransaction);
|
|
3420
|
+
abortNativeTransaction(
|
|
3421
|
+
headTransaction,
|
|
3422
|
+
'aborting conflicted transaction in chain after exhausting retries',
|
|
3423
|
+
this.writes[0]?.store ?? this.db,
|
|
3424
|
+
this.startedFrom
|
|
3425
|
+
);
|
|
3426
|
+
try {
|
|
3427
|
+
this.abort(false, false, headTransaction);
|
|
3428
|
+
} catch (abortError) {
|
|
3429
|
+
harperLogger.debug?.('cleaning up conflicted transaction in chain after exhausting retries', abortError);
|
|
3430
|
+
}
|
|
3431
|
+
return;
|
|
3432
|
+
}
|
|
2511
3433
|
for (let txn: DatabaseTransaction = this; txn; txn = txn.next) {
|
|
2512
3434
|
txn.open = TRANSACTION_STATE.CLOSED;
|
|
2513
3435
|
}
|
|
2514
|
-
for (let txn: DatabaseTransaction = this; txn; txn =
|
|
3436
|
+
for (let txn: DatabaseTransaction = this, next: DatabaseTransaction; txn; txn = next) {
|
|
3437
|
+
next = txn.next;
|
|
3438
|
+
txn.next = null;
|
|
2515
3439
|
// Detach first so the abort() below performs only non-native cleanup, and so its doneReadTxn
|
|
2516
3440
|
// loop cannot spin on a nulled handle. Not to avoid a double abort: rocksdb-js tolerates
|
|
2517
3441
|
// abort-after-abort, and it is abort-after-COMMIT that throws.
|
|
2518
|
-
const detached =
|
|
3442
|
+
const detached =
|
|
3443
|
+
txn === this &&
|
|
3444
|
+
hasSiblingInFlight &&
|
|
3445
|
+
txn.transaction &&
|
|
3446
|
+
txn.transaction !== headTransaction &&
|
|
3447
|
+
(this.isActiveCommitHandle(txn.transaction) || hasDurableLog(txn.transaction))
|
|
3448
|
+
? null
|
|
3449
|
+
: txn.detachOwnedTransaction();
|
|
2519
3450
|
const committingTransaction = txn === this ? headTransaction : detached;
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
3451
|
+
releaseTransactionLockHolds(committingTransaction);
|
|
3452
|
+
abortNativeTransaction(
|
|
3453
|
+
committingTransaction,
|
|
3454
|
+
'aborting conflicted transaction in chain after exhausting retries',
|
|
3455
|
+
txn.writes[0]?.store ?? txn.db,
|
|
3456
|
+
txn.startedFrom
|
|
3457
|
+
);
|
|
2525
3458
|
// With outstanding iterators, the head owns a retained read handle while the retry commits
|
|
2526
3459
|
// through a separate replay handle. Both must be aborted after retry exhaustion.
|
|
2527
3460
|
if (txn === this && detached && detached !== committingTransaction) {
|
|
2528
|
-
|
|
2529
|
-
detached
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
3461
|
+
abortNativeTransaction(
|
|
3462
|
+
detached,
|
|
3463
|
+
'aborting retained read transaction after exhausting retries',
|
|
3464
|
+
txn.writes[0]?.store ?? txn.db,
|
|
3465
|
+
txn.startedFrom
|
|
3466
|
+
);
|
|
2533
3467
|
}
|
|
2534
3468
|
try {
|
|
2535
3469
|
// abort() synchronously walks savedBlobs and can call write.store.getEntry(), which can throw
|
|
2536
3470
|
// (closed store, decode error). Catch and continue so one link's wrapper-cleanup failure can't
|
|
2537
3471
|
// strand later links' native handles — they were already detached/aborted above regardless.
|
|
2538
|
-
txn.abort();
|
|
3472
|
+
txn.abort(false, false, committingTransaction);
|
|
2539
3473
|
} catch (abortError) {
|
|
2540
3474
|
harperLogger.debug?.('cleaning up conflicted transaction in chain after exhausting retries', abortError);
|
|
2541
3475
|
}
|
|
@@ -2621,15 +3555,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2621
3555
|
*/
|
|
2622
3556
|
poisonAfterStalledSubmittedCommit(): void {
|
|
2623
3557
|
const root = this.root ?? this;
|
|
2624
|
-
const
|
|
2625
|
-
const collect = (start: DatabaseTransaction | undefined) => {
|
|
2626
|
-
for (let txn = start; txn; txn = txn.next) links.add(txn);
|
|
2627
|
-
};
|
|
2628
|
-
collect(root);
|
|
2629
|
-
collect(this);
|
|
2630
|
-
collect(root.submittedLink);
|
|
2631
|
-
if (root.submittedLinks) for (const txn of root.submittedLinks) collect(txn);
|
|
2632
|
-
for (const txn of links) {
|
|
3558
|
+
for (const txn of collectCommitLinks(root, this)) {
|
|
2633
3559
|
txn.postSubmitPoisoned = true;
|
|
2634
3560
|
// Not on a link an earlier poison already aborted: abort() cleared its write set, so excusing
|
|
2635
3561
|
// its continuation from the poison check would let it resume onto nothing and report success.
|
|
@@ -2650,7 +3576,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2650
3576
|
// throws. abort() rather than the native abort alone: it reclaims blobs a replayed write
|
|
2651
3577
|
// staged.
|
|
2652
3578
|
this.detachOwnedTransaction();
|
|
2653
|
-
abortNativeTransaction(
|
|
3579
|
+
abortNativeTransaction(
|
|
3580
|
+
transaction,
|
|
3581
|
+
'aborting a transaction whose synchronous commit failed',
|
|
3582
|
+
this.writes[0]?.store ?? this.db,
|
|
3583
|
+
this.startedFrom
|
|
3584
|
+
);
|
|
2654
3585
|
try {
|
|
2655
3586
|
this.abort();
|
|
2656
3587
|
} catch (abortError) {
|
|
@@ -2700,10 +3631,11 @@ export class ImmediateTransaction extends DatabaseTransaction {
|
|
|
2700
3631
|
}
|
|
2701
3632
|
save(...args: any[]): any {
|
|
2702
3633
|
const operation = args[0]; // the staged write, not a transaction — commit() re-enters here with it
|
|
2703
|
-
if (
|
|
3634
|
+
if (operation?.deferSave) operation.deferSave = false;
|
|
3635
|
+
if (this.isCommitting || args[1] || args[3]) {
|
|
2704
3636
|
// Stage into the transaction commit() is committing (on a retry round, the one being retried)
|
|
2705
3637
|
// and reload within it; with no handle super.save() would open its own and nest a commit().
|
|
2706
|
-
super.save(operation, args[1], true);
|
|
3638
|
+
super.save(operation, args[1], true, args[3]);
|
|
2707
3639
|
} else {
|
|
2708
3640
|
this.admitRequestWrite(); // a deferred save reaches here without passing through addWrite()
|
|
2709
3641
|
this.isCommitting = true;
|
|
@@ -2738,7 +3670,7 @@ export class ImmediateTransaction extends DatabaseTransaction {
|
|
|
2738
3670
|
const recordLocks = this.recordLocks;
|
|
2739
3671
|
if (!recordLocks) return;
|
|
2740
3672
|
for (const [store, storeMap] of recordLocks) {
|
|
2741
|
-
for (const [keyId, handle] of storeMap) if (handle.released) storeMap.delete(keyId);
|
|
3673
|
+
for (const [keyId, handle] of storeMap) if (handle.released || handle.isExpired()) storeMap.delete(keyId);
|
|
2742
3674
|
if (storeMap.size === 0) recordLocks.delete(store);
|
|
2743
3675
|
}
|
|
2744
3676
|
if (recordLocks.size === 0) this.recordLocks = undefined;
|
|
@@ -2907,6 +3839,13 @@ function reportIfLongLived(
|
|
|
2907
3839
|
/** Diagnostic grace after a native submission outlives the ordinary transaction timeout. */
|
|
2908
3840
|
const MAX_DEFERRED_POISON_TICKS = 2;
|
|
2909
3841
|
const STALLED_COMMIT_LOG_MIN_INTERVAL_MS = 1000;
|
|
3842
|
+
// A logged retry parked on a write intent moves its retry counter at least every native park timeout,
|
|
3843
|
+
// so a counter that has not moved for two of them is a native attempt that never settled — whatever
|
|
3844
|
+
// the monitor's tick interval (STORAGE_MAXTRANSACTIONOPENTIME can be set far below the park timeout).
|
|
3845
|
+
const configuredParkTimeoutMs = Number(process.env.ROCKSDB_JS_PARK_TIMEOUT_MS);
|
|
3846
|
+
// Same acceptance rule as rocksdb-js: anything but a finite positive number falls back to its 5000 ms default.
|
|
3847
|
+
const LOGGED_RETRY_STALL_MS =
|
|
3848
|
+
2 * (Number.isFinite(configuredParkTimeoutMs) && configuredParkTimeoutMs > 0 ? configuredParkTimeoutMs : 5000);
|
|
2910
3849
|
let lastStalledCommitWarningAt = -Infinity;
|
|
2911
3850
|
let lastStalledCommitErrorAt = -Infinity;
|
|
2912
3851
|
|
|
@@ -2920,6 +3859,28 @@ function submittedCommitStillInFlight(root: DatabaseTransaction): boolean {
|
|
|
2920
3859
|
return false;
|
|
2921
3860
|
}
|
|
2922
3861
|
|
|
3862
|
+
// Every link the monitor would poison for a stalled commit, so the two walks agree on which links count.
|
|
3863
|
+
function collectCommitLinks(root: DatabaseTransaction, txn: DatabaseTransaction): Set<DatabaseTransaction> {
|
|
3864
|
+
const links = new Set<DatabaseTransaction>();
|
|
3865
|
+
const collect = (start: DatabaseTransaction | undefined) => {
|
|
3866
|
+
for (let link = start; link; link = link.next) links.add(link);
|
|
3867
|
+
};
|
|
3868
|
+
collect(root);
|
|
3869
|
+
collect(txn);
|
|
3870
|
+
collect(root.submittedLink);
|
|
3871
|
+
if (root.submittedLinks) for (const link of root.submittedLinks) collect(link);
|
|
3872
|
+
return links;
|
|
3873
|
+
}
|
|
3874
|
+
|
|
3875
|
+
function chainDurableLogRetries(root: DatabaseTransaction, txn: DatabaseTransaction): number {
|
|
3876
|
+
let retries = -1;
|
|
3877
|
+
for (const link of collectCommitLinks(root, txn)) {
|
|
3878
|
+
const linkRetries = link.activeDurableLogRetries();
|
|
3879
|
+
if (linkRetries >= 0) retries = Math.max(retries, 0) + linkRetries;
|
|
3880
|
+
}
|
|
3881
|
+
return retries;
|
|
3882
|
+
}
|
|
3883
|
+
|
|
2923
3884
|
/**
|
|
2924
3885
|
* True when this tick must leave `txn` alone because some link has submitted writes whose native
|
|
2925
3886
|
* outcome is not known yet. Merely entering commit() is not enough: a wholly pre-submit transaction
|
|
@@ -2928,7 +3889,8 @@ function submittedCommitStillInFlight(root: DatabaseTransaction): boolean {
|
|
|
2928
3889
|
export function deferForCommitInFlight(
|
|
2929
3890
|
txn: DatabaseTransaction,
|
|
2930
3891
|
url: string | undefined,
|
|
2931
|
-
expiration = txnExpiration
|
|
3892
|
+
expiration = txnExpiration,
|
|
3893
|
+
tick = monitorTick
|
|
2932
3894
|
): boolean {
|
|
2933
3895
|
const root = txn.root ?? txn;
|
|
2934
3896
|
if (!txn.isChainCommitting() || !root.commitSubmitted) return false;
|
|
@@ -2940,23 +3902,34 @@ export function deferForCommitInFlight(
|
|
|
2940
3902
|
// diagnostic grace, nor report a legitimately slow one early.
|
|
2941
3903
|
const now = performance.now();
|
|
2942
3904
|
const deadline = (root.deferredPoisonDeadline ??= now + MAX_DEFERRED_POISON_TICKS * expiration);
|
|
3905
|
+
// A handle with a durable log entry retrying a conflict has a known outcome — it commits once the
|
|
3906
|
+
// conflicting intent clears — so while its retry counter keeps advancing it is neither stalled nor
|
|
3907
|
+
// poisoned, and noteOverdueLoggedCommit is the operator's signal. A counter that has not moved for
|
|
3908
|
+
// LOGGED_RETRY_STALL_MS is a native attempt that never settled, which is the stall handled below.
|
|
3909
|
+
const loggedRetries = chainDurableLogRetries(root, txn);
|
|
3910
|
+
let loggedRetry = false;
|
|
3911
|
+
if (loggedRetries >= 0) {
|
|
3912
|
+
if (root.loggedRetriesTick === tick) {
|
|
3913
|
+
loggedRetry = Boolean(root.loggedRetryAdvancing);
|
|
3914
|
+
} else {
|
|
3915
|
+
if (root.loggedRetriesSeen !== loggedRetries) {
|
|
3916
|
+
root.loggedRetriesSeen = loggedRetries;
|
|
3917
|
+
root.loggedRetriesChangedAt = now;
|
|
3918
|
+
}
|
|
3919
|
+
loggedRetry = now - root.loggedRetriesChangedAt < LOGGED_RETRY_STALL_MS;
|
|
3920
|
+
root.loggedRetriesTick = tick;
|
|
3921
|
+
root.loggedRetryAdvancing = loggedRetry;
|
|
3922
|
+
}
|
|
3923
|
+
}
|
|
2943
3924
|
let hasUnsubmittedCommit = false;
|
|
2944
3925
|
let unsubmittedCommitGraceExpired = false;
|
|
2945
3926
|
if (now >= deadline) {
|
|
2946
3927
|
// Only reclaim after the commit has genuinely stalled. Submitted writes no longer depend on these
|
|
2947
3928
|
// separate snapshots; pre-submit handles can still carry writes needed by the attempt.
|
|
2948
3929
|
txn.releaseStalledCommitReadResources();
|
|
2949
|
-
if (!txn.isProtectedCommit()) {
|
|
3930
|
+
if (!loggedRetry && !txn.isProtectedCommit()) {
|
|
2950
3931
|
if (!root.postSubmitPoisoned) txn.poisonAfterStalledSubmittedCommit();
|
|
2951
|
-
const
|
|
2952
|
-
const collect = (start: DatabaseTransaction | undefined) => {
|
|
2953
|
-
for (let link = start; link; link = link.next) links.add(link);
|
|
2954
|
-
};
|
|
2955
|
-
collect(root);
|
|
2956
|
-
collect(txn);
|
|
2957
|
-
collect(root.submittedLink);
|
|
2958
|
-
if (root.submittedLinks) for (const link of root.submittedLinks) collect(link);
|
|
2959
|
-
for (const link of links) {
|
|
3932
|
+
for (const link of collectCommitLinks(root, txn)) {
|
|
2960
3933
|
if (link.committing && !link.nativeCommitSubmitted) {
|
|
2961
3934
|
hasUnsubmittedCommit = true;
|
|
2962
3935
|
break;
|
|
@@ -2976,7 +3949,11 @@ export function deferForCommitInFlight(
|
|
|
2976
3949
|
!(txn === root && submittedCommitStillInFlight(root))
|
|
2977
3950
|
)
|
|
2978
3951
|
return hasUnsubmittedCommit && !unsubmittedCommitGraceExpired;
|
|
2979
|
-
if (
|
|
3952
|
+
if (
|
|
3953
|
+
!loggedRetry &&
|
|
3954
|
+
!root.stalledCommitLogged &&
|
|
3955
|
+
now - lastStalledCommitErrorAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS
|
|
3956
|
+
) {
|
|
2980
3957
|
lastStalledCommitErrorAt = now;
|
|
2981
3958
|
root.stalledCommitLogged = true;
|
|
2982
3959
|
harperLogger.error(
|
|
@@ -2988,7 +3965,7 @@ export function deferForCommitInFlight(
|
|
|
2988
3965
|
}`
|
|
2989
3966
|
);
|
|
2990
3967
|
}
|
|
2991
|
-
} else if (now - lastStalledCommitWarningAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS) {
|
|
3968
|
+
} else if (!loggedRetry && now - lastStalledCommitWarningAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS) {
|
|
2992
3969
|
lastStalledCommitWarningAt = now;
|
|
2993
3970
|
harperLogger.warn?.(
|
|
2994
3971
|
`Transaction exceeded the open-transaction limit after submitting a native commit; preserving its unknown outcome, from table: ${
|
|
@@ -3170,11 +4147,23 @@ export function setMaxOutstandingTxnDuration(ms: number): number {
|
|
|
3170
4147
|
return previous;
|
|
3171
4148
|
}
|
|
3172
4149
|
|
|
4150
|
+
/** Test seam: overrides the per-retry backoff cap and returns the previous value. */
|
|
4151
|
+
export function setMaxRetryDelayMs(ms: number): number {
|
|
4152
|
+
const previous = MAX_RETRY_DELAY_MS;
|
|
4153
|
+
MAX_RETRY_DELAY_MS = ms;
|
|
4154
|
+
return previous;
|
|
4155
|
+
}
|
|
4156
|
+
|
|
3173
4157
|
/** Test seam: whether the monitor supervises this logical transaction for its writes. */
|
|
3174
4158
|
export function isWriteSupervised(txn: DatabaseTransaction): boolean {
|
|
3175
4159
|
return supervisedWriteRoots.has(txn);
|
|
3176
4160
|
}
|
|
3177
4161
|
|
|
4162
|
+
/** Test seam: returns the current open-transaction monitor interval. */
|
|
4163
|
+
export function getTxnExpiration(): number {
|
|
4164
|
+
return txnExpiration;
|
|
4165
|
+
}
|
|
4166
|
+
|
|
3178
4167
|
export function setTxnExpiration(ms) {
|
|
3179
4168
|
clearInterval(timer);
|
|
3180
4169
|
txnExpiration = ms;
|