@harperfast/harper 5.3.2 → 5.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cliOperations.ts +3 -3
- package/bin/copyDb.ts +4 -3
- package/components/DESIGN.md +12 -0
- package/components/packageComponent.ts +42 -17
- package/config/DESIGN.md +4 -0
- package/config/configUtils.ts +8 -8
- package/config/parseConfigFile.ts +11 -7
- package/dist/bin/cliOperations.js +3 -3
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/copyDb.js +3 -3
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/components/packageComponent.d.ts +9 -7
- package/dist/components/packageComponent.js +41 -16
- package/dist/components/packageComponent.js.map +1 -1
- package/dist/config/configUtils.js +4 -3
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/parseConfigFile.d.ts +1 -0
- package/dist/config/parseConfigFile.js +7 -3
- package/dist/config/parseConfigFile.js.map +1 -1
- package/dist/resources/DatabaseTransaction.d.ts +75 -10
- package/dist/resources/DatabaseTransaction.js +1080 -277
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/Resource.js +40 -14
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.js +1 -0
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.js +16 -2
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/write.js +34 -16
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/databases.d.ts +5 -0
- package/dist/resources/databases.js +38 -4
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/recordLock.d.ts +9 -4
- package/dist/resources/recordLock.js +79 -19
- package/dist/resources/recordLock.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +2 -0
- package/dist/server/DurableSubscriptionsSession.js +20 -17
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/mqtt.js +41 -23
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/serverHelpers/NodeAdapterResponse.js +18 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -1
- package/dist/utility/npmUtilities.js +18 -7
- package/dist/utility/npmUtilities.js.map +1 -1
- package/package.json +1 -1
- package/resources/DESIGN.md +42 -10
- package/resources/DatabaseTransaction.ts +1244 -275
- package/resources/Resource.ts +39 -14
- package/resources/RocksTransactionLogStore.ts +1 -0
- package/resources/Table.ts +24 -2
- package/resources/analytics/DESIGN.md +17 -0
- package/resources/analytics/write.ts +28 -15
- package/resources/databases.ts +40 -4
- package/resources/record-locks.md +34 -17
- package/resources/recordLock.ts +89 -22
- package/server/DESIGN.md +2 -0
- package/server/DurableSubscriptionsSession.ts +18 -16
- package/server/mqtt.ts +39 -22
- package/server/serverHelpers/NodeAdapterResponse.ts +19 -1
- package/studio/web/assets/Chat-B-Rd-ft9.js +2100 -0
- package/studio/web/assets/FloatingChat-CFypO7kh.js +23 -0
- package/studio/web/assets/SearchDialog-BRIyPiBl.js +1 -0
- package/studio/web/assets/{abnfDiagram-O67JEVCF-raXE027J.js → abnfDiagram-YUKMZFIV-DPR2lFRk.js} +1 -1
- package/studio/web/assets/{apiToken-DNrfij0E.js → apiToken-BraYnzvC.js} +1 -1
- package/studio/web/assets/applications-XCds8-ZP.js +296 -0
- package/studio/web/assets/architecture-WOLXFQ4H-DtJHIK0_.js +1 -0
- package/studio/web/assets/{architectureDiagram-NJMV4G6O-C8_Eet7w.js → architectureDiagram-47P4ROYG-MDymp2dh.js} +3 -3
- package/studio/web/assets/authStore-DE1HFvVP.js +3 -0
- package/studio/web/assets/{blockDiagram-BEXU5L5S-DR6S5-1D.js → blockDiagram-E7TT5TSB-O5qRgZmd.js} +3 -3
- package/studio/web/assets/button-C37w80Uk.js +21 -0
- package/studio/web/assets/{c4Diagram-YGBWAQC7-BAfZ1G01.js → c4Diagram-CRA5TL53-BdPORGHo.js} +2 -2
- package/studio/web/assets/channel-yCStoq0_.js +1 -0
- package/studio/web/assets/{chevron-up-CFxbMGv2.js → chevron-up-DozwrIz-.js} +1 -1
- package/studio/web/assets/chunk-2BW5OAIV-DJ-iXJAw.js +10 -0
- package/studio/web/assets/{chunk-ZIGJFQKS-Cs8fUn7y.js → chunk-3YJQHVM4-BOgacqS1.js} +2 -2
- package/studio/web/assets/{chunk-7PRAP22T-KyqX6x3f.js → chunk-4EA7E6EY-C-7xqUBl.js} +1 -1
- package/studio/web/assets/{chunk-GWA4HPMP-DsejB6gA.js → chunk-7TKQ45FW-C3XA1bIN.js} +1 -1
- package/studio/web/assets/{chunk-GTNCS2PH-DGHLXyow.js → chunk-AW2ZBBNX-BdvgD1UW.js} +3 -3
- package/studio/web/assets/{chunk-5DYCD2WN-BceBfIHT.js → chunk-DBDB3WZW-CKKxIc6F.js} +1 -1
- package/studio/web/assets/{chunk-MBY4JIJT-BwSGweQP.js → chunk-E2ZNV5FY-bVefNHvz.js} +3 -3
- package/studio/web/assets/chunk-EU5HNXII-BHv2kWp4.js +213 -0
- package/studio/web/assets/chunk-FVRAUYC3-BoL_NO6b.js +2 -0
- package/studio/web/assets/{chunk-742MDFTN-CIkt8ijt.js → chunk-HJ2JQQFS-N_98oMRS.js} +1 -1
- package/studio/web/assets/{chunk-NETBCI7D-C8UnOw_f.js → chunk-HTAEGDNF-BQ1fSjPs.js} +1 -1
- package/studio/web/assets/{chunk-WEXAMYUT-BAIhTwyh.js → chunk-KQW6MTUR-qP0QiQAg.js} +1 -1
- package/studio/web/assets/chunk-NGNAAXSQ-DqoPQk1a.js +136 -0
- package/studio/web/assets/{chunk-UA2S7LBM-BHpEdH8D.js → chunk-NTY3LDVX-CSBFjlJg.js} +1 -1
- package/studio/web/assets/chunk-VPRB5NB3-BVHOe9Q-.js +129 -0
- package/studio/web/assets/{chunk-7INBJB4K-BpeAXyOi.js → chunk-XC4XBNZT-46uQwtF4.js} +10 -10
- package/studio/web/assets/{classDiagram-v2-NBCMYWYE-DmDPkt2u.js → classDiagram-v2-K4WV4PDN-Da0ps1Ui.js} +3 -3
- package/studio/web/assets/cn-DpF1PxBT.js +1 -0
- package/studio/web/assets/{createLucideIcon-fO5dL7D5.js → createLucideIcon-DLWRoWN-.js} +1 -1
- package/studio/web/assets/{cssMode-DBucrQdT.js → cssMode-BY1w91l8.js} +1 -1
- package/studio/web/assets/cynefin-EF2NZ3EQ-jQ2eCUaU.js +1 -0
- package/studio/web/assets/cynefinDiagram-3GCD6N5R-TNi5iS0E.js +62 -0
- package/studio/web/assets/dagre-W4DXFKR2-BnLHmA44.js +4 -0
- package/studio/web/assets/{diagram-22UHCM2B-DJCvcaWP.js → diagram-2UJZ2QOL-r_jehhZR.js} +7 -7
- package/studio/web/assets/{diagram-3UASUU5V-Du6VKWln.js → diagram-OVF4WLC6-BSUXGVh8.js} +11 -11
- package/studio/web/assets/{diagram-CDSNMT55-l5Ga0u-W.js → diagram-PFPMY2P6-BrVedycw.js} +3 -3
- package/studio/web/assets/{diagram-MLGK6HIB-CNQY47LA.js → diagram-UMYRVEAY-CBIaakBB.js} +1 -1
- package/studio/web/assets/diagram-YEKJPTXX-DFmmDXCl.js +41 -0
- package/studio/web/assets/diagram-ZIFT7M5P-CQfOWxOi.js +3 -0
- package/studio/web/assets/dist-9lpzt2Hn.js +11 -0
- package/studio/web/assets/{ebnfDiagram-ZINNZB2B-5OnkTpfi.js → ebnfDiagram-VR2GEFS7-FXA3Xe6e.js} +1 -1
- package/studio/web/assets/{editor-CHpQCuBq.js → editor-D5mpVs1b.js} +1 -1
- package/studio/web/assets/elk-IJKZMXRS-CC4gfVRU.js +27 -0
- package/studio/web/assets/{erDiagram-OPXOYQCR-CQlCX84D.js → erDiagram-O2IAPWRE-C-BkDTPl.js} +2 -2
- package/studio/web/assets/errorText-8xV8WZBt.js +1 -0
- package/studio/web/assets/eventmodeling-K75KTNOO-D0dJ_PbR.js +1 -0
- package/studio/web/assets/flowDiagram-OXPTDLAJ-C1LEc_Hf.js +1 -0
- package/studio/web/assets/{ganttDiagram-FUAMR5RP-bloX3Voy.js → ganttDiagram-R7TSEDQI-85F44BIo.js} +4 -4
- package/studio/web/assets/{getAnalytics-C0FHjtdQ.js → getAnalytics-B1xpd4kG.js} +1 -1
- package/studio/web/assets/gitGraph-VSP46ZUC-BBmqrmjK.js +1 -0
- package/studio/web/assets/gitGraphDiagram-XJZIOB7I-CFY6iMwg.js +106 -0
- package/studio/web/assets/graphlib-DS17s2tU.js +1 -0
- package/studio/web/assets/{html-B1iA8q38.js → html-DiXKiahz.js} +1 -1
- package/studio/web/assets/{htmlMode-CrL3YO5Y.js → htmlMode-DitVJ707.js} +1 -1
- package/studio/web/assets/index-CDnn0-PN.css +1 -0
- package/studio/web/assets/index-D8kyF6dt.js +833 -0
- package/studio/web/assets/index.lazy-DgbjUxLo.js +1 -0
- package/studio/web/assets/info-OHQRW6UA-Bs2aEco7.js +1 -0
- package/studio/web/assets/{infoDiagram-VRGFBTTK-B0J1132p.js → infoDiagram-5W2HQ5XZ-d6yG1yfZ.js} +1 -1
- package/studio/web/assets/invalidateEntityQueries-BkWXiqjz.js +1 -0
- package/studio/web/assets/{ishikawaDiagram-OU5B5YK6-BNk0tuQx.js → ishikawaDiagram-K3B6WC7H-CSYz-WRA.js} +2 -2
- package/studio/web/assets/{javascript-CXgQMkoa.js → javascript-BBVXeMDm.js} +1 -1
- package/studio/web/assets/{journeyDiagram-ZHPQQLJL-Be79RMQM.js → journeyDiagram-COXZFDF6-BDj6Wqcj.js} +3 -3
- package/studio/web/assets/{jsonMode-DycqRKOv.js → jsonMode-D85_j2oD.js} +1 -1
- package/studio/web/assets/{kanban-definition-PNTS6WVX-4uuoQP9W.js → kanban-definition-P3RFRI5V-C2Bjfbp-.js} +6 -6
- package/studio/web/assets/{languageServices-HYHV1wDJ.js → languageServices-U3mWhpRL.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-CH_6SYiA.js → lspLanguageFeatures-CoymyM3c.js} +1 -1
- package/studio/web/assets/{mermaid-parser.core-V1UYMeXV.js → mermaid-parser.core-Ch_T61iL.js} +6 -6
- package/studio/web/assets/mermaid.core-BOAgKzo5.js +44 -0
- package/studio/web/assets/{mindmap-definition-NLK3R4M7-C70Ctm0J.js → mindmap-definition-LZFPQGKD-DzOapzMM.js} +3 -3
- package/studio/web/assets/notifications-CPgEC6ZP.js +1 -0
- package/studio/web/assets/notifications-D_VqhHnx.js +1 -0
- package/studio/web/assets/operations-D8aaKYty.js +4 -0
- package/studio/web/assets/packet-JDAUHWVQ-CGHB_EIE.js +1 -0
- package/studio/web/assets/{pegDiagram-GJSIUBJH-d6z1cc7W.js → pegDiagram-BGZESJAR-ADzOIKrz.js} +1 -1
- package/studio/web/assets/pie-XZMESJXO-Bijz5Y2o.js +1 -0
- package/studio/web/assets/pieDiagram-CAPJLFHJ-VM7fvbwO.js +39 -0
- package/studio/web/assets/profile-DM5E6Xls.js +1 -0
- package/studio/web/assets/{quadrantDiagram-O4NWA36T-CIF5Qkou.js → quadrantDiagram-GDMTTRAM-Bmza84_j.js} +3 -3
- package/studio/web/assets/radar-ABXABTNO-IaC8pB63.js +1 -0
- package/studio/web/assets/railroad-I3PHUGI6-CCrSUqeq.js +1 -0
- package/studio/web/assets/railroad-abnf-LNEFI6M7-BhobVp-t.js +1 -0
- package/studio/web/assets/railroad-ebnf-DWYD2UWJ-OHhGUTEh.js +1 -0
- package/studio/web/assets/railroad-peg-JR7BVNR7-FupC3Jcn.js +1 -0
- package/studio/web/assets/{railroadDiagram-XR7U4H2S-y3Jv2U6N.js → railroadDiagram-SM67HX2B-CgFGOVI5.js} +1 -1
- package/studio/web/assets/regions-WFvXqryO.js +1 -0
- package/studio/web/assets/{register-DCN4QUii.js → register-B7DvfAGj.js} +3 -3
- package/studio/web/assets/request-ChoMERRk.js +2 -0
- package/studio/web/assets/{requirementDiagram-PLB6GJNP-ChINZEmu.js → requirementDiagram-X7JNWC4A-BHoQgkoT.js} +4 -4
- package/studio/web/assets/restartGate-D_e93lFE.js +1 -0
- package/studio/web/assets/restartUnconfirmed-BqcDZ__Y.js +1 -0
- package/studio/web/assets/sankeyDiagram-UM26HJRW-8tFu_oUe.js +40 -0
- package/studio/web/assets/sequenceDiagram-ZO4K6R2Y-jryb1rwT.js +169 -0
- package/studio/web/assets/setComponentFile-BHk22Rh6.js +1 -0
- package/studio/web/assets/setup-wNdpTrVH.js +2 -0
- package/studio/web/assets/{stateDiagram-v2-GCMORJYK-Bs8RIb9K.js → stateDiagram-v2-TBUQTH76-CnWE5yFn.js} +2 -2
- package/studio/web/assets/status-CYoonm9V.js +61 -0
- package/studio/web/assets/{swimlanes-2SLR337P-B52xrf6t.js → swimlanes-N4OXWK64-CBlFzDJn.js} +1 -1
- package/studio/web/assets/swimlanesDiagram-K3J5GTZL-DvFLtUPi.js +8 -0
- package/studio/web/assets/{timeline-definition-EJHVYXUP-DJSITpQX.js → timeline-definition-YOQAKGHF-CFWqHrqI.js} +4 -4
- package/studio/web/assets/{toggleHighContrast-CfAp6Sxb.js → toggleHighContrast-CKEm-P4K.js} +1 -1
- package/studio/web/assets/treeView-D4JQ5SDB-CpV9zuVb.js +1 -0
- package/studio/web/assets/treemap-SAJKECNS-Bc94Tqiq.js +1 -0
- package/studio/web/assets/{tsMode-HZR0bfq4.js → tsMode-Dv35qHmn.js} +1 -1
- package/studio/web/assets/{typescript-UMJSfZyF.js → typescript-Bly1EW8o.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-B379CS-h.js → useEntityRestURL-Cg3P0MOb.js} +1 -1
- package/studio/web/assets/usecaseDiagram-VIAY4XPW-Brf2xP3S.js +328 -0
- package/studio/web/assets/{vendor-core-6LhfJlRC.js → vendor-core-PejxhUWQ.js} +1 -1
- package/studio/web/assets/vendor-datadog-GjET_Cbn.js +6 -0
- package/studio/web/assets/{vendor-react-BOliS0BL.js → vendor-react-DSIx92Ld.js} +1 -1
- package/studio/web/assets/vendor-tanstack-C5Y-nrd2.js +2 -0
- package/studio/web/assets/vendor-ui-BBWHOEoQ.js +41 -0
- package/studio/web/assets/{vennDiagram-UO4OBE2U-DZTsDA4k.js → vennDiagram-BLWOH2XV-XxOyJPq_.js} +3 -3
- package/studio/web/assets/wardley-7MLQ67FV-DRW0zuE4.js +1 -0
- package/studio/web/assets/{wardleyDiagram-VNRHLVJA-qu_f7Tvl.js → wardleyDiagram-YMQ3BMBF-B3WskJ9W.js} +3 -3
- package/studio/web/assets/wasAReleasedBeforeB-D5grAaGM.js +1 -0
- package/studio/web/assets/{workers-Bn_0_L1R.js → workers-DVtjrvUa.js} +1 -1
- package/studio/web/assets/{xml-CG2N-1r1.js → xml-DjAPbgLN.js} +1 -1
- package/studio/web/assets/xychartDiagram-TAQBALBS-D5NV4Env.js +7 -0
- package/studio/web/assets/{yaml-68pda6Av.js → yaml-oTUyoJy1.js} +1 -1
- package/studio/web/index.html +17 -15
- package/utility/npmUtilities.ts +17 -7
- package/studio/web/assets/Chat-CrxPD4ph.js +0 -2294
- package/studio/web/assets/FloatingChat-EXzoUQ-9.js +0 -23
- package/studio/web/assets/SearchDialog-7qRL8Yqx.js +0 -1
- package/studio/web/assets/_basePickBy-QJXUw_pr.js +0 -1
- package/studio/web/assets/_baseUniq-DR4vVjyC.js +0 -1
- package/studio/web/assets/alertDialog-BbWD9Xxa.js +0 -1
- package/studio/web/assets/applications-CdqjiSQe.js +0 -296
- package/studio/web/assets/architecture-7GRP2DOG-DDmz3hpI.js +0 -1
- package/studio/web/assets/arrow-up-D2g_57vT.js +0 -1
- package/studio/web/assets/authStore-CJCaADw9.js +0 -3
- package/studio/web/assets/button-DWwfgGWI.js +0 -21
- package/studio/web/assets/channel-D0AkPjJ8.js +0 -1
- package/studio/web/assets/chunk-3FUC2YCW-DuRp-j19.js +0 -2
- package/studio/web/assets/chunk-7M6MHVWA-h6WlaTzO.js +0 -213
- package/studio/web/assets/chunk-FOHPRMQF-DzxwRta7.js +0 -161
- package/studio/web/assets/chunk-O7XYJQB3-CBS3nZYV.js +0 -128
- package/studio/web/assets/chunk-Z7XXMR3K-DSHbImdf.js +0 -10
- package/studio/web/assets/cn-6T8VranJ.js +0 -1
- package/studio/web/assets/cynefin-OW5HDTMX-DIvbXLSj.js +0 -1
- package/studio/web/assets/cynefinDiagram-VND7K2PF-BdtaXiCE.js +0 -62
- package/studio/web/assets/dagre-6A5THRUB-C7RUWHyz.js +0 -4
- package/studio/web/assets/diagram-ATOU4E4O-dxRfeG4B.js +0 -3
- package/studio/web/assets/diagram-MPIPVDR6-7prveD1B.js +0 -41
- package/studio/web/assets/dist-BrRuMUzt.js +0 -11
- package/studio/web/assets/download-Cn6tPG4L.js +0 -1
- package/studio/web/assets/elk-276RUBZZ-DxDQPjQT.js +0 -27
- package/studio/web/assets/errorText-NpBmvk9I.js +0 -1
- package/studio/web/assets/eventmodeling-NTZA5JFV-DONQf5aG.js +0 -1
- package/studio/web/assets/flowDiagram-KWPJA3E3-BoZ8_QR1.js +0 -1
- package/studio/web/assets/gitGraph-4MIJSDKK-BrXWmJt2.js +0 -1
- package/studio/web/assets/gitGraphDiagram-X574FWY7-C2KnnZeY.js +0 -106
- package/studio/web/assets/graphlib-C9Tl7QrK.js +0 -1
- package/studio/web/assets/index-QLZw6xrU.css +0 -1
- package/studio/web/assets/index-qCLyhyah.js +0 -828
- package/studio/web/assets/index.lazy-DCx-DxIV.js +0 -2
- package/studio/web/assets/info-A6RAGUB7-BhzBHe4U.js +0 -1
- package/studio/web/assets/mermaid.core-1PX6R49U.js +0 -44
- package/studio/web/assets/notifications-BSObVIY1.js +0 -1
- package/studio/web/assets/notifications-DFLXk68b.js +0 -1
- package/studio/web/assets/packet-AYTQ26CC-DWbXHx5g.js +0 -1
- package/studio/web/assets/pie-WAS4IAKB-DjoTm3yr.js +0 -1
- package/studio/web/assets/pieDiagram-5QR66LMP-Dz3RHpDA.js +0 -39
- package/studio/web/assets/profile-TBjzOMIy.js +0 -1
- package/studio/web/assets/radar-RG4KPBEZ-CVBgjCOj.js +0 -1
- package/studio/web/assets/railroad-74A4TZTK-BdhXw10Q.js +0 -1
- package/studio/web/assets/railroad-abnf-HS5TGJTU-BcoQPN39.js +0 -1
- package/studio/web/assets/railroad-ebnf-LZEXJU2U-B1Lke2JN.js +0 -1
- package/studio/web/assets/railroad-peg-WCYAUIDC-D-oJFkiV.js +0 -1
- package/studio/web/assets/regions-C4an4H7e.js +0 -1
- package/studio/web/assets/restartGate-nHAFLDh-.js +0 -1
- package/studio/web/assets/sankeyDiagram-IPEJSGJF-D9AyLce2.js +0 -40
- package/studio/web/assets/sequenceDiagram-PO4LG4MO-hfk4aIEm.js +0 -169
- package/studio/web/assets/setComponentFile-BrqOM1Zm.js +0 -1
- package/studio/web/assets/setup-0v2MCm-E.js +0 -2
- package/studio/web/assets/status-DdezofUu.js +0 -61
- package/studio/web/assets/swimlanesDiagram-TC7HE7FX-DfN7BAP5.js +0 -8
- package/studio/web/assets/tabs-BCTQzoQK.js +0 -1
- package/studio/web/assets/treeView-Q6P3EWNA-C-epdxRo.js +0 -1
- package/studio/web/assets/treemap-WGGIJYW6-DOVyp_zw.js +0 -1
- package/studio/web/assets/usecaseDiagram-POWQR4AR-B6252cbq.js +0 -328
- package/studio/web/assets/vendor-datadog-ozKf-pts.js +0 -6
- package/studio/web/assets/vendor-tanstack-Bntfka-M.js +0 -2
- package/studio/web/assets/vendor-ui-9EYYDgoN.js +0 -41
- package/studio/web/assets/wardley-WFR3VGLG-Cfqoe6z4.js +0 -1
- package/studio/web/assets/xychartDiagram-PMCCYNJV-UrOKRsmR.js +0 -7
|
@@ -60,7 +60,7 @@ const MAX_RETRIES = 40;
|
|
|
60
60
|
export const COMMIT_PHASE_GRACE = 10;
|
|
61
61
|
// Cap the per-retry backoff so replication-applied transactions, which retry conflicts without a
|
|
62
62
|
// cap (see the commit rejection handler), don't grow the delay unbounded.
|
|
63
|
-
|
|
63
|
+
let MAX_RETRY_DELAY_MS = 1000;
|
|
64
64
|
// Every native write commit currently outstanding on this worker thread, oldest first.
|
|
65
65
|
// checkOverloaded() sheds new application writes once the OLDEST of these has been outstanding
|
|
66
66
|
// past MAX_OUTSTANDING_TXN_DURATION, so a commit whose promise never settles (harper#2001) is
|
|
@@ -101,12 +101,20 @@ const OVERLOAD_LOG_MIN_INTERVAL_MS = 1000;
|
|
|
101
101
|
// wedge and would starve `abandon`, which names a different transaction and is the only server-side
|
|
102
102
|
// record of why a request was failed. Mutates only when it grants, so a caller with its own
|
|
103
103
|
// suppression as well (checkOverloaded's per-node `logged`) must test that first.
|
|
104
|
-
const lastStuckCommitLogAt = { shed: -Infinity, abandon: -Infinity };
|
|
105
|
-
function allowStuckCommitLog(site: 'shed' | 'abandon', now: number): boolean {
|
|
104
|
+
const lastStuckCommitLogAt = { shed: -Infinity, abandon: -Infinity, overdue: -Infinity };
|
|
105
|
+
function allowStuckCommitLog(site: 'shed' | 'abandon' | 'overdue', now: number): boolean {
|
|
106
106
|
if (now - lastStuckCommitLogAt[site] <= OVERLOAD_LOG_MIN_INTERVAL_MS) return false;
|
|
107
107
|
lastStuckCommitLogAt[site] = now;
|
|
108
108
|
return true;
|
|
109
109
|
}
|
|
110
|
+
/** Test seam: clears the stuck- and stalled-commit log cooldowns so consecutive tests can assert on log emission without sleeping. */
|
|
111
|
+
export function resetStuckCommitLogCooldownForTests(): void {
|
|
112
|
+
lastStuckCommitLogAt.shed = -Infinity;
|
|
113
|
+
lastStuckCommitLogAt.abandon = -Infinity;
|
|
114
|
+
lastStuckCommitLogAt.overdue = -Infinity;
|
|
115
|
+
lastStalledCommitWarningAt = -Infinity;
|
|
116
|
+
lastStalledCommitErrorAt = -Infinity;
|
|
117
|
+
}
|
|
110
118
|
|
|
111
119
|
// Which database/table, which native transaction, and which request a stuck commit belongs to —
|
|
112
120
|
// without it a wedge gives no indication of what to investigate (harper#2001).
|
|
@@ -323,12 +331,27 @@ let replayedWritesWarned = false;
|
|
|
323
331
|
* committed or aborted, and every caller is a cleanup path whose own callers have no handler — a
|
|
324
332
|
* throw there would abandon the rest of the cleanup.
|
|
325
333
|
*/
|
|
326
|
-
function abortNativeTransaction(
|
|
334
|
+
function abortNativeTransaction(
|
|
335
|
+
transaction: RocksTransaction | null | undefined,
|
|
336
|
+
context: string,
|
|
337
|
+
store?: any,
|
|
338
|
+
startedFrom?: { resourceName: string; method: string }
|
|
339
|
+
): void {
|
|
327
340
|
if (transaction == null) return;
|
|
341
|
+
const txn = transaction as RocksTransactionWithRetry;
|
|
342
|
+
if (txn.abortedByWrapper || txn.committed || txn.commitInFlight) return;
|
|
343
|
+
txn.abortedByWrapper = true;
|
|
328
344
|
try {
|
|
329
345
|
transaction.abort();
|
|
330
346
|
} catch (error) {
|
|
331
|
-
|
|
347
|
+
if ((error as any)?.code === 'ERR_TRANSACTION_ABANDONED') {
|
|
348
|
+
harperLogger.error?.(
|
|
349
|
+
`Aborted a transaction with a durable log entry (${context}), ${describeCommitIdentity(store, startedFrom, transaction)}`,
|
|
350
|
+
error
|
|
351
|
+
);
|
|
352
|
+
} else {
|
|
353
|
+
harperLogger.debug?.(context, error);
|
|
354
|
+
}
|
|
332
355
|
}
|
|
333
356
|
}
|
|
334
357
|
|
|
@@ -546,7 +569,13 @@ export type TransactionWrite = {
|
|
|
546
569
|
entry?: Partial<Entry>;
|
|
547
570
|
before?: () => void | Promise<void>;
|
|
548
571
|
beforeIntermediate?: () => void | Promise<void>;
|
|
549
|
-
commit?: (
|
|
572
|
+
commit?: (
|
|
573
|
+
txnTime: number,
|
|
574
|
+
existingEntry: Partial<Entry>,
|
|
575
|
+
retry: boolean,
|
|
576
|
+
transaction: any,
|
|
577
|
+
committedBy?: DatabaseTransaction
|
|
578
|
+
) => MaybePromise<void>;
|
|
550
579
|
// Once a write has been taken over, the transaction committing it is not the one that staged it, and
|
|
551
580
|
// overload accounting, the replay marker and a no-op write's removal all belong to the committer.
|
|
552
581
|
validate?: (txnTime: number, committedBy: DatabaseTransaction) => void;
|
|
@@ -563,12 +592,14 @@ export type TransactionWrite = {
|
|
|
563
592
|
result?: any;
|
|
564
593
|
// blobs that were pre-saved as part of this write; used to clean up files if the commit is skipped or aborted
|
|
565
594
|
savedBlobs?: Blob[];
|
|
566
|
-
//
|
|
567
|
-
//
|
|
595
|
+
// this round staged neither a record nor an audit entry. An earlier durable audit entry may
|
|
596
|
+
// still own savedBlobs. Reset at the top of each commit-handler invocation.
|
|
568
597
|
skipped?: boolean;
|
|
569
598
|
// sticky: a non-isRetry staging of this write appended its audit entry (set in save(); the retry
|
|
570
599
|
// dedup guards in the commit handler read it to ignore the write's own orphaned entry)
|
|
571
600
|
appendedAuditEntry?: boolean;
|
|
601
|
+
// the native handle whose batch holds that audit entry (an iterator replay discards the original's batch)
|
|
602
|
+
appendedTo?: RocksTransaction;
|
|
572
603
|
// the transaction holding this write in its `writes` (set in addWrite). A deferred write's save() is
|
|
573
604
|
// only its trigger, so it can be triggered after the context has moved on to another transaction;
|
|
574
605
|
// this is who commits it when the transaction current at that point is not a scope (#2292).
|
|
@@ -589,15 +620,15 @@ export type TransactionWrite = {
|
|
|
589
620
|
// stored, so blobs this write saved are only reachable through its audit entry. Reset per
|
|
590
621
|
// commit-handler round, like stagedEntry.
|
|
591
622
|
superseded?: boolean;
|
|
592
|
-
//
|
|
593
|
-
//
|
|
623
|
+
// Audit pruning owns blobs referenced by an appended audit entry once its handle commits or
|
|
624
|
+
// has a durable log; a staged entry in an unsubmitted handle owns none.
|
|
594
625
|
blobsAuditReferenced?: boolean;
|
|
595
626
|
// Lock handle set by Table._writeUpdate when the write is staged through a held record lock;
|
|
596
627
|
// used in DatabaseTransaction.save() to assign and track that handle's write versions.
|
|
597
628
|
lockHandle?: RecordLockHandle;
|
|
598
|
-
// Per-operation holder version, set once on first save() and reused on retry so
|
|
599
|
-
//
|
|
600
|
-
//
|
|
629
|
+
// Per-operation holder version, set once on first save() and reused on retry so each lock write
|
|
630
|
+
// keeps its own candidate version across retries instead of falling back to stagedTxnTime or an
|
|
631
|
+
// explicit this.timestamp shared on the transaction.
|
|
601
632
|
lockStamp?: number;
|
|
602
633
|
// Version staged by a write that actually changed the record this round. It advances any
|
|
603
634
|
// same-key held handle's floor only after the native transaction commits successfully.
|
|
@@ -614,6 +645,7 @@ export type TransactionWrite = {
|
|
|
614
645
|
// the commit derives stored state (folds, index diffs, residency) from its base entry, so
|
|
615
646
|
// save() must reload that base through the committing transaction's snapshot
|
|
616
647
|
reloadCommitBase?: boolean;
|
|
648
|
+
stagedOn?: RocksTransaction;
|
|
617
649
|
baseReadTxn?: any;
|
|
618
650
|
writeGeneration?: WriteGeneration;
|
|
619
651
|
instanceClosed?: boolean;
|
|
@@ -676,7 +708,39 @@ function clearAttemptState(txn: DatabaseTransaction): void {
|
|
|
676
708
|
txn.stalledCommitResourcesReleased = false;
|
|
677
709
|
}
|
|
678
710
|
|
|
679
|
-
|
|
711
|
+
// Returned by takeCompletions() when nothing is staged; only its length is read before Promise.all.
|
|
712
|
+
const NO_COMPLETIONS: Promise<void>[] = Object.freeze([]) as unknown as Promise<void>[];
|
|
713
|
+
|
|
714
|
+
type RocksTransactionWithRetry = RocksTransaction & {
|
|
715
|
+
isRetry?: boolean;
|
|
716
|
+
retries?: number;
|
|
717
|
+
hasStagedLog?: boolean;
|
|
718
|
+
stagedTxnTime?: number;
|
|
719
|
+
commitStartedAt?: number;
|
|
720
|
+
commitOverdueLogged?: boolean;
|
|
721
|
+
lockHolds?: Set<RecordLockHandle>;
|
|
722
|
+
holdBudgetSpent?: boolean;
|
|
723
|
+
abortedByWrapper?: boolean;
|
|
724
|
+
commitInFlight?: boolean;
|
|
725
|
+
committed?: boolean;
|
|
726
|
+
immediateCommit?: boolean;
|
|
727
|
+
};
|
|
728
|
+
|
|
729
|
+
// Crash-recovery replay sets `isRetry` before `save()`, so `hasStagedLog` is never set there.
|
|
730
|
+
function hasDurableLog(transaction: RocksTransaction | null | undefined, error?: { hasLog?: boolean }): boolean {
|
|
731
|
+
const txn = transaction as RocksTransactionWithRetry | null | undefined;
|
|
732
|
+
return error?.hasLog === true || Boolean(txn?.isRetry && txn?.hasStagedLog);
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
function releaseTransactionLockHolds(transaction: RocksTransaction | null | undefined): void {
|
|
736
|
+
const txn = transaction as RocksTransactionWithRetry | null | undefined;
|
|
737
|
+
const holds = txn?.lockHolds;
|
|
738
|
+
if (!holds) return;
|
|
739
|
+
txn.lockHolds = undefined;
|
|
740
|
+
for (const handle of holds) {
|
|
741
|
+
handle.releaseCommitHold?.();
|
|
742
|
+
}
|
|
743
|
+
}
|
|
680
744
|
|
|
681
745
|
export class DatabaseTransaction implements Transaction {
|
|
682
746
|
#context: Context;
|
|
@@ -694,6 +758,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
694
758
|
// the last staged write per store and key, used to chain repeat writes to the same key (linkWrite)
|
|
695
759
|
declare writesByKey?: Map<any, Map<unknown, TransactionWrite>>;
|
|
696
760
|
completions: Promise<void>[] = []; // the set of outstanding async operations to complete
|
|
761
|
+
declare private completionHandles?: (RocksTransaction | undefined)[];
|
|
697
762
|
db: RootDatabaseKind;
|
|
698
763
|
transaction: RocksTransactionWithRetry;
|
|
699
764
|
readTxn: ReadTransaction;
|
|
@@ -731,6 +796,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
731
796
|
// unrelated request on the thread. Cleared when the logical commit settles, so a reused
|
|
732
797
|
// transaction's next batch starts on a fresh budget.
|
|
733
798
|
declare commitStartedAt?: number;
|
|
799
|
+
declare commitOverdueLogged?: boolean;
|
|
734
800
|
// Whether this link is why its chain root is write-supervised (see endWriteSupervision).
|
|
735
801
|
declare writeSupervised?: boolean;
|
|
736
802
|
declare stale: boolean;
|
|
@@ -821,6 +887,11 @@ export class DatabaseTransaction implements Transaction {
|
|
|
821
887
|
declare ownedReadIterators?: Set<OwnedReadIterator>;
|
|
822
888
|
// Depth of commit() attempts on this link that have not reached a native outcome.
|
|
823
889
|
commitsInFlight = 0;
|
|
890
|
+
declare private activeCommitHandle?: RocksTransaction;
|
|
891
|
+
declare private activeCommitHandles?: Set<RocksTransaction>;
|
|
892
|
+
declare private activeHandleStagedTxnTime?: number;
|
|
893
|
+
declare private activeHandleInFlight?: boolean;
|
|
894
|
+
declare private activeHandleCommitted?: boolean;
|
|
824
895
|
// A poison landed while a commit was already in flight somewhere in the chain. Excuses the head's
|
|
825
896
|
// cascade into `next`, but not a link the monitor poisoned BEFORE any commit started.
|
|
826
897
|
declare poisonedMidCommit?: boolean;
|
|
@@ -829,6 +900,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
829
900
|
// can visit every link of a chain, so a counter would advance once per link per tick.
|
|
830
901
|
declare deferredPoisonDeadline?: number;
|
|
831
902
|
declare unsubmittedCommitDeadline?: number;
|
|
903
|
+
// Retry count of the chain's logged retry at the monitor's last tick, and the tick that decided whether
|
|
904
|
+
// it had moved: a count that stops moving is a native attempt that never settled, and every link the
|
|
905
|
+
// monitor visits in one tick must read the same decision.
|
|
906
|
+
declare loggedRetriesSeen?: number;
|
|
907
|
+
declare loggedRetriesChangedAt?: number;
|
|
908
|
+
declare loggedRetriesTick?: number;
|
|
909
|
+
declare loggedRetryAdvancing?: boolean;
|
|
832
910
|
// Per-link submission state. A commit method can still be parked in pre-commit work, where an abort
|
|
833
911
|
// is safe; once storage work has been submitted, its outcome is unknown and monitor cleanup must not
|
|
834
912
|
// clear writes or blobs that the eventual commit can reference.
|
|
@@ -1072,7 +1150,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1072
1150
|
*/
|
|
1073
1151
|
releaseReadTxn(): void {
|
|
1074
1152
|
const transaction = this.detachOwnedTransaction();
|
|
1075
|
-
abortNativeTransaction(
|
|
1153
|
+
abortNativeTransaction(
|
|
1154
|
+
transaction,
|
|
1155
|
+
'releasing timed-out read transaction',
|
|
1156
|
+
this.writes[0]?.store ?? this.db,
|
|
1157
|
+
this.startedFrom
|
|
1158
|
+
);
|
|
1076
1159
|
this.completeDeferredContextRelease();
|
|
1077
1160
|
}
|
|
1078
1161
|
|
|
@@ -1122,6 +1205,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1122
1205
|
const index = this.writes.indexOf(operation);
|
|
1123
1206
|
if (index > -1) this.writes[index] = null;
|
|
1124
1207
|
this.ownedWrites?.delete(operation);
|
|
1208
|
+
if (operation.stagedIn === this) operation.stagedIn = undefined;
|
|
1125
1209
|
if (operation.key === undefined) return;
|
|
1126
1210
|
const writesForStore = this.writesByKey?.get(operation.store);
|
|
1127
1211
|
if (!writesForStore) return;
|
|
@@ -1200,11 +1284,330 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1200
1284
|
if (recordLocks.size === 0) this.recordLocks = undefined;
|
|
1201
1285
|
}
|
|
1202
1286
|
|
|
1203
|
-
private
|
|
1287
|
+
private budgetCarrier(transaction?: RocksTransaction | null): {
|
|
1288
|
+
commitStartedAt?: number;
|
|
1289
|
+
commitOverdueLogged?: boolean;
|
|
1290
|
+
} {
|
|
1291
|
+
const txn = transaction as RocksTransactionWithRetry | null | undefined;
|
|
1292
|
+
if (txn?.commitStartedAt != null) return txn;
|
|
1293
|
+
if (this.saveCommits && txn && this.hasSiblingCommitInFlight(transaction ?? undefined)) {
|
|
1294
|
+
return txn;
|
|
1295
|
+
}
|
|
1296
|
+
if (txn?.immediateCommit) return txn;
|
|
1297
|
+
return this.saveCommits ? this : (this.root ?? this);
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
private backfillActiveHandleWrites(handle: RocksTransaction): void {
|
|
1301
|
+
if ((handle as RocksTransactionWithRetry).immediateCommit) return;
|
|
1302
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1303
|
+
const w = this.writes[i];
|
|
1304
|
+
if (w?.saved && !w.stagedOn) w.stagedOn = handle;
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
private isSingleHandleTarget(txn: RocksTransaction): boolean {
|
|
1309
|
+
return (
|
|
1310
|
+
!(txn as RocksTransactionWithRetry).immediateCommit &&
|
|
1311
|
+
(!this.activeCommitHandles || this.activeCommitHandles.size === 0) &&
|
|
1312
|
+
(!this.activeCommitHandle || this.activeCommitHandle === txn)
|
|
1313
|
+
);
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
private getHandleStagedTxnTime(txn?: RocksTransaction | null): number | undefined {
|
|
1317
|
+
if (!txn) return undefined;
|
|
1318
|
+
if (
|
|
1319
|
+
this.isSingleHandleTarget(txn) &&
|
|
1320
|
+
(this.activeCommitHandle === txn || (!this.activeCommitHandle && (!this.transaction || this.transaction === txn)))
|
|
1321
|
+
) {
|
|
1322
|
+
return this.activeHandleStagedTxnTime || (txn as RocksTransactionWithRetry).stagedTxnTime;
|
|
1323
|
+
}
|
|
1324
|
+
return (txn as RocksTransactionWithRetry).stagedTxnTime;
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
private setHandleStagedTxnTime(txn: RocksTransaction, txnTime: number): void {
|
|
1328
|
+
if (
|
|
1329
|
+
this.isSingleHandleTarget(txn) &&
|
|
1330
|
+
(this.activeCommitHandle === txn || (!this.activeCommitHandle && (!this.transaction || this.transaction === txn)))
|
|
1331
|
+
) {
|
|
1332
|
+
if (!this.activeHandleStagedTxnTime) this.activeHandleStagedTxnTime = txnTime;
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
(txn as RocksTransactionWithRetry).stagedTxnTime ??= txnTime;
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
private isHandleCommitInFlight(txn?: RocksTransaction | null): boolean {
|
|
1339
|
+
if (!txn) return false;
|
|
1340
|
+
if (this.activeCommitHandle === txn) {
|
|
1341
|
+
return Boolean(this.activeHandleInFlight || (txn as RocksTransactionWithRetry).commitInFlight);
|
|
1342
|
+
}
|
|
1343
|
+
return Boolean((txn as RocksTransactionWithRetry).commitInFlight);
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
private setHandleCommitInFlight(txn: RocksTransaction, inFlight: boolean): void {
|
|
1347
|
+
if (this.isSingleHandleTarget(txn) && inFlight && !this.activeCommitHandle) {
|
|
1348
|
+
this.activeCommitHandle = txn;
|
|
1349
|
+
}
|
|
1350
|
+
if (this.activeCommitHandle === txn) {
|
|
1351
|
+
this.activeHandleInFlight = inFlight;
|
|
1352
|
+
if (!inFlight) {
|
|
1353
|
+
(txn as RocksTransactionWithRetry).commitInFlight = false;
|
|
1354
|
+
}
|
|
1355
|
+
return;
|
|
1356
|
+
}
|
|
1357
|
+
(txn as RocksTransactionWithRetry).commitInFlight = inFlight;
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
private isHandleCommitted(txn?: RocksTransaction | null): boolean {
|
|
1361
|
+
if (!txn) return false;
|
|
1362
|
+
if (this.activeCommitHandle === txn) {
|
|
1363
|
+
return Boolean(this.activeHandleCommitted || (txn as RocksTransactionWithRetry).committed);
|
|
1364
|
+
}
|
|
1365
|
+
return Boolean((txn as RocksTransactionWithRetry).committed);
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
private setHandleCommitted(txn: RocksTransaction): void {
|
|
1369
|
+
if (this.activeCommitHandle === txn) {
|
|
1370
|
+
this.activeHandleInFlight = false;
|
|
1371
|
+
this.activeHandleCommitted = true;
|
|
1372
|
+
(txn as RocksTransactionWithRetry).commitInFlight = false;
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
(txn as RocksTransactionWithRetry).commitInFlight = false;
|
|
1376
|
+
(txn as RocksTransactionWithRetry).committed = true;
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
private trackActiveCommitHandle(transaction: RocksTransaction): void {
|
|
1380
|
+
if (
|
|
1381
|
+
!(transaction as RocksTransactionWithRetry).immediateCommit &&
|
|
1382
|
+
!this.activeCommitHandle &&
|
|
1383
|
+
(!this.activeCommitHandles || this.activeCommitHandles.size === 0)
|
|
1384
|
+
) {
|
|
1385
|
+
this.activeCommitHandle = transaction;
|
|
1386
|
+
return;
|
|
1387
|
+
}
|
|
1388
|
+
if (this.activeCommitHandle === transaction) return;
|
|
1389
|
+
let handles = this.activeCommitHandles;
|
|
1390
|
+
if (!handles) {
|
|
1391
|
+
handles = this.activeCommitHandles = new Set();
|
|
1392
|
+
}
|
|
1393
|
+
if (this.activeCommitHandle) {
|
|
1394
|
+
const first = this.activeCommitHandle as RocksTransactionWithRetry;
|
|
1395
|
+
if (this.activeHandleStagedTxnTime && !first.stagedTxnTime) {
|
|
1396
|
+
first.stagedTxnTime = this.activeHandleStagedTxnTime;
|
|
1397
|
+
}
|
|
1398
|
+
if (this.activeHandleInFlight) first.commitInFlight = true;
|
|
1399
|
+
if (this.activeHandleCommitted) first.committed = true;
|
|
1400
|
+
this.activeHandleStagedTxnTime = 0;
|
|
1401
|
+
this.activeHandleInFlight = false;
|
|
1402
|
+
this.activeHandleCommitted = false;
|
|
1403
|
+
this.backfillActiveHandleWrites(first);
|
|
1404
|
+
if (this.saveCommits) {
|
|
1405
|
+
if (this.commitStartedAt != null && first.commitStartedAt == null) {
|
|
1406
|
+
first.commitStartedAt = this.commitStartedAt;
|
|
1407
|
+
this.commitStartedAt = undefined;
|
|
1408
|
+
}
|
|
1409
|
+
if (this.commitOverdueLogged) {
|
|
1410
|
+
first.commitOverdueLogged = true;
|
|
1411
|
+
this.commitOverdueLogged = undefined;
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
handles.add(this.activeCommitHandle);
|
|
1415
|
+
this.activeCommitHandle = undefined;
|
|
1416
|
+
}
|
|
1417
|
+
handles.add(transaction);
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
private releaseActiveCommitHandle(transaction: RocksTransaction): void {
|
|
1421
|
+
if (this.activeCommitHandle === transaction) {
|
|
1422
|
+
this.activeCommitHandle = undefined;
|
|
1423
|
+
this.activeHandleStagedTxnTime = 0;
|
|
1424
|
+
this.activeHandleInFlight = false;
|
|
1425
|
+
this.activeHandleCommitted = false;
|
|
1426
|
+
return;
|
|
1427
|
+
}
|
|
1428
|
+
const handles = this.activeCommitHandles;
|
|
1429
|
+
if (!handles) return;
|
|
1430
|
+
handles.delete(transaction);
|
|
1431
|
+
if (handles.size === 1) {
|
|
1432
|
+
for (const remaining of handles) {
|
|
1433
|
+
const rem = remaining as RocksTransactionWithRetry;
|
|
1434
|
+
if (!rem.immediateCommit) {
|
|
1435
|
+
this.activeCommitHandle = remaining;
|
|
1436
|
+
this.activeHandleStagedTxnTime = rem.stagedTxnTime ?? 0;
|
|
1437
|
+
this.activeHandleInFlight = Boolean(rem.commitInFlight);
|
|
1438
|
+
this.activeHandleCommitted = Boolean(rem.committed);
|
|
1439
|
+
handles.clear();
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
private clearActiveCommitHandles(): void {
|
|
1446
|
+
if (this.activeCommitHandle !== undefined || this.activeHandleStagedTxnTime) {
|
|
1447
|
+
this.activeCommitHandle = undefined;
|
|
1448
|
+
this.activeHandleStagedTxnTime = 0;
|
|
1449
|
+
this.activeHandleInFlight = false;
|
|
1450
|
+
this.activeHandleCommitted = false;
|
|
1451
|
+
}
|
|
1452
|
+
this.activeCommitHandles?.clear();
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
private hasActiveCommitHandle(): boolean {
|
|
1456
|
+
return (
|
|
1457
|
+
this.activeCommitHandle !== undefined ||
|
|
1458
|
+
(this.activeCommitHandles !== undefined && this.activeCommitHandles.size > 0)
|
|
1459
|
+
);
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
/**
|
|
1463
|
+
* Retries so far of the handles with a durable transaction-log entry still retrying their commit on
|
|
1464
|
+
* this link, summed over overlapping commits so the total moves whenever any of them does, or -1
|
|
1465
|
+
* when there is none.
|
|
1466
|
+
*/
|
|
1467
|
+
activeDurableLogRetries(): number {
|
|
1468
|
+
let retries = -1;
|
|
1469
|
+
const note = (handle: RocksTransaction | null | undefined) => {
|
|
1470
|
+
const txn = handle as RocksTransactionWithRetry | null | undefined;
|
|
1471
|
+
if (!txn || !hasDurableLog(txn) || this.isHandleCommitted(txn) || txn.abortedByWrapper) return;
|
|
1472
|
+
retries = Math.max(retries, 0) + (txn.retries ?? 0);
|
|
1473
|
+
};
|
|
1474
|
+
note(this.activeCommitHandle);
|
|
1475
|
+
if (this.activeCommitHandles) for (const handle of this.activeCommitHandles) note(handle);
|
|
1476
|
+
note(this.transaction);
|
|
1477
|
+
return retries;
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1480
|
+
private isActiveCommitHandle(transaction: RocksTransaction): boolean {
|
|
1481
|
+
return this.activeCommitHandle === transaction || Boolean(this.activeCommitHandles?.has(transaction));
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
private hasSiblingCommitInFlight(targetTransaction?: RocksTransaction): boolean {
|
|
1485
|
+
const excluded = targetTransaction ?? this.transaction;
|
|
1486
|
+
if (this.activeCommitHandle) return this.activeCommitHandle !== excluded;
|
|
1487
|
+
const active = this.activeCommitHandles;
|
|
1488
|
+
if (!active || active.size === 0) return false;
|
|
1489
|
+
if (!excluded) return true;
|
|
1490
|
+
return active.size > (active.has(excluded) ? 1 : 0);
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
private isWriteOwnedByActiveCommit(
|
|
1494
|
+
write: TransactionWrite | undefined,
|
|
1495
|
+
targetTransaction: RocksTransaction | undefined,
|
|
1496
|
+
hasSiblingInFlight: boolean,
|
|
1497
|
+
fromAbort = false,
|
|
1498
|
+
excludeImmediate = false
|
|
1499
|
+
): boolean {
|
|
1500
|
+
if (!write) return false;
|
|
1501
|
+
if ((targetTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit) {
|
|
1502
|
+
return write.stagedOn !== targetTransaction;
|
|
1503
|
+
}
|
|
1504
|
+
if (write.deferSave && !write.saved && this.saveCommits)
|
|
1505
|
+
return (
|
|
1506
|
+
(!fromAbort || Boolean(targetTransaction) || Boolean((this as { isCommitting?: boolean }).isCommitting)) &&
|
|
1507
|
+
!write.stagedOn &&
|
|
1508
|
+
!write.lockHandle?.released &&
|
|
1509
|
+
!write.lockHandle?.isExpired()
|
|
1510
|
+
);
|
|
1511
|
+
if (!write.saved)
|
|
1512
|
+
return !fromAbort && Boolean(targetTransaction) && this.open === TRANSACTION_STATE.OPEN && !write.stagedOn;
|
|
1513
|
+
const stagedOn = (write.stagedOn ?? this.activeCommitHandle ?? this.transaction) as
|
|
1514
|
+
RocksTransactionWithRetry | undefined;
|
|
1515
|
+
if (!stagedOn || stagedOn === targetTransaction || stagedOn.abortedByWrapper || this.isHandleCommitted(stagedOn))
|
|
1516
|
+
return false;
|
|
1517
|
+
if (excludeImmediate && stagedOn.immediateCommit) return false;
|
|
1518
|
+
if (
|
|
1519
|
+
!fromAbort &&
|
|
1520
|
+
Boolean(targetTransaction) &&
|
|
1521
|
+
this.open === TRANSACTION_STATE.OPEN &&
|
|
1522
|
+
stagedOn === this.transaction
|
|
1523
|
+
)
|
|
1524
|
+
return true;
|
|
1525
|
+
if (
|
|
1526
|
+
hasSiblingInFlight &&
|
|
1527
|
+
(Boolean(targetTransaction) ||
|
|
1528
|
+
(!fromAbort && this.isActiveCommitHandle(stagedOn)) ||
|
|
1529
|
+
(fromAbort &&
|
|
1530
|
+
this.isActiveCommitHandle(stagedOn) &&
|
|
1531
|
+
((!excludeImmediate && Boolean(stagedOn.immediateCommit)) || Boolean(stagedOn.isRetry))))
|
|
1532
|
+
)
|
|
1533
|
+
return true;
|
|
1534
|
+
return Boolean(this.isHandleCommitInFlight(stagedOn) || hasDurableLog(stagedOn));
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
private hasWriteOwnedByActiveCommit(
|
|
1538
|
+
targetTransaction: RocksTransaction | undefined,
|
|
1539
|
+
hasSiblingInFlight: boolean,
|
|
1540
|
+
fromAbort = false,
|
|
1541
|
+
excludeImmediate = false
|
|
1542
|
+
): boolean {
|
|
1543
|
+
if (!this.stagesWriteOnSave) return false;
|
|
1544
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1545
|
+
if (
|
|
1546
|
+
this.isWriteOwnedByActiveCommit(
|
|
1547
|
+
this.writes[i],
|
|
1548
|
+
targetTransaction,
|
|
1549
|
+
hasSiblingInFlight,
|
|
1550
|
+
fromAbort,
|
|
1551
|
+
excludeImmediate
|
|
1552
|
+
)
|
|
1553
|
+
)
|
|
1554
|
+
return true;
|
|
1555
|
+
}
|
|
1556
|
+
return false;
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
private hasSavedWrite(): boolean {
|
|
1560
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1561
|
+
if (this.writes[i]?.saved) return true;
|
|
1562
|
+
}
|
|
1563
|
+
return false;
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
private hasSavedWriteForHandle(transaction: RocksTransaction, isImmediateCommit: boolean): boolean {
|
|
1567
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1568
|
+
const write = this.writes[i];
|
|
1569
|
+
if (!write?.saved) continue;
|
|
1570
|
+
if (isImmediateCommit ? write.stagedOn === transaction : !write.stagedOn || write.stagedOn === transaction) {
|
|
1571
|
+
return true;
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
return false;
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
private retainTransactionLockHolds(transaction: RocksTransaction | null | undefined): void {
|
|
1578
|
+
if (!transaction || !this.hasLeaseProtectedWrite) return;
|
|
1579
|
+
const txn = transaction as RocksTransactionWithRetry;
|
|
1580
|
+
if (txn.lockHolds || txn.holdBudgetSpent) return;
|
|
1581
|
+
const carrier = this.budgetCarrier(transaction);
|
|
1582
|
+
if (carrier.commitStartedAt == null) carrier.commitStartedAt = performance.now();
|
|
1583
|
+
const budget = this.commitConflictBudget();
|
|
1584
|
+
const holdDeadlineMono = Number.isFinite(budget) ? carrier.commitStartedAt + budget : undefined;
|
|
1585
|
+
let holds: Set<RecordLockHandle> | undefined;
|
|
1586
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1587
|
+
const write = this.writes[i];
|
|
1588
|
+
if (!write?.saved || (write.stagedOn && write.stagedOn !== transaction)) continue;
|
|
1589
|
+
const handle = write.lockHandle;
|
|
1590
|
+
if (!handle?.retainForCommit || (holds && holds.has(handle))) continue;
|
|
1591
|
+
handle.retainForCommit(holdDeadlineMono);
|
|
1592
|
+
(holds ??= new Set()).add(handle);
|
|
1593
|
+
}
|
|
1594
|
+
if (holds) txn.lockHolds = holds;
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
private noteCommittedLockVersions(committedTransaction?: RocksTransaction): void {
|
|
1204
1598
|
const recordLocks = this.recordLocks;
|
|
1205
1599
|
if (!recordLocks) return;
|
|
1600
|
+
const isImmediateCommit = Boolean((committedTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1601
|
+
const filterByStagedOn = Boolean(committedTransaction || this.hasSiblingCommitInFlight(committedTransaction));
|
|
1206
1602
|
for (const write of this.writes) {
|
|
1207
|
-
if (write?.
|
|
1603
|
+
if (!write?.saved) continue;
|
|
1604
|
+
if (
|
|
1605
|
+
isImmediateCommit
|
|
1606
|
+
? write.stagedOn !== committedTransaction
|
|
1607
|
+
: filterByStagedOn && write.stagedOn && write.stagedOn !== committedTransaction
|
|
1608
|
+
)
|
|
1609
|
+
continue;
|
|
1610
|
+
if (write.appliedRecordVersion == null) continue;
|
|
1208
1611
|
const handle = write.lockHandle ?? recordLocks.get(write.store)?.get(writeKeyId(write.key));
|
|
1209
1612
|
if (handle && !handle.released) handle.noteHolderVersion(write.appliedRecordVersion);
|
|
1210
1613
|
}
|
|
@@ -1212,9 +1615,33 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1212
1615
|
|
|
1213
1616
|
/**
|
|
1214
1617
|
* Discard the staged write set (committed or aborted); the per-key chain must go with it so a
|
|
1215
|
-
* reused transaction never bases a write on a previous batch's staged state.
|
|
1618
|
+
* reused transaction never bases a write on a previous batch's staged state. When concurrent
|
|
1619
|
+
* commits share one DatabaseTransaction, only the writes staged on `committedTransaction` are
|
|
1620
|
+
* removed while another native commit is still in flight.
|
|
1216
1621
|
*/
|
|
1217
|
-
clearWrites(): void {
|
|
1622
|
+
clearWrites(committedTransaction?: RocksTransaction, fromAbort = false): void {
|
|
1623
|
+
if (committedTransaction) this.releaseActiveCommitHandle(committedTransaction);
|
|
1624
|
+
const isImmediateCommit = Boolean((committedTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1625
|
+
const hasSiblingInFlight = this.hasSiblingCommitInFlight(committedTransaction);
|
|
1626
|
+
if (
|
|
1627
|
+
this.stagesWriteOnSave &&
|
|
1628
|
+
(isImmediateCommit ||
|
|
1629
|
+
((hasSiblingInFlight || !committedTransaction || this.commitsInFlight > 1 || this.saveCommits) &&
|
|
1630
|
+
this.hasWriteOwnedByActiveCommit(committedTransaction, hasSiblingInFlight, fromAbort)))
|
|
1631
|
+
) {
|
|
1632
|
+
for (const write of [...this.writes]) {
|
|
1633
|
+
if (!write || this.isWriteOwnedByActiveCommit(write, committedTransaction, hasSiblingInFlight, fromAbort))
|
|
1634
|
+
continue;
|
|
1635
|
+
this.detachWrite(write);
|
|
1636
|
+
}
|
|
1637
|
+
this.writes = this.writes.filter((write) => write);
|
|
1638
|
+
this.validated = this.writes.reduce((acc, w, idx) => (w?.saved ? idx + 1 : acc), 0);
|
|
1639
|
+
if (this.hasLeaseProtectedWrite && !this.writes.some((write) => write?.lockHandle)) {
|
|
1640
|
+
this.hasLeaseProtectedWrite = false;
|
|
1641
|
+
}
|
|
1642
|
+
return;
|
|
1643
|
+
}
|
|
1644
|
+
this.clearActiveCommitHandles();
|
|
1218
1645
|
// A deferred write's `stagedIn` must not outlive this transaction's ability to commit it: save() can
|
|
1219
1646
|
// fire after a commit or abort, and routing it back here would revive a write this transaction
|
|
1220
1647
|
// already rolled back — whose blobs abort() has reclaimed. Cleared here, save() resolves the
|
|
@@ -1224,6 +1651,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1224
1651
|
if (write) this.ownedWrites?.delete(write);
|
|
1225
1652
|
}
|
|
1226
1653
|
this.writes = [];
|
|
1654
|
+
this.validated = 0;
|
|
1227
1655
|
this.writesByKey = undefined;
|
|
1228
1656
|
// The commit-fence latch belongs to the discarded batch. A reused transaction that once staged
|
|
1229
1657
|
// a locked write would otherwise re-scan every write of every later unlocked batch. Assigned
|
|
@@ -1315,12 +1743,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1315
1743
|
* no record), and writes with no stored entry at all — a delete leaves a readable tombstone on any
|
|
1316
1744
|
* audited or delete-tracking table, so only a delete on a table with neither is entry-less. Such a
|
|
1317
1745
|
* `put(A); delete(B)` confirms on A's entry, whose version is this transaction's (every write in a
|
|
1318
|
-
* transaction is stamped with one version).
|
|
1746
|
+
* transaction is stamped with one version). `savedOnly` also skips a deferred write not yet saved,
|
|
1747
|
+
* which a commit leaves in place for a later one.
|
|
1319
1748
|
*/
|
|
1320
|
-
lastConfirmableEntry(): Partial<Entry> | undefined {
|
|
1749
|
+
lastConfirmableEntry(savedOnly = false): Partial<Entry> | undefined {
|
|
1321
1750
|
for (let i = this.writes.length - 1; i >= 0; i--) {
|
|
1322
1751
|
const write = this.writes[i];
|
|
1323
|
-
if (!write || write.skipReplicationConfirmation) continue;
|
|
1752
|
+
if (!write || write.skipReplicationConfirmation || (savedOnly && !write.saved)) continue;
|
|
1324
1753
|
const entry = write.store.getEntry(write.key);
|
|
1325
1754
|
if (entry) return entry;
|
|
1326
1755
|
}
|
|
@@ -1332,9 +1761,61 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1332
1761
|
* rejection in that window is an unhandled rejection (fatal under --unhandled-rejections=strict).
|
|
1333
1762
|
* The rejection still surfaces through commit()'s Promise.all.
|
|
1334
1763
|
*/
|
|
1335
|
-
stageCompletion(completion: Promise<void
|
|
1764
|
+
stageCompletion(completion: Promise<void>, transaction?: RocksTransaction) {
|
|
1336
1765
|
completion.then(undefined, () => {});
|
|
1766
|
+
const index = this.completions.length;
|
|
1337
1767
|
this.completions.push(completion);
|
|
1768
|
+
if (
|
|
1769
|
+
transaction &&
|
|
1770
|
+
(this.saveCommits ||
|
|
1771
|
+
Boolean((transaction as RocksTransactionWithRetry).immediateCommit) ||
|
|
1772
|
+
this.hasActiveCommitHandle() ||
|
|
1773
|
+
Boolean(this.completionHandles))
|
|
1774
|
+
) {
|
|
1775
|
+
(this.completionHandles ??= [])[index] = transaction;
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
|
|
1779
|
+
private takeCompletions(transaction?: RocksTransaction): Promise<void>[] {
|
|
1780
|
+
if (this.completions.length === 0) return NO_COMPLETIONS;
|
|
1781
|
+
const handles = this.completionHandles;
|
|
1782
|
+
const isImmediate = Boolean((transaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1783
|
+
if (!handles) {
|
|
1784
|
+
if (isImmediate) return NO_COMPLETIONS;
|
|
1785
|
+
const completions = this.completions;
|
|
1786
|
+
this.completions = [];
|
|
1787
|
+
return completions;
|
|
1788
|
+
}
|
|
1789
|
+
let allMatch = true;
|
|
1790
|
+
for (let i = 0; i < this.completions.length; i++) {
|
|
1791
|
+
const handle = handles[i];
|
|
1792
|
+
if (!(handle ? handle === transaction : !isImmediate)) {
|
|
1793
|
+
allMatch = false;
|
|
1794
|
+
break;
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
if (allMatch) {
|
|
1798
|
+
const completions = this.completions;
|
|
1799
|
+
this.completions = [];
|
|
1800
|
+
this.completionHandles = undefined;
|
|
1801
|
+
return completions;
|
|
1802
|
+
}
|
|
1803
|
+
const taken: Promise<void>[] = [];
|
|
1804
|
+
const remaining: Promise<void>[] = [];
|
|
1805
|
+
const remainingHandles: (RocksTransaction | undefined)[] = [];
|
|
1806
|
+
for (let i = 0; i < this.completions.length; i++) {
|
|
1807
|
+
const completion = this.completions[i];
|
|
1808
|
+
const handle = handles[i];
|
|
1809
|
+
if (handle ? handle === transaction : !isImmediate) {
|
|
1810
|
+
taken.push(completion);
|
|
1811
|
+
} else {
|
|
1812
|
+
remaining.push(completion);
|
|
1813
|
+
remainingHandles.push(handle);
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
this.completions = remaining;
|
|
1817
|
+
this.completionHandles = remainingHandles.length > 0 ? remainingHandles : undefined;
|
|
1818
|
+
return taken;
|
|
1338
1819
|
}
|
|
1339
1820
|
|
|
1340
1821
|
/**
|
|
@@ -1343,11 +1824,38 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1343
1824
|
* Clearing them also keeps a reused transaction's next commit() from rejecting with the previous
|
|
1344
1825
|
* batch's error.
|
|
1345
1826
|
*/
|
|
1346
|
-
drainCompletions(): void {
|
|
1827
|
+
drainCompletions(targetTransaction?: RocksTransaction, preserveActiveWrites = false): void {
|
|
1347
1828
|
if (this.completions.length === 0) return;
|
|
1348
|
-
const
|
|
1349
|
-
|
|
1350
|
-
|
|
1829
|
+
const handles = this.completionHandles;
|
|
1830
|
+
const isImmediateTarget = Boolean((targetTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1831
|
+
let drained: Promise<void>[];
|
|
1832
|
+
if (!preserveActiveWrites) {
|
|
1833
|
+
drained = this.completions;
|
|
1834
|
+
this.completions = [];
|
|
1835
|
+
this.completionHandles = undefined;
|
|
1836
|
+
} else {
|
|
1837
|
+
drained = [];
|
|
1838
|
+
const remaining: Promise<void>[] = [];
|
|
1839
|
+
const remainingHandles: (RocksTransaction | undefined)[] = [];
|
|
1840
|
+
for (let i = 0; i < this.completions.length; i++) {
|
|
1841
|
+
const completion = this.completions[i];
|
|
1842
|
+
const handle = handles?.[i];
|
|
1843
|
+
const keep = isImmediateTarget
|
|
1844
|
+
? handle !== targetTransaction
|
|
1845
|
+
: Boolean(handle) &&
|
|
1846
|
+
handle !== targetTransaction &&
|
|
1847
|
+
(this.isActiveCommitHandle(handle) || hasDurableLog(handle));
|
|
1848
|
+
if (keep) {
|
|
1849
|
+
remaining.push(completion);
|
|
1850
|
+
remainingHandles.push(handle);
|
|
1851
|
+
} else {
|
|
1852
|
+
drained.push(completion);
|
|
1853
|
+
}
|
|
1854
|
+
}
|
|
1855
|
+
this.completions = remaining;
|
|
1856
|
+
this.completionHandles = remainingHandles.length > 0 ? remainingHandles : undefined;
|
|
1857
|
+
}
|
|
1858
|
+
for (const completion of drained)
|
|
1351
1859
|
completion.then(undefined, (error) =>
|
|
1352
1860
|
harperLogger.warn?.('A staged transaction completion failed after the transaction was aborted', error)
|
|
1353
1861
|
);
|
|
@@ -1419,7 +1927,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1419
1927
|
// (the same rule the pre-submit fence applies below). Without the distinction,
|
|
1420
1928
|
// `lock(); save(); unlock(); commit()` succeeded normally and threw 409 only when a conflict
|
|
1421
1929
|
// retry or an open read iterator forced the replay to re-save it.
|
|
1422
|
-
if (operation.saved ? lockHandle?.isLeaseExpired() : lockHandle?.isExpired()) {
|
|
1930
|
+
if (operation.saved ? lockHandle?.isLeaseExpired() && !hasDurableLog(transaction) : lockHandle?.isExpired()) {
|
|
1423
1931
|
// Through `detachWrite`, which also repairs the per-key chain: dropping it from `writes` alone
|
|
1424
1932
|
// leaves `writesByKey` pointing at a write that was refused, so a caller that catches this 409
|
|
1425
1933
|
// and stages the same key again would take the rejected operation as its merge basis.
|
|
@@ -1442,7 +1950,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1442
1950
|
if (!operation.lockStamp) operation.lockStamp = lockHandle.holderVersionCandidate();
|
|
1443
1951
|
}
|
|
1444
1952
|
}
|
|
1445
|
-
|
|
1953
|
+
const concurrentClosedWrite =
|
|
1954
|
+
Boolean((transaction as RocksTransactionWithRetry | undefined)?.immediateCommit) ||
|
|
1955
|
+
(!transaction && this.open !== TRANSACTION_STATE.OPEN && !this.saveCommits);
|
|
1956
|
+
let txnTime =
|
|
1957
|
+
operation.lockStamp ?? this.getHandleStagedTxnTime(transaction) ?? (concurrentClosedWrite ? 0 : this.timestamp);
|
|
1958
|
+
const mintedTimestamp = !txnTime;
|
|
1446
1959
|
// Only an OPEN transaction accepts new staged writes. After commit, this.transaction may still
|
|
1447
1960
|
// be retained for outstanding read iterators; staging into it would silently discard the write
|
|
1448
1961
|
// when doneReadTxn() aborts the handle, so such writes commit immediately on a fresh
|
|
@@ -1474,19 +1987,39 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1474
1987
|
} else {
|
|
1475
1988
|
// if it is closed, we have to immediately commit, using our immediate transaction
|
|
1476
1989
|
immediateCommit = true;
|
|
1990
|
+
if (!this.saveCommits) (transaction as RocksTransactionWithRetry).immediateCommit = true;
|
|
1477
1991
|
}
|
|
1478
1992
|
if (txnTime) {
|
|
1479
1993
|
transaction.setTimestamp(txnTime);
|
|
1480
1994
|
}
|
|
1481
1995
|
}
|
|
1996
|
+
if (this.activeCommitHandle && this.activeCommitHandle !== transaction) {
|
|
1997
|
+
this.backfillActiveHandleWrites(this.activeCommitHandle);
|
|
1998
|
+
} else if (!this.activeCommitHandle && this.transaction && this.transaction !== transaction) {
|
|
1999
|
+
this.backfillActiveHandleWrites(this.transaction);
|
|
2000
|
+
}
|
|
2001
|
+
const shouldStampStagedOn =
|
|
2002
|
+
this.saveCommits ||
|
|
2003
|
+
immediateCommit ||
|
|
2004
|
+
Boolean((transaction as RocksTransactionWithRetry | undefined)?.immediateCommit) ||
|
|
2005
|
+
operation.stagedOn !== undefined ||
|
|
2006
|
+
(this.activeCommitHandles !== undefined && this.activeCommitHandles.size > 0) ||
|
|
2007
|
+
(this.activeCommitHandle !== undefined && this.activeCommitHandle !== transaction) ||
|
|
2008
|
+
(this.transaction != null && this.transaction !== transaction);
|
|
1482
2009
|
if (this.isReplay) {
|
|
1483
2010
|
// Replayed writes came FROM the transaction log; never re-append them —
|
|
1484
2011
|
// replay iterates that same log, so re-appending prevents convergence
|
|
1485
2012
|
// (boot hangs replaying its own output). Conflict retries stamp isRetry
|
|
1486
2013
|
// at the retry sites in commit(); this is the replay-path equivalent.
|
|
1487
2014
|
(transaction as RocksTransactionWithRetry).isRetry = true;
|
|
2015
|
+
(transaction as RocksTransactionWithRetry).retries = this.retries;
|
|
1488
2016
|
}
|
|
1489
|
-
if (!txnTime)
|
|
2017
|
+
if (!txnTime) {
|
|
2018
|
+
txnTime = transaction.getTimestamp();
|
|
2019
|
+
if (!concurrentClosedWrite) this.timestamp = txnTime;
|
|
2020
|
+
}
|
|
2021
|
+
if (!operation.lockStamp) this.setHandleStagedTxnTime(transaction, txnTime);
|
|
2022
|
+
if (this.saveCommits && mintedTimestamp) this.timestamp = 0;
|
|
1490
2023
|
if (!operation.saved && operation.pendingPriorWrite) {
|
|
1491
2024
|
const pendingWrites = [];
|
|
1492
2025
|
for (let pending = operation.pendingPriorWrite; pending;) {
|
|
@@ -1514,45 +2047,82 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1514
2047
|
// An entry read uncached through this same pinned-snapshot handle is already that base, until a retry resets
|
|
1515
2048
|
// the snapshot or an earlier staged write to the key changes what a read returns; the resource withholds the
|
|
1516
2049
|
// handle once its entry was replaced or evicted, and locked, snapshot-free and replayed writes always reload.
|
|
2050
|
+
const txnRetries = (transaction as RocksTransactionWithRetry | undefined)?.retries ?? 0;
|
|
1517
2051
|
const reusesBaseRead =
|
|
1518
2052
|
operation.baseReadTxn === transaction &&
|
|
1519
|
-
|
|
2053
|
+
txnRetries === 0 &&
|
|
1520
2054
|
!reloadEntry &&
|
|
1521
2055
|
!operation.priorWrite &&
|
|
1522
2056
|
!operation.lockHandle &&
|
|
1523
2057
|
!this.snapshotFree &&
|
|
1524
2058
|
!(transaction as any).snapshotDisabled &&
|
|
1525
2059
|
!this.isReplay;
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
if (
|
|
2060
|
+
const wasSaved = Boolean(operation.saved);
|
|
2061
|
+
let completion: Promise<void>;
|
|
2062
|
+
try {
|
|
2063
|
+
if (!reusesBaseRead && (reloadEntry || operation.entry === undefined || reloadsCommitBase)) {
|
|
2064
|
+
const uncachedRead = !!operation.reloadCommitBase || reloadEntry || this.isReplay;
|
|
2065
|
+
operation.entry = operation.store.getEntry(operation.key, { transaction, uncachedRead });
|
|
2066
|
+
}
|
|
2067
|
+
if (!operation.saved) {
|
|
2068
|
+
// immediately execute in this transaction
|
|
2069
|
+
const validated = validateWrite(operation, writeVersion, this);
|
|
2070
|
+
if ((validated as any) === false) {
|
|
2071
|
+
operation.saved = true;
|
|
2072
|
+
if (shouldStampStagedOn) operation.stagedOn = transaction;
|
|
2073
|
+
operation.commit = () => {}; // noop if we try again
|
|
2074
|
+
closeWriteInstance(operation);
|
|
2075
|
+
if (immediateCommit) {
|
|
2076
|
+
this.detachWrite(operation);
|
|
2077
|
+
if (!this.activeCommitHandle) this.activeHandleStagedTxnTime = 0;
|
|
2078
|
+
abortNativeTransaction(
|
|
2079
|
+
transaction,
|
|
2080
|
+
'aborting an immediate transaction with nothing to write',
|
|
2081
|
+
operation.store,
|
|
2082
|
+
this.startedFrom
|
|
2083
|
+
);
|
|
2084
|
+
}
|
|
2085
|
+
return;
|
|
2086
|
+
}
|
|
1534
2087
|
operation.saved = true;
|
|
1535
|
-
|
|
2088
|
+
let result: Promise<void> = operation.before?.() as Promise<void>;
|
|
2089
|
+
if (result?.then) this.stageCompletion(result, transaction);
|
|
2090
|
+
result = operation.beforeIntermediate?.() as Promise<void>;
|
|
2091
|
+
if (result?.then) this.stageCompletion(result, transaction);
|
|
2092
|
+
}
|
|
2093
|
+
if (shouldStampStagedOn) operation.stagedOn = transaction;
|
|
2094
|
+
if (lockHandle || this.recordLocks) operation.trackRecordVersion = true;
|
|
2095
|
+
if (operation.trackRecordVersion) operation.recordVersionApplied = false;
|
|
2096
|
+
try {
|
|
2097
|
+
completion = operation.commit(
|
|
2098
|
+
writeVersion,
|
|
2099
|
+
operation.entry,
|
|
2100
|
+
(wasSaved || this.isReplay) && txnRetries > 0,
|
|
2101
|
+
transaction,
|
|
2102
|
+
this
|
|
2103
|
+
) as Promise<void>;
|
|
2104
|
+
} finally {
|
|
1536
2105
|
closeWriteInstance(operation);
|
|
1537
|
-
return;
|
|
1538
2106
|
}
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
2107
|
+
} catch (error) {
|
|
2108
|
+
// A handle created for an immediate commit is owned by nobody else: a throw before its commit
|
|
2109
|
+
// is submitted must release it rather than leak it.
|
|
2110
|
+
if (immediateCommit) {
|
|
2111
|
+
this.drainCompletions(transaction, true);
|
|
2112
|
+
this.detachWrite(operation);
|
|
2113
|
+
if (!this.activeCommitHandle) this.activeHandleStagedTxnTime = 0;
|
|
2114
|
+
abortNativeTransaction(
|
|
2115
|
+
transaction,
|
|
2116
|
+
'aborting an immediate transaction whose write threw',
|
|
2117
|
+
operation.store,
|
|
2118
|
+
this.startedFrom
|
|
2119
|
+
);
|
|
2120
|
+
}
|
|
2121
|
+
throw error;
|
|
1552
2122
|
}
|
|
1553
2123
|
if (operation.trackRecordVersion)
|
|
1554
2124
|
operation.appliedRecordVersion = operation.recordVersionApplied ? writeVersion : undefined;
|
|
1555
|
-
if (typeof completion?.then === 'function') this.stageCompletion(completion);
|
|
2125
|
+
if (typeof completion?.then === 'function') this.stageCompletion(completion, transaction);
|
|
1556
2126
|
// Sticky record that THIS write staged with its audit entry appended (log entries batch on the
|
|
1557
2127
|
// native transaction and are durably written by its commit attempt — even a failed one — so
|
|
1558
2128
|
// they survive the abort-after-failed-commit of the retry paths). isRetry stagings
|
|
@@ -1561,6 +2131,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1561
2131
|
// retries where a recommit round self-skips before a fresh-transaction replay.
|
|
1562
2132
|
if (!operation.skipped && !(transaction as RocksTransactionWithRetry).isRetry) {
|
|
1563
2133
|
operation.appendedAuditEntry = true;
|
|
2134
|
+
operation.appendedTo = transaction;
|
|
1564
2135
|
}
|
|
1565
2136
|
if (immediateCommit) {
|
|
1566
2137
|
// immediately commit if the harper transaction is closed
|
|
@@ -1757,6 +2328,9 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1757
2328
|
private endCommitAttempt(): void {
|
|
1758
2329
|
this.commitsInFlight--;
|
|
1759
2330
|
if (!this.commitsInFlight) {
|
|
2331
|
+
this.clearActiveCommitHandles();
|
|
2332
|
+
this.commitStartedAt = undefined;
|
|
2333
|
+
if (this.commitOverdueLogged) this.commitOverdueLogged = undefined;
|
|
1760
2334
|
this.nativeCommitSubmitted = false;
|
|
1761
2335
|
this.nativeCommitAttemptEnded();
|
|
1762
2336
|
}
|
|
@@ -1774,7 +2348,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1774
2348
|
root.submittedCommitSupervised = false;
|
|
1775
2349
|
root.deferredPoisonDeadline = undefined;
|
|
1776
2350
|
root.unsubmittedCommitDeadline = undefined;
|
|
2351
|
+
root.loggedRetriesSeen = undefined;
|
|
2352
|
+
root.loggedRetriesChangedAt = undefined;
|
|
2353
|
+
root.loggedRetriesTick = undefined;
|
|
2354
|
+
root.loggedRetryAdvancing = undefined;
|
|
1777
2355
|
root.stalledCommitLogged = false;
|
|
2356
|
+
root.commitStartedAt = undefined;
|
|
2357
|
+
if (root.commitOverdueLogged) root.commitOverdueLogged = undefined;
|
|
1778
2358
|
const submittedLink = root.submittedLink;
|
|
1779
2359
|
root.submittedLink = undefined;
|
|
1780
2360
|
const submittedLinks = root.submittedLinks;
|
|
@@ -1843,31 +2423,61 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1843
2423
|
// reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain —
|
|
1844
2424
|
// but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction
|
|
1845
2425
|
let transaction = options.transaction ?? this.transaction;
|
|
2426
|
+
const retryRound = Boolean((options.transaction as RocksTransactionWithRetry)?.isRetry);
|
|
2427
|
+
const isImmediateCommit = Boolean((transaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
1846
2428
|
try {
|
|
2429
|
+
if (retryRound && this.activeCommitHandle && this.activeCommitHandle !== transaction) {
|
|
2430
|
+
this.backfillActiveHandleWrites(this.activeCommitHandle);
|
|
2431
|
+
}
|
|
1847
2432
|
for (let i = 0; i < this.writes.length; i++) {
|
|
1848
2433
|
let operation = this.writes[i];
|
|
1849
|
-
if (
|
|
1850
|
-
|
|
2434
|
+
if (
|
|
2435
|
+
!operation ||
|
|
2436
|
+
(isImmediateCommit
|
|
2437
|
+
? !retryRound || !operation.saved || operation.stagedOn !== transaction
|
|
2438
|
+
: retryRound
|
|
2439
|
+
? !operation.saved || (Boolean(operation.stagedOn) && operation.stagedOn !== transaction)
|
|
2440
|
+
: Boolean(operation.saved) || (this.saveCommits && Boolean(operation.deferSave)))
|
|
2441
|
+
)
|
|
2442
|
+
continue;
|
|
2443
|
+
this.save(operation, transaction, Boolean(operation.saved) || i < this.validated, options);
|
|
2444
|
+
if (!transaction) transaction = this.transaction;
|
|
1851
2445
|
}
|
|
2446
|
+
if (transaction && (this.writes.length > 0 || hasDurableLog(transaction)))
|
|
2447
|
+
this.trackActiveCommitHandle(transaction);
|
|
1852
2448
|
} catch (error) {
|
|
2449
|
+
if (!transaction) transaction = this.transaction ?? undefined;
|
|
1853
2450
|
// abort() releases only this.transaction; a retry round's handle was detached before its
|
|
1854
2451
|
// first submission and would otherwise hold its write intents until GC.
|
|
1855
|
-
if (transaction !== this.transaction)
|
|
1856
|
-
abortNativeTransaction(
|
|
1857
|
-
|
|
2452
|
+
if (transaction && transaction !== this.transaction)
|
|
2453
|
+
abortNativeTransaction(
|
|
2454
|
+
transaction,
|
|
2455
|
+
'aborting a retry transaction whose re-save threw',
|
|
2456
|
+
this.writes[0]?.store ?? this.db,
|
|
2457
|
+
this.startedFrom
|
|
2458
|
+
);
|
|
2459
|
+
this.abort(false, true, transaction);
|
|
1858
2460
|
throw error;
|
|
1859
2461
|
}
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
2462
|
+
if (!isImmediateCommit) {
|
|
2463
|
+
this.validated = this.saveCommits
|
|
2464
|
+
? this.writes.reduce((acc, w, idx) => (w && (w.saved || !w.deferSave) ? idx + 1 : acc), 0)
|
|
2465
|
+
: this.writes.length;
|
|
2466
|
+
}
|
|
2467
|
+
const completions = this.takeCompletions(transaction);
|
|
1863
2468
|
const stagedWrites = this.writes.length;
|
|
1864
|
-
|
|
2469
|
+
let inCommitPhase = false;
|
|
2470
|
+
if (completions.length > 0 && !isImmediateCommit) {
|
|
2471
|
+
inCommitPhase = true;
|
|
1865
2472
|
this.setCommitPhase(true);
|
|
1866
2473
|
}
|
|
1867
2474
|
let commitResult: MaybePromise<CommitResolution>;
|
|
1868
2475
|
try {
|
|
1869
2476
|
commitResult = when(completions.length > 0 ? Promise.all(completions) : null, () => {
|
|
1870
|
-
if (
|
|
2477
|
+
if (inCommitPhase) {
|
|
2478
|
+
inCommitPhase = false;
|
|
2479
|
+
this.setCommitPhase(false);
|
|
2480
|
+
}
|
|
1871
2481
|
// The transaction can be aborted underneath us while we are parked in the await above — by the
|
|
1872
2482
|
// monitor once the commit phase outlives its grace, or through the multi-store poison chain.
|
|
1873
2483
|
// abort() cleared the write set and released the handle, so resuming would commit nothing and
|
|
@@ -1879,7 +2489,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1879
2489
|
}
|
|
1880
2490
|
if (stagedWrites > 0 && this.writes.length === 0 && this.open === TRANSACTION_STATE.CLOSED)
|
|
1881
2491
|
throw new ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
|
|
1882
|
-
if (
|
|
2492
|
+
if (
|
|
2493
|
+
!retryRound &&
|
|
2494
|
+
!isImmediateCommit &&
|
|
2495
|
+
(this.saveCommits
|
|
2496
|
+
? this.writes.reduce((acc, w, idx) => (w && (w.saved || !w.deferSave) ? idx + 1 : acc), 0)
|
|
2497
|
+
: this.writes.length) > this.validated
|
|
2498
|
+
) {
|
|
1883
2499
|
// check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
|
|
1884
2500
|
return this.commit({ ...options, continuation: true });
|
|
1885
2501
|
}
|
|
@@ -1890,8 +2506,8 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1890
2506
|
// the local is empty: a truthy one is what the loop staged into, and the retained-handle
|
|
1891
2507
|
// and replay branches below deliberately commit a handle other than this.transaction.
|
|
1892
2508
|
if (!transaction) transaction = this.transaction;
|
|
1893
|
-
if (!options.transaction && this.
|
|
1894
|
-
this.open = TRANSACTION_STATE.CLOSED;
|
|
2509
|
+
if (!options.transaction && this.hasSavedWrite()) this.renewChainForNativeCommit();
|
|
2510
|
+
if (!isImmediateCommit) this.open = TRANSACTION_STATE.CLOSED;
|
|
1895
2511
|
// RocksTransaction.commit() resolves with RETRY_NOW_VALUE (a number) under
|
|
1896
2512
|
// coordinatedRetry, or void on a normal commit/abort.
|
|
1897
2513
|
let commitResolution: Promise<number | void> | void;
|
|
@@ -1899,11 +2515,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1899
2515
|
// recursions, immediate-commit re-entries, and a second top-level commit() (wrapper
|
|
1900
2516
|
// commit after an explicit in-handler commit) must not steal a reference owned by an
|
|
1901
2517
|
// outstanding iterator (doneReadTxn() would then never release the native handle).
|
|
1902
|
-
if (!this.baseReadRefConsumed) {
|
|
2518
|
+
if (!isImmediateCommit && !this.baseReadRefConsumed) {
|
|
1903
2519
|
this.baseReadRefConsumed = true;
|
|
1904
2520
|
this.readTxnsUsed--;
|
|
1905
2521
|
}
|
|
1906
|
-
|
|
2522
|
+
let hadLeaseProtectedWrite = false;
|
|
2523
|
+
if (!isImmediateCommit && this.readTxnsUsed > 0) {
|
|
1907
2524
|
// Outstanding iterators still stream through this.transaction — their native iterators
|
|
1908
2525
|
// live inside it (GetIterator wraps its write batch + snapshot), so committing or
|
|
1909
2526
|
// aborting the handle now would invalidate them mid-stream. Leave it open for the
|
|
@@ -1915,7 +2532,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1915
2532
|
// kept the last iterator from finishing cleanly — a hung stream, the long-transaction
|
|
1916
2533
|
// monitor's timeout abort — dropped writes the caller had already been told committed.
|
|
1917
2534
|
this.writes = this.writes.filter((write) => write); // filter out removed entries
|
|
1918
|
-
if (this.
|
|
2535
|
+
if (this.hasSavedWrite() || hasDurableLog(transaction)) {
|
|
1919
2536
|
if (!options.transaction) {
|
|
1920
2537
|
if (!replayedWritesWarned) {
|
|
1921
2538
|
replayedWritesWarned = true;
|
|
@@ -1937,23 +2554,45 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1937
2554
|
{ coordinatedRetry: true }
|
|
1938
2555
|
);
|
|
1939
2556
|
if (this.timestamp) replayTransaction.setTimestamp(this.timestamp);
|
|
1940
|
-
|
|
2557
|
+
const retries = ((transaction as RocksTransactionWithRetry | undefined)?.retries ?? this.retries) + 1;
|
|
2558
|
+
(replayTransaction as RocksTransactionWithRetry).retries = retries;
|
|
2559
|
+
this.retries = retries; // a replay round: commit handlers re-base on the reloaded entries
|
|
2560
|
+
const prevStagedTxnTime = this.getHandleStagedTxnTime(transaction);
|
|
2561
|
+
if (transaction) this.releaseActiveCommitHandle(transaction);
|
|
2562
|
+
this.trackActiveCommitHandle(replayTransaction);
|
|
2563
|
+
if (prevStagedTxnTime) this.setHandleStagedTxnTime(replayTransaction, prevStagedTxnTime);
|
|
2564
|
+
transaction = replayTransaction;
|
|
1941
2565
|
try {
|
|
1942
2566
|
for (const operation of this.writes) {
|
|
2567
|
+
if (!operation?.saved) continue;
|
|
1943
2568
|
this.save(operation, replayTransaction, true, options);
|
|
1944
2569
|
}
|
|
1945
2570
|
} catch (error) {
|
|
1946
|
-
abortNativeTransaction(
|
|
1947
|
-
|
|
2571
|
+
abortNativeTransaction(
|
|
2572
|
+
replayTransaction,
|
|
2573
|
+
'aborting a replay transaction whose re-save threw',
|
|
2574
|
+
this.writes[0]?.store ?? this.db,
|
|
2575
|
+
this.startedFrom
|
|
2576
|
+
);
|
|
2577
|
+
this.releaseActiveCommitHandle(replayTransaction);
|
|
2578
|
+
this.abort(
|
|
2579
|
+
false,
|
|
2580
|
+
true,
|
|
2581
|
+
this.hasSiblingCommitInFlight(replayTransaction) ? replayTransaction : undefined
|
|
2582
|
+
);
|
|
1948
2583
|
throw error;
|
|
1949
2584
|
}
|
|
1950
|
-
transaction = replayTransaction;
|
|
1951
2585
|
}
|
|
1952
2586
|
// with options.transaction set this is a retry round — the save loop above already
|
|
1953
2587
|
// re-staged the writes into it
|
|
1954
2588
|
this.assertDatabaseCommitAllowed(transaction);
|
|
1955
2589
|
this.markNativeCommitSubmitted();
|
|
1956
2590
|
commitResolution = transaction.commit() as Promise<void>;
|
|
2591
|
+
this.setHandleCommitInFlight(transaction, true);
|
|
2592
|
+
if (this.hasLeaseProtectedWrite) {
|
|
2593
|
+
hadLeaseProtectedWrite = true;
|
|
2594
|
+
this.retainTransactionLockHolds(transaction);
|
|
2595
|
+
}
|
|
1957
2596
|
recordCommitLatency(commitResolution, performance.now());
|
|
1958
2597
|
// Write-queue-depth accounting for this replay commit happens uniformly below, via
|
|
1959
2598
|
// trackOutstandingCommit(commitResolution) — see that function's comment. Omitting
|
|
@@ -1973,10 +2612,10 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1973
2612
|
this.releaseRetainedWriteIntents();
|
|
1974
2613
|
} else {
|
|
1975
2614
|
// no more reads need to be performed, just commit/abort based if there are any writes
|
|
1976
|
-
this.detachOwnedTransaction(); // any further operations operate immediately
|
|
2615
|
+
if (!isImmediateCommit) this.detachOwnedTransaction(); // any further operations operate immediately
|
|
1977
2616
|
if (transaction) {
|
|
1978
2617
|
this.writes = this.writes.filter((write) => write); // filter out removed entries
|
|
1979
|
-
if (this.
|
|
2618
|
+
if (this.hasSavedWriteForHandle(transaction, isImmediateCommit) || hasDurableLog(transaction)) {
|
|
1980
2619
|
// Commit retries can construct fresh ranges on this live handle after read ownership ends.
|
|
1981
2620
|
readTransactionOwners.delete(transaction);
|
|
1982
2621
|
// Re-fence before submitting: the loop above skips operations already marked saved, so
|
|
@@ -1984,22 +2623,35 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1984
2623
|
// retry/replay path re-saves every operation and is fenced there instead. Lease expiry
|
|
1985
2624
|
// only: an unlock() inside the lease leaves the staged write valid, an elapsed lease
|
|
1986
2625
|
// does not, whether or not the caller also unlocked.
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2626
|
+
if (this.hasLeaseProtectedWrite && !hasDurableLog(transaction)) {
|
|
2627
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
2628
|
+
const write = this.writes[i];
|
|
2629
|
+
if (
|
|
2630
|
+
!write?.saved ||
|
|
2631
|
+
(isImmediateCommit
|
|
2632
|
+
? write.stagedOn !== transaction
|
|
2633
|
+
: Boolean(write.stagedOn) && write.stagedOn !== transaction)
|
|
2634
|
+
)
|
|
2635
|
+
continue;
|
|
2636
|
+
const lapsed = write.lockHandle;
|
|
2637
|
+
if (!lapsed?.isLeaseExpired()) continue;
|
|
2638
|
+
abortNativeTransaction(
|
|
2639
|
+
transaction,
|
|
2640
|
+
'aborting a transaction whose record lock lapsed before submission',
|
|
2641
|
+
this.writes[0]?.store ?? this.db,
|
|
2642
|
+
this.startedFrom
|
|
2643
|
+
);
|
|
2644
|
+
// Every other terminal exit from commit() runs the logical cleanup too. Throwing
|
|
2645
|
+
// straight out would strand the OTHER locks this transaction holds, its staged
|
|
2646
|
+
// blobs, and the context's back-reference to a CLOSED transaction — and
|
|
2647
|
+
// transaction()'s onComplete has no rejection path to run it later.
|
|
2648
|
+
try {
|
|
2649
|
+
this.abort(false, true, transaction);
|
|
2650
|
+
} catch (error) {
|
|
2651
|
+
harperLogger.debug?.('cleaning up a transaction whose record lock lapsed', error);
|
|
2652
|
+
}
|
|
2653
|
+
throw lockNotHeldError(lapsed);
|
|
2001
2654
|
}
|
|
2002
|
-
throw lockNotHeldError(lapsed);
|
|
2003
2655
|
}
|
|
2004
2656
|
// The transaction was created with coordinatedRetry:true (see
|
|
2005
2657
|
// getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
|
|
@@ -2008,6 +2660,11 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2008
2660
|
this.assertDatabaseCommitAllowed(transaction);
|
|
2009
2661
|
this.markNativeCommitSubmitted();
|
|
2010
2662
|
commitResolution = transaction.commit();
|
|
2663
|
+
this.setHandleCommitInFlight(transaction, true);
|
|
2664
|
+
if (this.hasLeaseProtectedWrite) {
|
|
2665
|
+
hadLeaseProtectedWrite = true;
|
|
2666
|
+
this.retainTransactionLockHolds(transaction);
|
|
2667
|
+
}
|
|
2011
2668
|
// Record how long this commit stays outstanding (submit → settle) as a distribution
|
|
2012
2669
|
// metric. This is the same clock the overload check uses (trackOutstandingCommit
|
|
2013
2670
|
// stamps each attempt at submit), so a rising p99/p999 is the leading indicator for the
|
|
@@ -2025,7 +2682,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2025
2682
|
} else {
|
|
2026
2683
|
try {
|
|
2027
2684
|
commitResolution = transaction.abort();
|
|
2028
|
-
} catch {
|
|
2685
|
+
} catch (abortError) {
|
|
2686
|
+
if ((abortError as any)?.code === 'ERR_TRANSACTION_ABANDONED') {
|
|
2687
|
+
harperLogger.error?.(
|
|
2688
|
+
`Aborted a transaction with a durable log entry (empty write set), ${describeCommitIdentity(this.db, this.startedFrom, transaction)}`,
|
|
2689
|
+
abortError
|
|
2690
|
+
);
|
|
2691
|
+
}
|
|
2029
2692
|
// The transaction has uncommitted writes that were already cleared from
|
|
2030
2693
|
// this.writes by a concurrent immediate-commit path (e.g. writes made with
|
|
2031
2694
|
// an explicitly-reused closed transaction). Those writes are handled by the
|
|
@@ -2039,41 +2702,56 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2039
2702
|
// Read the table off the write itself, not this.db, which is whichever table first
|
|
2040
2703
|
// claimed this per-database transaction in txnForContext and so can name the wrong
|
|
2041
2704
|
// table when a transaction spans more than one table in the same database.
|
|
2042
|
-
trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
|
|
2705
|
+
trackOutstandingCommit(commitResolution, this.writes[0]?.store ?? this.db, this.startedFrom, transaction);
|
|
2043
2706
|
// Every retry round and every chained store re-enters here and must inherit the chain
|
|
2044
|
-
// root's clock rather than restart it
|
|
2045
|
-
|
|
2046
|
-
|
|
2707
|
+
// root's clock rather than restart it (or the per-handle clock on an unchained
|
|
2708
|
+
// ImmediateTransaction), so only the first submission stamps.
|
|
2709
|
+
const carrier = this.budgetCarrier(transaction);
|
|
2710
|
+
if (carrier.commitStartedAt == null) carrier.commitStartedAt = performance.now();
|
|
2047
2711
|
const completions = [];
|
|
2048
2712
|
const commitOutcome = commitResolution.then(
|
|
2049
2713
|
(commitResult) => {
|
|
2714
|
+
const txnWithRetry = transaction as RocksTransactionWithRetry;
|
|
2050
2715
|
if (commitResult === RETRY_NOW_VALUE) {
|
|
2051
|
-
this.
|
|
2052
|
-
|
|
2716
|
+
this.setHandleCommitInFlight(transaction, false);
|
|
2717
|
+
const retries = (txnWithRetry.retries = (txnWithRetry.retries ?? 0) + 1);
|
|
2718
|
+
if (!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) this.retries = retries;
|
|
2719
|
+
harperLogger.debug?.('coordinated retry', transaction.id, retries);
|
|
2053
2720
|
// Mark this specific native transaction as a retry so RocksTransactionLogStore
|
|
2054
2721
|
// skips re-writing its already-staged txn-log entries (#2).
|
|
2055
|
-
|
|
2056
|
-
const
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2722
|
+
txnWithRetry.isRetry = true;
|
|
2723
|
+
const logged = hasDurableLog(transaction);
|
|
2724
|
+
const pastBudget = this.elapsedPastCommitBudget(transaction);
|
|
2725
|
+
if (pastBudget) {
|
|
2726
|
+
if (logged) this.noteOverdueLoggedCommit(transaction, pastBudget);
|
|
2727
|
+
else this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
2728
|
+
}
|
|
2729
|
+
if (retries > MAX_RETRIES && !this.sourceApply && !logged) {
|
|
2730
|
+
// giving up: poison and abort the whole linked chain so no link leaks its native
|
|
2731
|
+
// handle / read snapshot — or any unpublished transaction-log position — until GC.
|
|
2732
|
+
this.abortChainAfterRetries(transaction);
|
|
2733
|
+
throw new ServerError(
|
|
2734
|
+
`After ${MAX_RETRIES} coordinated retries, unable to commit transaction, transaction is in conflict with ongoing writes`
|
|
2735
|
+
);
|
|
2736
|
+
}
|
|
2737
|
+
if ((this.sourceApply || logged) && retries > 0 && retries % MAX_RETRIES === 0) {
|
|
2738
|
+
harperLogger.warn?.(
|
|
2739
|
+
`${this.sourceApply ? 'Source-applied' : 'Logged'} transaction ${transaction.id} still in conflict after ${retries} coordinated retries; continuing to retry`
|
|
2740
|
+
);
|
|
2741
|
+
}
|
|
2742
|
+
// Randomized so two uncapped logged commits crossing on the same keys stop waking in lockstep.
|
|
2743
|
+
if (logged && retries > 2) {
|
|
2744
|
+
const backoff = delay(Math.ceil(Math.min(retries * 2, MAX_RETRY_DELAY_MS) * Math.random()));
|
|
2745
|
+
trackOutstandingCommit(backoff, this.writes[0]?.store ?? this.db, this.startedFrom, transaction);
|
|
2746
|
+
return backoff.then(() => this.commit({ ...options, transaction }));
|
|
2074
2747
|
}
|
|
2075
2748
|
return this.commit({ ...options, transaction });
|
|
2076
2749
|
}
|
|
2750
|
+
const committedTransaction = transaction;
|
|
2751
|
+
const committedImmediate = Boolean(
|
|
2752
|
+
(committedTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit
|
|
2753
|
+
);
|
|
2754
|
+
this.setHandleCommitted(transaction);
|
|
2077
2755
|
// onCommit may be async (e.g. RocksTransactionLogStore emits 'aftercommit'). Surface a
|
|
2078
2756
|
// rejection — or a synchronous throw — via logging rather than failing the commit, since
|
|
2079
2757
|
// the write is already durable.
|
|
@@ -2084,7 +2762,15 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2084
2762
|
} catch (error) {
|
|
2085
2763
|
harperLogger.warn?.('onCommit handler failed after commit', error);
|
|
2086
2764
|
}
|
|
2087
|
-
if (this.
|
|
2765
|
+
if (this.recordLocks) this.noteCommittedLockVersions(committedTransaction);
|
|
2766
|
+
releaseTransactionLockHolds(transaction);
|
|
2767
|
+
if (
|
|
2768
|
+
committedImmediate
|
|
2769
|
+
? this.open !== TRANSACTION_STATE.OPEN && !this.hasSiblingCommitInFlight(committedTransaction)
|
|
2770
|
+
: !this.saveCommits || !this.hasSiblingCommitInFlight(committedTransaction)
|
|
2771
|
+
)
|
|
2772
|
+
this.releaseRecordLocks();
|
|
2773
|
+
if (this.next && !committedImmediate) {
|
|
2088
2774
|
// never forward options.transaction (a retry/replay round's HEAD-store handle) to
|
|
2089
2775
|
// the next store — it must commit its own writes through its own transaction
|
|
2090
2776
|
let nextCommit: MaybePromise<CommitResolution>;
|
|
@@ -2105,7 +2791,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2105
2791
|
// if we want to wait for replication confirmation, we need to track the transaction times
|
|
2106
2792
|
// and when replication notifications come in, we count the number of confirms until we reach the desired number
|
|
2107
2793
|
const databaseName = this.writes[0]?.store.rootStore.databaseName;
|
|
2108
|
-
const lastEntry = this.lastConfirmableEntry();
|
|
2794
|
+
const lastEntry = this.lastConfirmableEntry(true);
|
|
2109
2795
|
if (confirmReplication && databaseName && lastEntry) {
|
|
2110
2796
|
completions.push(
|
|
2111
2797
|
confirmReplication(databaseName, (lastEntry as any).version, this.replicatedConfirmation)
|
|
@@ -2118,23 +2804,43 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2118
2804
|
// later write retained survives). Deferred until here so a retry that *would* have
|
|
2119
2805
|
// referenced the blob can flip skipped/superseded back to false first.
|
|
2120
2806
|
for (const write of this.writes) {
|
|
2121
|
-
if (write?.
|
|
2807
|
+
if (!write?.saved || !write.savedBlobs) continue;
|
|
2808
|
+
if (
|
|
2809
|
+
committedImmediate
|
|
2810
|
+
? write.stagedOn !== committedTransaction
|
|
2811
|
+
: Boolean(write.stagedOn) && write.stagedOn !== committedTransaction
|
|
2812
|
+
)
|
|
2813
|
+
continue;
|
|
2814
|
+
if (
|
|
2815
|
+
(write.skipped &&
|
|
2816
|
+
!(
|
|
2817
|
+
write.blobsAuditReferenced &&
|
|
2818
|
+
(write.appendedTo === committedTransaction || hasDurableLog(write.appendedTo))
|
|
2819
|
+
)) ||
|
|
2820
|
+
(write.superseded && !write.blobsAuditReferenced)
|
|
2821
|
+
)
|
|
2122
2822
|
cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
|
|
2123
2823
|
}
|
|
2124
|
-
|
|
2824
|
+
const committedStagedTxnTime = this.getHandleStagedTxnTime(committedTransaction);
|
|
2125
2825
|
// now reset transactions tracking; this transaction be reused and committed again
|
|
2126
|
-
this.
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2826
|
+
this.clearWrites(committedTransaction);
|
|
2827
|
+
if (!committedImmediate) {
|
|
2828
|
+
this.retries = 0;
|
|
2829
|
+
if (options.doneWriting) this.endScopeOwnership();
|
|
2830
|
+
this.releaseContext(!!options.doneWriting);
|
|
2831
|
+
}
|
|
2832
|
+
let txnTime = committedStagedTxnTime || this.timestamp || 0;
|
|
2833
|
+
if (
|
|
2834
|
+
(!committedImmediate || this.open !== TRANSACTION_STATE.OPEN) &&
|
|
2835
|
+
this.writes.length === 0 &&
|
|
2836
|
+
!this.hasActiveCommitHandle()
|
|
2837
|
+
)
|
|
2838
|
+
this.timestamp = 0; // reset the timestamp as well
|
|
2133
2839
|
return Promise.all(completions).then(
|
|
2134
2840
|
() => {
|
|
2135
2841
|
// Only once the chained store's commit has settled, as on the synchronous path: a
|
|
2136
2842
|
// partially failed mid-scope commit must not leave the scope resumable.
|
|
2137
|
-
this.completeMidScopeCommit(options);
|
|
2843
|
+
this.completeMidScopeCommit(options, committedTransaction);
|
|
2138
2844
|
return {
|
|
2139
2845
|
txnTime,
|
|
2140
2846
|
};
|
|
@@ -2143,12 +2849,15 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2143
2849
|
// As on the synchronous path: a completion that failed (a chained store's commit,
|
|
2144
2850
|
// a replication confirmation) leaves this commit partly landed, so ownership goes
|
|
2145
2851
|
// with it rather than letting a later commit rotate on top.
|
|
2146
|
-
|
|
2852
|
+
(committedTransaction as RocksTransactionWithRetry).committed = true;
|
|
2853
|
+
if (!committedImmediate) this.endScopeOwnership();
|
|
2147
2854
|
throw error;
|
|
2148
2855
|
}
|
|
2149
2856
|
);
|
|
2150
2857
|
},
|
|
2151
2858
|
(error) => {
|
|
2859
|
+
const txnWithRetry = transaction as RocksTransactionWithRetry;
|
|
2860
|
+
this.setHandleCommitInFlight(transaction, false);
|
|
2152
2861
|
// Coordinated transactions surface conflicts as RETRY_NOW (handled in the
|
|
2153
2862
|
// resolve branch above) and never reach here with ERR_BUSY. But not every
|
|
2154
2863
|
// write transaction is coordinated — a write that reaches save() with no
|
|
@@ -2165,8 +2874,9 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2165
2874
|
if (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') {
|
|
2166
2875
|
// if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
|
|
2167
2876
|
// for future transactions
|
|
2168
|
-
|
|
2169
|
-
|
|
2877
|
+
const retries = (txnWithRetry.retries = (txnWithRetry.retries ?? 0) + 1);
|
|
2878
|
+
if (!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) this.retries = retries;
|
|
2879
|
+
harperLogger.debug?.('retrying', transaction.id, retries);
|
|
2170
2880
|
// ERR_BUSY and ERR_TRY_AGAIN are both retried by recommitting the SAME native
|
|
2171
2881
|
// transaction. ERR_BUSY recovers because the save loop re-writes each key, re-tracking
|
|
2172
2882
|
// it at the current sequence. ERR_TRY_AGAIN — a snapshot stranded outside the memtable
|
|
@@ -2179,95 +2889,155 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2179
2889
|
// already-staged change-feed entry publishes only when the retry really commits — no
|
|
2180
2890
|
// premature publish, and no fresh-transaction replay that would drop the entry.
|
|
2181
2891
|
// Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
|
|
2182
|
-
|
|
2892
|
+
txnWithRetry.isRetry = true;
|
|
2893
|
+
if ((error as any)?.hasLog === true) {
|
|
2894
|
+
txnWithRetry.hasStagedLog = true;
|
|
2895
|
+
}
|
|
2896
|
+
const logged = hasDurableLog(transaction, error as any);
|
|
2183
2897
|
// Before the backoff gate below: the budget can already be spent on the first
|
|
2184
2898
|
// retry when an earlier store in this chain consumed it.
|
|
2185
|
-
const pastBudget = this.elapsedPastCommitBudget();
|
|
2186
|
-
if (pastBudget)
|
|
2187
|
-
|
|
2899
|
+
const pastBudget = this.elapsedPastCommitBudget(transaction);
|
|
2900
|
+
if (pastBudget) {
|
|
2901
|
+
if (logged) this.noteOverdueLoggedCommit(transaction, pastBudget);
|
|
2902
|
+
else this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
2903
|
+
}
|
|
2904
|
+
if (retries > 2) {
|
|
2188
2905
|
// Transactions applying data from a canonical source of truth (replication peer or
|
|
2189
2906
|
// external caching source) must never drop a write on a transient conflict: there is no
|
|
2190
2907
|
// re-subscribe / sequence-id-resume path, so a dropped write would leave this node
|
|
2191
2908
|
// permanently diverged (harper-pro#348). Such transactions retry without a cap; the source
|
|
2192
2909
|
// apply loop serializes commits (backpressure), so contention clears rather than
|
|
2193
|
-
// compounding.
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
}
|
|
2910
|
+
// compounding. Durably logged transactions likewise retry without a cap so a written
|
|
2911
|
+
// transaction-log entry is never abandoned.
|
|
2912
|
+
const neverDropOnConflict = this.sourceApply || logged;
|
|
2913
|
+
if (retries > MAX_RETRIES && !neverDropOnConflict) {
|
|
2914
|
+
// giving up: poison and abort the whole linked chain so no link leaks its native
|
|
2915
|
+
// handle / read snapshot until GC.
|
|
2916
|
+
this.abortChainAfterRetries(transaction);
|
|
2917
|
+
throw new ServerError(
|
|
2918
|
+
`After ${MAX_RETRIES} retries, unable to commit transaction, transaction is in conflict with ongoing writes`
|
|
2919
|
+
);
|
|
2920
|
+
}
|
|
2921
|
+
// Uncapped retry can otherwise stall silently (debug logging is off in production);
|
|
2922
|
+
// surface periodic visibility into a stalled source-apply or logged commit.
|
|
2923
|
+
if (neverDropOnConflict && retries % MAX_RETRIES === 0) {
|
|
2924
|
+
harperLogger.warn?.(
|
|
2925
|
+
`${this.sourceApply ? 'Source-applied' : 'Logged'} transaction ${transaction.id} still in conflict after ${retries} retries; continuing to retry`
|
|
2926
|
+
);
|
|
2211
2927
|
}
|
|
2212
2928
|
// start delaying, back off to try to space out transactions and avoid excessive conflicts
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2929
|
+
const backoff = delay(Math.min(retries * retries, MAX_RETRY_DELAY_MS));
|
|
2930
|
+
if (logged) {
|
|
2931
|
+
// waitForDrain() must outlast the backoff too
|
|
2932
|
+
trackOutstandingCommit(backoff, this.writes[0]?.store ?? this.db, this.startedFrom, transaction);
|
|
2933
|
+
}
|
|
2934
|
+
return backoff.then(() => this.commit({ ...options, transaction }));
|
|
2216
2935
|
}
|
|
2217
2936
|
return this.commit({ ...options, transaction }); // try again
|
|
2218
2937
|
} else {
|
|
2219
2938
|
// terminal (non-conflict) failure: release the native handle so it doesn't leak;
|
|
2220
2939
|
// usually already released by the failed commit itself, abort for the unexpected
|
|
2221
2940
|
// case (same defensive pattern as the retry-exhaustion give-up above)
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
harperLogger.debug?.('aborting transaction after failed commit', abortError);
|
|
2941
|
+
if ((error as any)?.hasLog) {
|
|
2942
|
+
txnWithRetry.isRetry = true;
|
|
2943
|
+
txnWithRetry.hasStagedLog = true;
|
|
2226
2944
|
}
|
|
2945
|
+
abortNativeTransaction(
|
|
2946
|
+
transaction,
|
|
2947
|
+
'aborting transaction after failed commit',
|
|
2948
|
+
this.writes[0]?.store ?? this.db,
|
|
2949
|
+
this.startedFrom
|
|
2950
|
+
);
|
|
2227
2951
|
// A terminal failure is just as final as a success — release the context's
|
|
2228
2952
|
// back-reference while handling the native transaction. performCommit's outer
|
|
2229
2953
|
// rejection handler then aborts the wrapper and its linked chain.
|
|
2230
2954
|
// A failed commit must never be followed by a resumed segment: this generation is
|
|
2231
2955
|
// finished and its durability is unknown, so ownership goes with it.
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2956
|
+
if (
|
|
2957
|
+
!isImmediateCommit &&
|
|
2958
|
+
!this.hasWriteOwnedByActiveCommit(transaction, this.hasSiblingCommitInFlight(transaction), true, true)
|
|
2959
|
+
) {
|
|
2960
|
+
this.endScopeOwnership();
|
|
2961
|
+
this.releaseContext(!!options.doneWriting);
|
|
2962
|
+
}
|
|
2963
|
+
if (!hasDurableLog(transaction, error as any)) {
|
|
2964
|
+
for (const write of this.writes) {
|
|
2965
|
+
if (
|
|
2966
|
+
!write?.saved ||
|
|
2967
|
+
!write.savedBlobs ||
|
|
2968
|
+
(isImmediateCommit
|
|
2969
|
+
? write.stagedOn !== transaction
|
|
2970
|
+
: Boolean(write.stagedOn) && write.stagedOn !== transaction)
|
|
2971
|
+
)
|
|
2972
|
+
continue;
|
|
2973
|
+
cleanupUnusedBlobs(
|
|
2974
|
+
write.savedBlobs,
|
|
2975
|
+
collectRetainedFileIds(write.store.getEntry(write.key)?.value)
|
|
2976
|
+
);
|
|
2977
|
+
}
|
|
2978
|
+
}
|
|
2979
|
+
this.clearWrites(transaction);
|
|
2980
|
+
releaseTransactionLockHolds(transaction);
|
|
2981
|
+
if (
|
|
2982
|
+
(!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) &&
|
|
2983
|
+
!this.hasActiveCommitHandle() &&
|
|
2984
|
+
this.writes.length === 0
|
|
2985
|
+
)
|
|
2986
|
+
this.releaseRecordLocks();
|
|
2987
|
+
if ((!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) && !this.hasActiveCommitHandle())
|
|
2988
|
+
this.timestamp = 0;
|
|
2236
2989
|
throw error;
|
|
2237
2990
|
}
|
|
2238
2991
|
}
|
|
2239
2992
|
);
|
|
2240
|
-
//
|
|
2241
|
-
//
|
|
2242
|
-
//
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
chainRoot.commitStartedAt = undefined;
|
|
2249
|
-
return resolution;
|
|
2250
|
-
},
|
|
2251
|
-
(error) => {
|
|
2252
|
-
chainRoot.commitStartedAt = undefined;
|
|
2253
|
-
throw error;
|
|
2254
|
-
}
|
|
2255
|
-
);
|
|
2993
|
+
// Release any lock holds remaining if `commitOutcome` rejects (for example when a chained
|
|
2994
|
+
// store's commit or a completion fails) through the returned promise rather than a second
|
|
2995
|
+
// subscriber that would mark an unawaited commit rejection as handled.
|
|
2996
|
+
if (!hadLeaseProtectedWrite) return commitOutcome;
|
|
2997
|
+
return commitOutcome.then(undefined, (error) => {
|
|
2998
|
+
releaseTransactionLockHolds(transaction);
|
|
2999
|
+
throw error;
|
|
3000
|
+
});
|
|
2256
3001
|
}
|
|
2257
3002
|
for (const write of this.writes) {
|
|
2258
|
-
if (write?.
|
|
3003
|
+
if (!write?.saved) continue;
|
|
3004
|
+
if (
|
|
3005
|
+
isImmediateCommit
|
|
3006
|
+
? write.stagedOn !== transaction
|
|
3007
|
+
: this.hasSiblingCommitInFlight(transaction) && Boolean(write.stagedOn) && write.stagedOn !== transaction
|
|
3008
|
+
)
|
|
3009
|
+
continue;
|
|
3010
|
+
if (
|
|
3011
|
+
write.savedBlobs &&
|
|
3012
|
+
((write.skipped &&
|
|
3013
|
+
!(write.blobsAuditReferenced && (write.appendedTo === transaction || hasDurableLog(write.appendedTo)))) ||
|
|
3014
|
+
(write.superseded && !write.blobsAuditReferenced))
|
|
3015
|
+
)
|
|
2259
3016
|
cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
|
|
2260
3017
|
}
|
|
2261
|
-
if (this.recordLocks) this.noteCommittedLockVersions();
|
|
2262
|
-
this.
|
|
2263
|
-
this.
|
|
2264
|
-
if (
|
|
2265
|
-
|
|
3018
|
+
if (this.recordLocks) this.noteCommittedLockVersions(transaction);
|
|
3019
|
+
const committedStagedTxnTime = this.getHandleStagedTxnTime(transaction);
|
|
3020
|
+
this.clearWrites(transaction);
|
|
3021
|
+
if (
|
|
3022
|
+
isImmediateCommit
|
|
3023
|
+
? this.open !== TRANSACTION_STATE.OPEN && !this.hasActiveCommitHandle() && this.writes.length === 0
|
|
3024
|
+
: !this.saveCommits || (!this.hasActiveCommitHandle() && this.writes.length === 0)
|
|
3025
|
+
)
|
|
3026
|
+
this.releaseRecordLocks();
|
|
3027
|
+
if (!isImmediateCommit) {
|
|
3028
|
+
if (options.doneWriting) this.endScopeOwnership();
|
|
3029
|
+
this.releaseContext(!!options.doneWriting);
|
|
3030
|
+
}
|
|
2266
3031
|
const txnResolution: CommitResolution = {
|
|
2267
|
-
txnTime: this.timestamp,
|
|
3032
|
+
txnTime: committedStagedTxnTime || this.timestamp,
|
|
2268
3033
|
};
|
|
2269
|
-
|
|
2270
|
-
|
|
3034
|
+
if (
|
|
3035
|
+
(!isImmediateCommit || this.open !== TRANSACTION_STATE.OPEN) &&
|
|
3036
|
+
this.writes.length === 0 &&
|
|
3037
|
+
!this.hasActiveCommitHandle()
|
|
3038
|
+
)
|
|
3039
|
+
this.timestamp = 0;
|
|
3040
|
+
if (this.next && !isImmediateCommit) {
|
|
2271
3041
|
// now run any other transactions
|
|
2272
3042
|
options.timestamp = txnResolution.txnTime;
|
|
2273
3043
|
// as above: the next store must not inherit this store's explicit native transaction
|
|
@@ -2286,7 +3056,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2286
3056
|
(nextResolution) => {
|
|
2287
3057
|
// Only once the chained store's own commit has SETTLED: rotating first would leave the
|
|
2288
3058
|
// scope resumable after a partially failed mid-scope commit.
|
|
2289
|
-
this.completeMidScopeCommit(options);
|
|
3059
|
+
this.completeMidScopeCommit(options, transaction);
|
|
2290
3060
|
return {
|
|
2291
3061
|
txnTime: txnResolution.txnTime,
|
|
2292
3062
|
next: nextResolution,
|
|
@@ -2303,17 +3073,17 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2303
3073
|
);
|
|
2304
3074
|
txnResolution.next = nextResolution as any;
|
|
2305
3075
|
}
|
|
2306
|
-
this.completeMidScopeCommit(options);
|
|
3076
|
+
this.completeMidScopeCommit(options, transaction);
|
|
2307
3077
|
return txnResolution;
|
|
2308
3078
|
});
|
|
2309
3079
|
} catch (error) {
|
|
2310
|
-
this.setCommitPhase(false);
|
|
2311
|
-
this.abortAfterCommitError(error);
|
|
3080
|
+
if (inCommitPhase) this.setCommitPhase(false);
|
|
3081
|
+
this.abortAfterCommitError(error, transaction);
|
|
2312
3082
|
}
|
|
2313
3083
|
if ((commitResult as Promise<CommitResolution>)?.then)
|
|
2314
3084
|
return (commitResult as Promise<CommitResolution>).catch((error) => {
|
|
2315
|
-
this.setCommitPhase(false);
|
|
2316
|
-
this.abortAfterCommitError(error);
|
|
3085
|
+
if (inCommitPhase) this.setCommitPhase(false);
|
|
3086
|
+
this.abortAfterCommitError(error, transaction);
|
|
2317
3087
|
});
|
|
2318
3088
|
return commitResult;
|
|
2319
3089
|
}
|
|
@@ -2342,8 +3112,10 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2342
3112
|
this.snapshotFree = false;
|
|
2343
3113
|
}
|
|
2344
3114
|
|
|
2345
|
-
private completeMidScopeCommit(options: CommitOptions): void {
|
|
3115
|
+
private completeMidScopeCommit(options: CommitOptions, committedTransaction?: RocksTransaction | null): void {
|
|
3116
|
+
if ((committedTransaction as RocksTransactionWithRetry | null | undefined)?.immediateCommit) return;
|
|
2346
3117
|
this.next = null;
|
|
3118
|
+
if (this.saveCommits && this.hasActiveCommitHandle()) return;
|
|
2347
3119
|
this.rotateAfterMidScopeCommit(options);
|
|
2348
3120
|
}
|
|
2349
3121
|
|
|
@@ -2361,75 +3133,149 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2361
3133
|
this.open = TRANSACTION_STATE.OPEN;
|
|
2362
3134
|
this.snapshotFree = true;
|
|
2363
3135
|
this.writesAbandoned = false;
|
|
3136
|
+
if (this.#context && (this.#context.transaction === this || this.#context.transaction === RELEASED_TRANSACTION)) {
|
|
3137
|
+
this.#context.transaction = this;
|
|
3138
|
+
}
|
|
2364
3139
|
}
|
|
2365
3140
|
|
|
2366
3141
|
private assertDatabaseCommitAllowed(transaction: RocksTransaction): void {
|
|
3142
|
+
if (hasDurableLog(transaction)) return;
|
|
2367
3143
|
const rootStore = this.writes[0]?.store?.rootStore;
|
|
2368
3144
|
if (!databaseCommitsSuspended(rootStore)) return;
|
|
2369
3145
|
const retryable = !this.root && !this.snapshotFree;
|
|
2370
3146
|
if (this.transaction === transaction) this.detachOwnedTransaction();
|
|
2371
|
-
abortNativeTransaction(
|
|
3147
|
+
abortNativeTransaction(
|
|
3148
|
+
transaction,
|
|
3149
|
+
'aborting a transaction while its database is closing',
|
|
3150
|
+
this.writes[0]?.store ?? this.db,
|
|
3151
|
+
this.startedFrom
|
|
3152
|
+
);
|
|
2372
3153
|
try {
|
|
2373
|
-
this.abort();
|
|
3154
|
+
this.abort(false, true, transaction);
|
|
2374
3155
|
} catch (error) {
|
|
2375
3156
|
harperLogger.debug?.('cleaning up a transaction while its database is closing', error);
|
|
2376
3157
|
}
|
|
2377
3158
|
throw new DatabaseClosingError(rootStore?.databaseName ?? rootStore?.path ?? 'unknown', retryable);
|
|
2378
3159
|
}
|
|
2379
3160
|
|
|
2380
|
-
protected abortAfterCommitError(error): never {
|
|
3161
|
+
protected abortAfterCommitError(error, targetTransaction?: RocksTransaction): never {
|
|
2381
3162
|
try {
|
|
3163
|
+
const retryTxn = targetTransaction as RocksTransactionWithRetry | undefined;
|
|
3164
|
+
const retryTxnCommitted = this.isHandleCommitted(retryTxn);
|
|
3165
|
+
const retryTxnInFlight = this.isHandleCommitInFlight(retryTxn);
|
|
3166
|
+
if (retryTxn && retryTxnCommitted) retryTxn.committed = true;
|
|
3167
|
+
if (retryTxn) this.releaseActiveCommitHandle(retryTxn);
|
|
3168
|
+
if (retryTxn && retryTxn !== this.transaction && !retryTxnCommitted && !retryTxnInFlight) {
|
|
3169
|
+
releaseTransactionLockHolds(retryTxn);
|
|
3170
|
+
abortNativeTransaction(
|
|
3171
|
+
retryTxn,
|
|
3172
|
+
'aborting a retry transaction after pre-commit failure',
|
|
3173
|
+
this.writes[0]?.store ?? this.db,
|
|
3174
|
+
this.startedFrom
|
|
3175
|
+
);
|
|
3176
|
+
}
|
|
2382
3177
|
// Not "always retain": abort(true) keeps the native handle only while read iterators still
|
|
2383
3178
|
// own it. resources/transaction.ts's own post-error abort passes the same argument, so the
|
|
2384
3179
|
// two layers cannot contradict each other one frame apart.
|
|
2385
|
-
this.abort(true);
|
|
3180
|
+
this.abort(true, true, targetTransaction);
|
|
2386
3181
|
} catch (abortError) {
|
|
2387
3182
|
harperLogger.debug?.('aborting transaction after a failed commit', abortError);
|
|
2388
3183
|
}
|
|
2389
3184
|
throw error;
|
|
2390
3185
|
}
|
|
2391
|
-
abort(retainReadTransaction = false, cascade = true): void {
|
|
2392
|
-
const
|
|
2393
|
-
|
|
3186
|
+
abort(retainReadTransaction = false, cascade = true, targetTransaction?: RocksTransaction): void {
|
|
3187
|
+
const nativeTxn = targetTransaction ?? this.transaction;
|
|
3188
|
+
const isImmediateTarget = Boolean((targetTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
3189
|
+
const hasSiblingInFlight = this.hasSiblingCommitInFlight(targetTransaction);
|
|
3190
|
+
const preserveActiveWrites =
|
|
3191
|
+
isImmediateTarget || this.hasWriteOwnedByActiveCommit(targetTransaction, hasSiblingInFlight, true);
|
|
3192
|
+
const preserveNonImmediateActiveWrites =
|
|
3193
|
+
isImmediateTarget || this.hasWriteOwnedByActiveCommit(targetTransaction, hasSiblingInFlight, true, true);
|
|
3194
|
+
const cascadeNext = cascade && !preserveNonImmediateActiveWrites;
|
|
3195
|
+
const next = cascadeNext ? this.next : undefined;
|
|
3196
|
+
if (cascadeNext && !retainReadTransaction) this.next = null;
|
|
2394
3197
|
try {
|
|
2395
|
-
const
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
3198
|
+
const ownsCurrentTransaction =
|
|
3199
|
+
!isImmediateTarget && (!hasSiblingInFlight || !targetTransaction || this.transaction === targetTransaction);
|
|
3200
|
+
const currentTxn = this.transaction as RocksTransactionWithRetry | null | undefined;
|
|
3201
|
+
const currentTxnActive =
|
|
3202
|
+
Boolean(currentTxn) &&
|
|
3203
|
+
!currentTxn.abortedByWrapper &&
|
|
3204
|
+
!this.isHandleCommitted(currentTxn) &&
|
|
3205
|
+
(this.isHandleCommitInFlight(currentTxn) ||
|
|
3206
|
+
hasDurableLog(currentTxn) ||
|
|
3207
|
+
(Boolean(targetTransaction) && currentTxn !== targetTransaction && this.isActiveCommitHandle(currentTxn)));
|
|
3208
|
+
if (ownsCurrentTransaction && !currentTxnActive) {
|
|
3209
|
+
if (this.transaction) this.releaseActiveCommitHandle(this.transaction);
|
|
3210
|
+
const hasOpenReadIterator =
|
|
3211
|
+
retainReadTransaction &&
|
|
3212
|
+
this.transaction &&
|
|
3213
|
+
(this.readTxnsUsed > 1 || (this.baseReadRefConsumed && this.readTxnsUsed > 0));
|
|
3214
|
+
if (hasOpenReadIterator) {
|
|
3215
|
+
this.releaseRetainedWriteIntents();
|
|
3216
|
+
if (!this.baseReadRefConsumed) {
|
|
3217
|
+
this.doneReadTxn();
|
|
3218
|
+
this.baseReadRefConsumed = true;
|
|
3219
|
+
}
|
|
3220
|
+
} else {
|
|
3221
|
+
while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
|
|
2404
3222
|
}
|
|
2405
|
-
|
|
2406
|
-
|
|
3223
|
+
// A write-only transaction never took a read reference (getReadTxn was never called), so the
|
|
3224
|
+
// loop above releases nothing even though save() created a native handle. Keep an iterator's
|
|
3225
|
+
// handle alive, but release every other remaining handle through the shared cleanup path.
|
|
3226
|
+
if (this.transaction && !hasOpenReadIterator) this.releaseReadTxn();
|
|
2407
3227
|
}
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
if (this.transaction && !hasOpenReadIterator) this.releaseReadTxn();
|
|
2412
|
-
this.open = TRANSACTION_STATE.CLOSED;
|
|
2413
|
-
this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
|
|
2414
|
-
this.drainCompletions();
|
|
3228
|
+
if (!isImmediateTarget) this.open = TRANSACTION_STATE.CLOSED;
|
|
3229
|
+
this.drainCompletions(targetTransaction, preserveActiveWrites);
|
|
3230
|
+
const targetLogged = hasDurableLog(targetTransaction);
|
|
2415
3231
|
for (const write of this.writes) {
|
|
2416
|
-
if (write
|
|
3232
|
+
if (!write) continue;
|
|
3233
|
+
if (preserveActiveWrites && this.isWriteOwnedByActiveCommit(write, targetTransaction, hasSiblingInFlight, true))
|
|
3234
|
+
continue;
|
|
3235
|
+
if (
|
|
3236
|
+
targetLogged &&
|
|
3237
|
+
write.saved &&
|
|
3238
|
+
(isImmediateTarget
|
|
3239
|
+
? write.stagedOn === targetTransaction
|
|
3240
|
+
: !write.stagedOn || write.stagedOn === targetTransaction)
|
|
3241
|
+
)
|
|
3242
|
+
continue;
|
|
3243
|
+
if (write.savedBlobs)
|
|
2417
3244
|
cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
|
|
2418
3245
|
}
|
|
2419
3246
|
} finally {
|
|
2420
3247
|
try {
|
|
2421
|
-
this.open = TRANSACTION_STATE.CLOSED;
|
|
2422
|
-
|
|
2423
|
-
this.
|
|
2424
|
-
this.
|
|
3248
|
+
if (!isImmediateTarget) this.open = TRANSACTION_STATE.CLOSED;
|
|
3249
|
+
const endScopeOnAbort = !isImmediateTarget && (!targetTransaction || !preserveNonImmediateActiveWrites);
|
|
3250
|
+
if (endScopeOnAbort) this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
|
|
3251
|
+
this.clearWrites(targetTransaction, true);
|
|
3252
|
+
if (targetTransaction) {
|
|
3253
|
+
releaseTransactionLockHolds(targetTransaction);
|
|
3254
|
+
}
|
|
3255
|
+
if ((!isImmediateTarget || this.open !== TRANSACTION_STATE.OPEN) && !this.hasActiveCommitHandle()) {
|
|
3256
|
+
this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
|
|
3257
|
+
this.retries = 0;
|
|
3258
|
+
}
|
|
3259
|
+
if (
|
|
3260
|
+
(isImmediateTarget
|
|
3261
|
+
? this.open !== TRANSACTION_STATE.OPEN && !this.hasActiveCommitHandle()
|
|
3262
|
+
: !preserveActiveWrites) &&
|
|
3263
|
+
this.writes.length === 0
|
|
3264
|
+
) {
|
|
3265
|
+
releaseTransactionLockHolds(nativeTxn);
|
|
3266
|
+
if (this.transaction && this.transaction !== nativeTxn) releaseTransactionLockHolds(this.transaction);
|
|
3267
|
+
this.releaseRecordLocks();
|
|
3268
|
+
}
|
|
2425
3269
|
// A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
|
|
2426
3270
|
// Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
|
|
2427
3271
|
// starting a fresh one) so the rest of the logical operation fails atomically via the
|
|
2428
3272
|
// poison check in addWrite()/commit(), rather than silently landing a later write on a
|
|
2429
3273
|
// brand-new transaction after an earlier one was rolled back (#1411). Releasing here would
|
|
2430
3274
|
// make that check see `undefined?.timedOut` and take the "start fresh" branch instead.
|
|
2431
|
-
|
|
2432
|
-
|
|
3275
|
+
if (endScopeOnAbort) {
|
|
3276
|
+
this.releaseContext(!this.timedOut && !this.disconnected);
|
|
3277
|
+
if (!this.root) this.releaseRequestAbortListener();
|
|
3278
|
+
}
|
|
2433
3279
|
} finally {
|
|
2434
3280
|
if (next && !(next.nativeCommitSubmitted && next.commitsInFlight)) {
|
|
2435
3281
|
try {
|
|
@@ -2455,9 +3301,9 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2455
3301
|
* Source-applied writes are exempt for the same reason they are exempt from the attempt cap:
|
|
2456
3302
|
* there is no resubscribe/sequence-resume path, so dropping one permanently diverges this node.
|
|
2457
3303
|
*/
|
|
2458
|
-
private elapsedPastCommitBudget(): number {
|
|
3304
|
+
private elapsedPastCommitBudget(transaction?: RocksTransaction): number {
|
|
2459
3305
|
if (this.sourceApply) return 0;
|
|
2460
|
-
const startedAt =
|
|
3306
|
+
const startedAt = this.budgetCarrier(transaction).commitStartedAt;
|
|
2461
3307
|
if (startedAt == null) return 0;
|
|
2462
3308
|
const elapsed = performance.now() - startedAt;
|
|
2463
3309
|
return elapsed > this.commitConflictBudget() ? elapsed : 0;
|
|
@@ -2478,13 +3324,18 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2478
3324
|
const elapsed = Math.round(elapsedMs);
|
|
2479
3325
|
const budget = this.commitConflictBudget();
|
|
2480
3326
|
const retryable = !this.root && !this.snapshotFree;
|
|
3327
|
+
const retries = (headTransaction as RocksTransactionWithRetry | undefined)?.retries ?? this.retries;
|
|
2481
3328
|
if (allowStuckCommitLog('abandon', performance.now())) {
|
|
2482
|
-
|
|
3329
|
+
let headTransactionId;
|
|
3330
|
+
try {
|
|
3331
|
+
headTransactionId = headTransaction?.id;
|
|
3332
|
+
} catch {}
|
|
3333
|
+
harperLogger.error?.(
|
|
2483
3334
|
`Abandoning a write transaction: its commit has been in write-intent conflict for ${elapsed}ms ` +
|
|
2484
|
-
`(exceeds the ${budget}ms limit) across ${
|
|
3335
|
+
`(exceeds the ${budget}ms limit) across ${retries} retries, ` +
|
|
2485
3336
|
describeCommitIdentity(this.writes[0]?.store, this.startedFrom, headTransaction) +
|
|
2486
3337
|
`.` +
|
|
2487
|
-
describeHolderCandidates(this.writes[0]?.store?.rootStore?.path,
|
|
3338
|
+
describeHolderCandidates(this.writes[0]?.store?.rootStore?.path, headTransactionId) +
|
|
2488
3339
|
` Another transaction holds a conflicting write intent and has not completed; the request is ` +
|
|
2489
3340
|
`failed with a 503${retryable ? '' : ' (not retryable — an earlier store in this transaction already committed)'} ` +
|
|
2490
3341
|
`rather than waiting further.`
|
|
@@ -2492,11 +3343,41 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2492
3343
|
}
|
|
2493
3344
|
this.abortChainAfterRetries(headTransaction);
|
|
2494
3345
|
throw new TransactionCommitConflictTimeoutError(
|
|
2495
|
-
`Commit was in conflict with ongoing writes for ${elapsed}ms, exceeding the ${budget}ms limit; transaction abandoned after ${
|
|
3346
|
+
`Commit was in conflict with ongoing writes for ${elapsed}ms, exceeding the ${budget}ms limit; transaction abandoned after ${retries} retries`,
|
|
2496
3347
|
retryable
|
|
2497
3348
|
);
|
|
2498
3349
|
}
|
|
2499
3350
|
|
|
3351
|
+
/**
|
|
3352
|
+
* A durably logged commit past its conflict budget cannot be aborted; release its deferred
|
|
3353
|
+
* record-lock holds (`holdBudgetSpent` prevents re-retaining them) and log once per chain/cooldown.
|
|
3354
|
+
*/
|
|
3355
|
+
private noteOverdueLoggedCommit(headTransaction: RocksTransaction, elapsedMs: number): void {
|
|
3356
|
+
(headTransaction as RocksTransactionWithRetry).holdBudgetSpent = true;
|
|
3357
|
+
releaseTransactionLockHolds(headTransaction);
|
|
3358
|
+
const carrier = this.budgetCarrier(headTransaction);
|
|
3359
|
+
if (carrier.commitOverdueLogged) return;
|
|
3360
|
+
if (!allowStuckCommitLog('overdue', performance.now())) return;
|
|
3361
|
+
carrier.commitOverdueLogged = true;
|
|
3362
|
+
const elapsed = Math.round(elapsedMs);
|
|
3363
|
+
const budget = this.commitConflictBudget();
|
|
3364
|
+
const retries = (headTransaction as RocksTransactionWithRetry | undefined)?.retries ?? this.retries;
|
|
3365
|
+
let headTransactionId;
|
|
3366
|
+
try {
|
|
3367
|
+
headTransactionId = headTransaction?.id;
|
|
3368
|
+
} catch {}
|
|
3369
|
+
harperLogger.error?.(
|
|
3370
|
+
`A durably logged write transaction is still in write-intent conflict after ${elapsed}ms ` +
|
|
3371
|
+
`(exceeds the ${budget}ms limit) across ${retries} retries, ` +
|
|
3372
|
+
describeCommitIdentity(this.writes[0]?.store, this.startedFrom, headTransaction) +
|
|
3373
|
+
`.` +
|
|
3374
|
+
describeHolderCandidates(this.writes[0]?.store?.rootStore?.path, headTransactionId) +
|
|
3375
|
+
` Because its transaction-log entry is already durable on disk, it cannot be aborted and ` +
|
|
3376
|
+
`will continue retrying until the conflicting write intent releases (pinning the local ` +
|
|
3377
|
+
`transaction-log read watermark for subscriptions and replication while uncommitted).`
|
|
3378
|
+
);
|
|
3379
|
+
}
|
|
3380
|
+
|
|
2500
3381
|
/**
|
|
2501
3382
|
* Give up on a chain of linked transactions after exhausting conflict retries: poison every link
|
|
2502
3383
|
* first, then abort each link's native transaction and release its DatabaseTransaction-level
|
|
@@ -2508,34 +3389,67 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2508
3389
|
* from the replay transaction; retry exhaustion aborts both.
|
|
2509
3390
|
*/
|
|
2510
3391
|
abortChainAfterRetries(headTransaction: RocksTransaction): void {
|
|
3392
|
+
const isImmediateCommit = Boolean((headTransaction as RocksTransactionWithRetry | undefined)?.immediateCommit);
|
|
3393
|
+
const hasSiblingInFlight = this.hasSiblingCommitInFlight(headTransaction);
|
|
3394
|
+
if (
|
|
3395
|
+
isImmediateCommit ||
|
|
3396
|
+
(hasSiblingInFlight && this.hasWriteOwnedByActiveCommit(headTransaction, hasSiblingInFlight, true, true))
|
|
3397
|
+
) {
|
|
3398
|
+
if (!isImmediateCommit && this.transaction === headTransaction) this.detachOwnedTransaction();
|
|
3399
|
+
releaseTransactionLockHolds(headTransaction);
|
|
3400
|
+
abortNativeTransaction(
|
|
3401
|
+
headTransaction,
|
|
3402
|
+
'aborting conflicted transaction in chain after exhausting retries',
|
|
3403
|
+
this.writes[0]?.store ?? this.db,
|
|
3404
|
+
this.startedFrom
|
|
3405
|
+
);
|
|
3406
|
+
try {
|
|
3407
|
+
this.abort(false, false, headTransaction);
|
|
3408
|
+
} catch (abortError) {
|
|
3409
|
+
harperLogger.debug?.('cleaning up conflicted transaction in chain after exhausting retries', abortError);
|
|
3410
|
+
}
|
|
3411
|
+
return;
|
|
3412
|
+
}
|
|
2511
3413
|
for (let txn: DatabaseTransaction = this; txn; txn = txn.next) {
|
|
2512
3414
|
txn.open = TRANSACTION_STATE.CLOSED;
|
|
2513
3415
|
}
|
|
2514
|
-
for (let txn: DatabaseTransaction = this; txn; txn =
|
|
3416
|
+
for (let txn: DatabaseTransaction = this, next: DatabaseTransaction; txn; txn = next) {
|
|
3417
|
+
next = txn.next;
|
|
3418
|
+
txn.next = null;
|
|
2515
3419
|
// Detach first so the abort() below performs only non-native cleanup, and so its doneReadTxn
|
|
2516
3420
|
// loop cannot spin on a nulled handle. Not to avoid a double abort: rocksdb-js tolerates
|
|
2517
3421
|
// abort-after-abort, and it is abort-after-COMMIT that throws.
|
|
2518
|
-
const detached =
|
|
3422
|
+
const detached =
|
|
3423
|
+
txn === this &&
|
|
3424
|
+
hasSiblingInFlight &&
|
|
3425
|
+
txn.transaction &&
|
|
3426
|
+
txn.transaction !== headTransaction &&
|
|
3427
|
+
(this.isActiveCommitHandle(txn.transaction) || hasDurableLog(txn.transaction))
|
|
3428
|
+
? null
|
|
3429
|
+
: txn.detachOwnedTransaction();
|
|
2519
3430
|
const committingTransaction = txn === this ? headTransaction : detached;
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
3431
|
+
releaseTransactionLockHolds(committingTransaction);
|
|
3432
|
+
abortNativeTransaction(
|
|
3433
|
+
committingTransaction,
|
|
3434
|
+
'aborting conflicted transaction in chain after exhausting retries',
|
|
3435
|
+
txn.writes[0]?.store ?? txn.db,
|
|
3436
|
+
txn.startedFrom
|
|
3437
|
+
);
|
|
2525
3438
|
// With outstanding iterators, the head owns a retained read handle while the retry commits
|
|
2526
3439
|
// through a separate replay handle. Both must be aborted after retry exhaustion.
|
|
2527
3440
|
if (txn === this && detached && detached !== committingTransaction) {
|
|
2528
|
-
|
|
2529
|
-
detached
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
3441
|
+
abortNativeTransaction(
|
|
3442
|
+
detached,
|
|
3443
|
+
'aborting retained read transaction after exhausting retries',
|
|
3444
|
+
txn.writes[0]?.store ?? txn.db,
|
|
3445
|
+
txn.startedFrom
|
|
3446
|
+
);
|
|
2533
3447
|
}
|
|
2534
3448
|
try {
|
|
2535
3449
|
// abort() synchronously walks savedBlobs and can call write.store.getEntry(), which can throw
|
|
2536
3450
|
// (closed store, decode error). Catch and continue so one link's wrapper-cleanup failure can't
|
|
2537
3451
|
// strand later links' native handles — they were already detached/aborted above regardless.
|
|
2538
|
-
txn.abort();
|
|
3452
|
+
txn.abort(false, false, committingTransaction);
|
|
2539
3453
|
} catch (abortError) {
|
|
2540
3454
|
harperLogger.debug?.('cleaning up conflicted transaction in chain after exhausting retries', abortError);
|
|
2541
3455
|
}
|
|
@@ -2621,15 +3535,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2621
3535
|
*/
|
|
2622
3536
|
poisonAfterStalledSubmittedCommit(): void {
|
|
2623
3537
|
const root = this.root ?? this;
|
|
2624
|
-
const
|
|
2625
|
-
const collect = (start: DatabaseTransaction | undefined) => {
|
|
2626
|
-
for (let txn = start; txn; txn = txn.next) links.add(txn);
|
|
2627
|
-
};
|
|
2628
|
-
collect(root);
|
|
2629
|
-
collect(this);
|
|
2630
|
-
collect(root.submittedLink);
|
|
2631
|
-
if (root.submittedLinks) for (const txn of root.submittedLinks) collect(txn);
|
|
2632
|
-
for (const txn of links) {
|
|
3538
|
+
for (const txn of collectCommitLinks(root, this)) {
|
|
2633
3539
|
txn.postSubmitPoisoned = true;
|
|
2634
3540
|
// Not on a link an earlier poison already aborted: abort() cleared its write set, so excusing
|
|
2635
3541
|
// its continuation from the poison check would let it resume onto nothing and report success.
|
|
@@ -2650,7 +3556,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
2650
3556
|
// throws. abort() rather than the native abort alone: it reclaims blobs a replayed write
|
|
2651
3557
|
// staged.
|
|
2652
3558
|
this.detachOwnedTransaction();
|
|
2653
|
-
abortNativeTransaction(
|
|
3559
|
+
abortNativeTransaction(
|
|
3560
|
+
transaction,
|
|
3561
|
+
'aborting a transaction whose synchronous commit failed',
|
|
3562
|
+
this.writes[0]?.store ?? this.db,
|
|
3563
|
+
this.startedFrom
|
|
3564
|
+
);
|
|
2654
3565
|
try {
|
|
2655
3566
|
this.abort();
|
|
2656
3567
|
} catch (abortError) {
|
|
@@ -2700,10 +3611,11 @@ export class ImmediateTransaction extends DatabaseTransaction {
|
|
|
2700
3611
|
}
|
|
2701
3612
|
save(...args: any[]): any {
|
|
2702
3613
|
const operation = args[0]; // the staged write, not a transaction — commit() re-enters here with it
|
|
2703
|
-
if (
|
|
3614
|
+
if (operation?.deferSave) operation.deferSave = false;
|
|
3615
|
+
if (this.isCommitting || args[1] || args[3]) {
|
|
2704
3616
|
// Stage into the transaction commit() is committing (on a retry round, the one being retried)
|
|
2705
3617
|
// and reload within it; with no handle super.save() would open its own and nest a commit().
|
|
2706
|
-
super.save(operation, args[1], true);
|
|
3618
|
+
super.save(operation, args[1], true, args[3]);
|
|
2707
3619
|
} else {
|
|
2708
3620
|
this.admitRequestWrite(); // a deferred save reaches here without passing through addWrite()
|
|
2709
3621
|
this.isCommitting = true;
|
|
@@ -2738,7 +3650,7 @@ export class ImmediateTransaction extends DatabaseTransaction {
|
|
|
2738
3650
|
const recordLocks = this.recordLocks;
|
|
2739
3651
|
if (!recordLocks) return;
|
|
2740
3652
|
for (const [store, storeMap] of recordLocks) {
|
|
2741
|
-
for (const [keyId, handle] of storeMap) if (handle.released) storeMap.delete(keyId);
|
|
3653
|
+
for (const [keyId, handle] of storeMap) if (handle.released || handle.isExpired()) storeMap.delete(keyId);
|
|
2742
3654
|
if (storeMap.size === 0) recordLocks.delete(store);
|
|
2743
3655
|
}
|
|
2744
3656
|
if (recordLocks.size === 0) this.recordLocks = undefined;
|
|
@@ -2907,6 +3819,13 @@ function reportIfLongLived(
|
|
|
2907
3819
|
/** Diagnostic grace after a native submission outlives the ordinary transaction timeout. */
|
|
2908
3820
|
const MAX_DEFERRED_POISON_TICKS = 2;
|
|
2909
3821
|
const STALLED_COMMIT_LOG_MIN_INTERVAL_MS = 1000;
|
|
3822
|
+
// A logged retry parked on a write intent moves its retry counter at least every native park timeout,
|
|
3823
|
+
// so a counter that has not moved for two of them is a native attempt that never settled — whatever
|
|
3824
|
+
// the monitor's tick interval (STORAGE_MAXTRANSACTIONOPENTIME can be set far below the park timeout).
|
|
3825
|
+
const configuredParkTimeoutMs = Number(process.env.ROCKSDB_JS_PARK_TIMEOUT_MS);
|
|
3826
|
+
// Same acceptance rule as rocksdb-js: anything but a finite positive number falls back to its 5000 ms default.
|
|
3827
|
+
const LOGGED_RETRY_STALL_MS =
|
|
3828
|
+
2 * (Number.isFinite(configuredParkTimeoutMs) && configuredParkTimeoutMs > 0 ? configuredParkTimeoutMs : 5000);
|
|
2910
3829
|
let lastStalledCommitWarningAt = -Infinity;
|
|
2911
3830
|
let lastStalledCommitErrorAt = -Infinity;
|
|
2912
3831
|
|
|
@@ -2920,6 +3839,28 @@ function submittedCommitStillInFlight(root: DatabaseTransaction): boolean {
|
|
|
2920
3839
|
return false;
|
|
2921
3840
|
}
|
|
2922
3841
|
|
|
3842
|
+
// Every link the monitor would poison for a stalled commit, so the two walks agree on which links count.
|
|
3843
|
+
function collectCommitLinks(root: DatabaseTransaction, txn: DatabaseTransaction): Set<DatabaseTransaction> {
|
|
3844
|
+
const links = new Set<DatabaseTransaction>();
|
|
3845
|
+
const collect = (start: DatabaseTransaction | undefined) => {
|
|
3846
|
+
for (let link = start; link; link = link.next) links.add(link);
|
|
3847
|
+
};
|
|
3848
|
+
collect(root);
|
|
3849
|
+
collect(txn);
|
|
3850
|
+
collect(root.submittedLink);
|
|
3851
|
+
if (root.submittedLinks) for (const link of root.submittedLinks) collect(link);
|
|
3852
|
+
return links;
|
|
3853
|
+
}
|
|
3854
|
+
|
|
3855
|
+
function chainDurableLogRetries(root: DatabaseTransaction, txn: DatabaseTransaction): number {
|
|
3856
|
+
let retries = -1;
|
|
3857
|
+
for (const link of collectCommitLinks(root, txn)) {
|
|
3858
|
+
const linkRetries = link.activeDurableLogRetries();
|
|
3859
|
+
if (linkRetries >= 0) retries = Math.max(retries, 0) + linkRetries;
|
|
3860
|
+
}
|
|
3861
|
+
return retries;
|
|
3862
|
+
}
|
|
3863
|
+
|
|
2923
3864
|
/**
|
|
2924
3865
|
* True when this tick must leave `txn` alone because some link has submitted writes whose native
|
|
2925
3866
|
* outcome is not known yet. Merely entering commit() is not enough: a wholly pre-submit transaction
|
|
@@ -2928,7 +3869,8 @@ function submittedCommitStillInFlight(root: DatabaseTransaction): boolean {
|
|
|
2928
3869
|
export function deferForCommitInFlight(
|
|
2929
3870
|
txn: DatabaseTransaction,
|
|
2930
3871
|
url: string | undefined,
|
|
2931
|
-
expiration = txnExpiration
|
|
3872
|
+
expiration = txnExpiration,
|
|
3873
|
+
tick = monitorTick
|
|
2932
3874
|
): boolean {
|
|
2933
3875
|
const root = txn.root ?? txn;
|
|
2934
3876
|
if (!txn.isChainCommitting() || !root.commitSubmitted) return false;
|
|
@@ -2940,23 +3882,34 @@ export function deferForCommitInFlight(
|
|
|
2940
3882
|
// diagnostic grace, nor report a legitimately slow one early.
|
|
2941
3883
|
const now = performance.now();
|
|
2942
3884
|
const deadline = (root.deferredPoisonDeadline ??= now + MAX_DEFERRED_POISON_TICKS * expiration);
|
|
3885
|
+
// A handle with a durable log entry retrying a conflict has a known outcome — it commits once the
|
|
3886
|
+
// conflicting intent clears — so while its retry counter keeps advancing it is neither stalled nor
|
|
3887
|
+
// poisoned, and noteOverdueLoggedCommit is the operator's signal. A counter that has not moved for
|
|
3888
|
+
// LOGGED_RETRY_STALL_MS is a native attempt that never settled, which is the stall handled below.
|
|
3889
|
+
const loggedRetries = chainDurableLogRetries(root, txn);
|
|
3890
|
+
let loggedRetry = false;
|
|
3891
|
+
if (loggedRetries >= 0) {
|
|
3892
|
+
if (root.loggedRetriesTick === tick) {
|
|
3893
|
+
loggedRetry = Boolean(root.loggedRetryAdvancing);
|
|
3894
|
+
} else {
|
|
3895
|
+
if (root.loggedRetriesSeen !== loggedRetries) {
|
|
3896
|
+
root.loggedRetriesSeen = loggedRetries;
|
|
3897
|
+
root.loggedRetriesChangedAt = now;
|
|
3898
|
+
}
|
|
3899
|
+
loggedRetry = now - root.loggedRetriesChangedAt < LOGGED_RETRY_STALL_MS;
|
|
3900
|
+
root.loggedRetriesTick = tick;
|
|
3901
|
+
root.loggedRetryAdvancing = loggedRetry;
|
|
3902
|
+
}
|
|
3903
|
+
}
|
|
2943
3904
|
let hasUnsubmittedCommit = false;
|
|
2944
3905
|
let unsubmittedCommitGraceExpired = false;
|
|
2945
3906
|
if (now >= deadline) {
|
|
2946
3907
|
// Only reclaim after the commit has genuinely stalled. Submitted writes no longer depend on these
|
|
2947
3908
|
// separate snapshots; pre-submit handles can still carry writes needed by the attempt.
|
|
2948
3909
|
txn.releaseStalledCommitReadResources();
|
|
2949
|
-
if (!txn.isProtectedCommit()) {
|
|
3910
|
+
if (!loggedRetry && !txn.isProtectedCommit()) {
|
|
2950
3911
|
if (!root.postSubmitPoisoned) txn.poisonAfterStalledSubmittedCommit();
|
|
2951
|
-
const
|
|
2952
|
-
const collect = (start: DatabaseTransaction | undefined) => {
|
|
2953
|
-
for (let link = start; link; link = link.next) links.add(link);
|
|
2954
|
-
};
|
|
2955
|
-
collect(root);
|
|
2956
|
-
collect(txn);
|
|
2957
|
-
collect(root.submittedLink);
|
|
2958
|
-
if (root.submittedLinks) for (const link of root.submittedLinks) collect(link);
|
|
2959
|
-
for (const link of links) {
|
|
3912
|
+
for (const link of collectCommitLinks(root, txn)) {
|
|
2960
3913
|
if (link.committing && !link.nativeCommitSubmitted) {
|
|
2961
3914
|
hasUnsubmittedCommit = true;
|
|
2962
3915
|
break;
|
|
@@ -2976,7 +3929,11 @@ export function deferForCommitInFlight(
|
|
|
2976
3929
|
!(txn === root && submittedCommitStillInFlight(root))
|
|
2977
3930
|
)
|
|
2978
3931
|
return hasUnsubmittedCommit && !unsubmittedCommitGraceExpired;
|
|
2979
|
-
if (
|
|
3932
|
+
if (
|
|
3933
|
+
!loggedRetry &&
|
|
3934
|
+
!root.stalledCommitLogged &&
|
|
3935
|
+
now - lastStalledCommitErrorAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS
|
|
3936
|
+
) {
|
|
2980
3937
|
lastStalledCommitErrorAt = now;
|
|
2981
3938
|
root.stalledCommitLogged = true;
|
|
2982
3939
|
harperLogger.error(
|
|
@@ -2988,7 +3945,7 @@ export function deferForCommitInFlight(
|
|
|
2988
3945
|
}`
|
|
2989
3946
|
);
|
|
2990
3947
|
}
|
|
2991
|
-
} else if (now - lastStalledCommitWarningAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS) {
|
|
3948
|
+
} else if (!loggedRetry && now - lastStalledCommitWarningAt >= STALLED_COMMIT_LOG_MIN_INTERVAL_MS) {
|
|
2992
3949
|
lastStalledCommitWarningAt = now;
|
|
2993
3950
|
harperLogger.warn?.(
|
|
2994
3951
|
`Transaction exceeded the open-transaction limit after submitting a native commit; preserving its unknown outcome, from table: ${
|
|
@@ -3170,11 +4127,23 @@ export function setMaxOutstandingTxnDuration(ms: number): number {
|
|
|
3170
4127
|
return previous;
|
|
3171
4128
|
}
|
|
3172
4129
|
|
|
4130
|
+
/** Test seam: overrides the per-retry backoff cap and returns the previous value. */
|
|
4131
|
+
export function setMaxRetryDelayMs(ms: number): number {
|
|
4132
|
+
const previous = MAX_RETRY_DELAY_MS;
|
|
4133
|
+
MAX_RETRY_DELAY_MS = ms;
|
|
4134
|
+
return previous;
|
|
4135
|
+
}
|
|
4136
|
+
|
|
3173
4137
|
/** Test seam: whether the monitor supervises this logical transaction for its writes. */
|
|
3174
4138
|
export function isWriteSupervised(txn: DatabaseTransaction): boolean {
|
|
3175
4139
|
return supervisedWriteRoots.has(txn);
|
|
3176
4140
|
}
|
|
3177
4141
|
|
|
4142
|
+
/** Test seam: returns the current open-transaction monitor interval. */
|
|
4143
|
+
export function getTxnExpiration(): number {
|
|
4144
|
+
return txnExpiration;
|
|
4145
|
+
}
|
|
4146
|
+
|
|
3178
4147
|
export function setTxnExpiration(ms) {
|
|
3179
4148
|
clearInterval(timer);
|
|
3180
4149
|
txnExpiration = ms;
|