@harperfast/harper 5.2.13 → 5.3.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/mcpTools.ts +1 -1
- package/agent/session.ts +25 -14
- package/bin/cliOperations.ts +58 -9
- package/bin/copyDb.ts +282 -59
- package/bin/deploySetup.ts +16 -5
- package/bin/harper.ts +1 -1
- package/bin/help.ts +4 -1
- package/bin/lite.ts +4 -1
- package/bin/restart.ts +120 -4
- package/bin/run.ts +6 -11
- package/bin/upgrade.js +7 -3
- package/bin/workloadIdentity.ts +119 -0
- package/components/Application.ts +3436 -242
- package/components/ApplicationScope.ts +8 -0
- package/components/EntryHandler.ts +59 -39
- package/components/OptionsWatcher.ts +440 -98
- package/components/RuntimeModuleTracker.ts +38 -7
- package/components/Scope.ts +56 -15
- package/components/awaitRestart.ts +84 -0
- package/components/componentLoader.ts +381 -32
- package/components/componentPreparationLock.ts +16 -5
- package/components/deploymentOperations.ts +4 -1
- package/components/deploymentRecorder.ts +9 -2
- package/components/mcp/adapters/harperHttp.ts +4 -0
- package/components/mcp/listChanged.ts +4 -0
- package/components/mcp/toolRegistry.ts +2 -0
- package/components/mcp/tools/operations.ts +9 -0
- package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
- package/components/operations.js +537 -113
- package/components/operationsValidation.js +98 -3
- package/components/packageComponent.ts +25 -1
- package/components/requestRestart.ts +11 -0
- package/components/status/ComponentStatusRegistry.ts +59 -0
- package/config/RootConfigWatcher.ts +240 -40
- package/config/configReadRetry.ts +62 -0
- package/config/configUtils.ts +357 -48
- package/config/harperConfigEnvVars.ts +170 -27
- package/config/parseConfigFile.ts +34 -0
- package/config/readConfigFileSync.ts +44 -0
- package/config/watcherArming.ts +59 -0
- package/config-root.schema.json +33 -0
- package/dataLayer/blobBackup.ts +160 -50
- package/dataLayer/delete.ts +6 -1
- package/dataLayer/harperBridge/ResourceBridge.ts +80 -10
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dataLayer/hdbInfoController.ts +34 -1
- package/dataLayer/insert.ts +44 -1
- package/dataLayer/rocksdbBackup.ts +53 -10
- package/dataLayer/schema.ts +11 -1
- package/dataLayer/schemaDescribe.ts +8 -1
- package/dist/agent/mcpTools.js +1 -1
- package/dist/agent/mcpTools.js.map +1 -1
- package/dist/agent/session.d.ts +22 -0
- package/dist/agent/session.js +26 -15
- package/dist/agent/session.js.map +1 -1
- package/dist/bin/cliOperations.js +61 -9
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/copyDb.d.ts +12 -1
- package/dist/bin/copyDb.js +248 -60
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/bin/deploySetup.d.ts +2 -0
- package/dist/bin/deploySetup.js +11 -3
- package/dist/bin/deploySetup.js.map +1 -1
- package/dist/bin/harper.js +1 -1
- package/dist/bin/harper.js.map +1 -1
- package/dist/bin/help.js +4 -1
- package/dist/bin/help.js.map +1 -1
- package/dist/bin/lite.js +4 -1
- package/dist/bin/lite.js.map +1 -1
- package/dist/bin/restart.js +91 -6
- package/dist/bin/restart.js.map +1 -1
- package/dist/bin/run.js +4 -10
- package/dist/bin/run.js.map +1 -1
- package/dist/bin/upgrade.js +4 -3
- package/dist/bin/upgrade.js.map +1 -1
- package/dist/bin/workloadIdentity.d.ts +18 -0
- package/dist/bin/workloadIdentity.js +100 -0
- package/dist/bin/workloadIdentity.js.map +1 -0
- package/dist/components/Application.d.ts +235 -17
- package/dist/components/Application.js +3038 -238
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +8 -0
- package/dist/components/ApplicationScope.js +7 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/EntryHandler.js +26 -10
- package/dist/components/EntryHandler.js.map +1 -1
- package/dist/components/OptionsWatcher.d.ts +4 -0
- package/dist/components/OptionsWatcher.js +440 -99
- package/dist/components/OptionsWatcher.js.map +1 -1
- package/dist/components/RuntimeModuleTracker.js +40 -6
- package/dist/components/RuntimeModuleTracker.js.map +1 -1
- package/dist/components/Scope.js +52 -13
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/awaitRestart.d.ts +33 -0
- package/dist/components/awaitRestart.js +61 -0
- package/dist/components/awaitRestart.js.map +1 -0
- package/dist/components/componentLoader.d.ts +38 -1
- package/dist/components/componentLoader.js +313 -24
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentPreparationLock.d.ts +5 -0
- package/dist/components/componentPreparationLock.js +14 -6
- package/dist/components/componentPreparationLock.js.map +1 -1
- package/dist/components/deploymentOperations.js +4 -1
- package/dist/components/deploymentOperations.js.map +1 -1
- package/dist/components/deploymentRecorder.d.ts +4 -2
- package/dist/components/deploymentRecorder.js +1 -0
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.js +4 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/listChanged.js +5 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +1 -0
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +5 -0
- package/dist/components/mcp/tools/operations.js +9 -0
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/operations.d.ts +28 -0
- package/dist/components/operations.js +476 -113
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.js +97 -3
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/packageComponent.js +24 -0
- package/dist/components/packageComponent.js.map +1 -1
- package/dist/components/requestRestart.d.ts +1 -0
- package/dist/components/requestRestart.js +7 -0
- package/dist/components/requestRestart.js.map +1 -1
- package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
- package/dist/components/status/ComponentStatusRegistry.js +63 -0
- package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
- package/dist/config/RootConfigWatcher.d.ts +9 -1
- package/dist/config/RootConfigWatcher.js +228 -37
- package/dist/config/RootConfigWatcher.js.map +1 -1
- package/dist/config/configReadRetry.d.ts +8 -0
- package/dist/config/configReadRetry.js +62 -0
- package/dist/config/configReadRetry.js.map +1 -0
- package/dist/config/configUtils.d.ts +11 -2
- package/dist/config/configUtils.js +304 -47
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/harperConfigEnvVars.d.ts +16 -0
- package/dist/config/harperConfigEnvVars.js +162 -25
- package/dist/config/harperConfigEnvVars.js.map +1 -1
- package/dist/config/parseConfigFile.d.ts +4 -0
- package/dist/config/parseConfigFile.js +35 -0
- package/dist/config/parseConfigFile.js.map +1 -0
- package/dist/config/readConfigFileSync.d.ts +1 -0
- package/dist/config/readConfigFileSync.js +47 -0
- package/dist/config/readConfigFileSync.js.map +1 -0
- package/dist/config/watcherArming.d.ts +15 -0
- package/dist/config/watcherArming.js +59 -0
- package/dist/config/watcherArming.js.map +1 -0
- package/dist/dataLayer/blobBackup.d.ts +49 -20
- package/dist/dataLayer/blobBackup.js +139 -50
- package/dist/dataLayer/blobBackup.js.map +1 -1
- package/dist/dataLayer/delete.js +1 -1
- package/dist/dataLayer/delete.js.map +1 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.js +72 -12
- package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
- package/dist/dataLayer/hdbInfoController.d.ts +10 -0
- package/dist/dataLayer/hdbInfoController.js +30 -1
- package/dist/dataLayer/hdbInfoController.js.map +1 -1
- package/dist/dataLayer/insert.d.ts +9 -1
- package/dist/dataLayer/insert.js +30 -0
- package/dist/dataLayer/insert.js.map +1 -1
- package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
- package/dist/dataLayer/rocksdbBackup.js +45 -8
- package/dist/dataLayer/rocksdbBackup.js.map +1 -1
- package/dist/dataLayer/schema.js +8 -0
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +8 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +55 -0
- package/dist/resources/DatabaseTransaction.d.ts +77 -0
- package/dist/resources/DatabaseTransaction.js +586 -54
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +2 -1
- package/dist/resources/LMDBTransaction.js +43 -6
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
- package/dist/resources/PrimaryRocksDatabase.js +49 -5
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
- package/dist/resources/RecordEncoder.d.ts +20 -0
- package/dist/resources/RecordEncoder.js +110 -9
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +2 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.js +117 -24
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +28 -1
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/RocksIndexStore.d.ts +6 -1
- package/dist/resources/RocksIndexStore.js +24 -9
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.d.ts +24 -1
- package/dist/resources/RocksTransactionLogStore.js +160 -49
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.d.ts +165 -10
- package/dist/resources/Table.js +1805 -303
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/write.js +10 -3
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +181 -2
- package/dist/resources/auditStore.js +640 -29
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/blob.d.ts +129 -9
- package/dist/resources/blob.js +992 -114
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/branchDatabase.d.ts +48 -0
- package/dist/resources/branchDatabase.js +892 -0
- package/dist/resources/branchDatabase.js.map +1 -0
- package/dist/resources/crdt.js +50 -12
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/dataLoader.js +3 -4
- package/dist/resources/dataLoader.js.map +1 -1
- package/dist/resources/databases.d.ts +157 -14
- package/dist/resources/databases.js +1462 -325
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/defineTable.d.ts +10 -2
- package/dist/resources/defineTable.js +9 -1
- package/dist/resources/defineTable.js.map +1 -1
- package/dist/resources/derivedIndexRegistry.d.ts +5 -0
- package/dist/resources/derivedIndexRegistry.js +68 -0
- package/dist/resources/derivedIndexRegistry.js.map +1 -0
- package/dist/resources/derivedIndexRuntime.d.ts +215 -0
- package/dist/resources/derivedIndexRuntime.js +2027 -0
- package/dist/resources/derivedIndexRuntime.js.map +1 -0
- package/dist/resources/graphql.d.ts +1 -1
- package/dist/resources/graphql.js +52 -16
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +103 -9
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +854 -41
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
- package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
- package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
- package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
- package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
- package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
- package/dist/resources/longLivedTransactions.d.ts +71 -0
- package/dist/resources/longLivedTransactions.js +358 -0
- package/dist/resources/longLivedTransactions.js.map +1 -0
- package/dist/resources/models/backendRegistry.d.ts +26 -0
- package/dist/resources/models/backendRegistry.js +60 -2
- package/dist/resources/models/backendRegistry.js.map +1 -1
- package/dist/resources/models/bootstrap.d.ts +33 -1
- package/dist/resources/models/bootstrap.js +416 -31
- package/dist/resources/models/bootstrap.js.map +1 -1
- package/dist/resources/nodeIdMapping.d.ts +5 -0
- package/dist/resources/nodeIdMapping.js +49 -0
- package/dist/resources/nodeIdMapping.js.map +1 -1
- package/dist/resources/recordLock.d.ts +123 -0
- package/dist/resources/recordLock.js +315 -0
- package/dist/resources/recordLock.js.map +1 -0
- package/dist/resources/recordLockCoordinator.d.ts +557 -0
- package/dist/resources/recordLockCoordinator.js +2565 -0
- package/dist/resources/recordLockCoordinator.js.map +1 -0
- package/dist/resources/replayLogs.d.ts +14 -1
- package/dist/resources/replayLogs.js +157 -24
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/replayLogsGuards.d.ts +94 -7
- package/dist/resources/replayLogsGuards.js +111 -7
- package/dist/resources/replayLogsGuards.js.map +1 -1
- package/dist/resources/replicatedApplyFailure.d.ts +16 -0
- package/dist/resources/replicatedApplyFailure.js +63 -0
- package/dist/resources/replicatedApplyFailure.js.map +1 -0
- package/dist/resources/scheduler/scheduler.js +3 -3
- package/dist/resources/scheduler/scheduler.js.map +1 -1
- package/dist/resources/search.d.ts +10 -4
- package/dist/resources/search.js +283 -37
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/tracked.d.ts +5 -1
- package/dist/resources/tracked.js +74 -23
- package/dist/resources/tracked.js.map +1 -1
- package/dist/resources/transactionBroadcast.d.ts +1 -1
- package/dist/resources/transactionBroadcast.js +2 -2
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/auth.js +61 -30
- package/dist/security/auth.js.map +1 -1
- package/dist/security/authn/oidc/claims.d.ts +22 -0
- package/dist/security/authn/oidc/claims.js +71 -0
- package/dist/security/authn/oidc/claims.js.map +1 -0
- package/dist/security/authn/oidc/identityToken.d.ts +27 -0
- package/dist/security/authn/oidc/identityToken.js +111 -0
- package/dist/security/authn/oidc/identityToken.js.map +1 -0
- package/dist/security/authn/oidc/jwks.d.ts +25 -0
- package/dist/security/authn/oidc/jwks.js +261 -0
- package/dist/security/authn/oidc/jwks.js.map +1 -0
- package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
- package/dist/security/authn/oidc/providers/generic.js +34 -0
- package/dist/security/authn/oidc/providers/generic.js.map +1 -0
- package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
- package/dist/security/authn/oidc/providers/githubActions.js +129 -0
- package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
- package/dist/security/authn/oidc/providers/index.d.ts +37 -0
- package/dist/security/authn/oidc/providers/index.js +24 -0
- package/dist/security/authn/oidc/providers/index.js.map +1 -0
- package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
- package/dist/security/authn/oidc/tokenExchange.js +306 -0
- package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
- package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
- package/dist/security/authn/oidc/types.d.ts +38 -0
- package/dist/security/authn/oidc/types.js +6 -0
- package/dist/security/authn/oidc/types.js.map +1 -0
- package/dist/security/certificateVerification/index.js +40 -11
- package/dist/security/certificateVerification/index.js.map +1 -1
- package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
- package/dist/security/certificateVerification/trustedIssuers.js +79 -0
- package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
- package/dist/security/certificateVerification/types.d.ts +1 -0
- package/dist/security/credentialProvenance.d.ts +35 -0
- package/dist/security/credentialProvenance.js +51 -0
- package/dist/security/credentialProvenance.js.map +1 -0
- package/dist/security/credentialRejection.d.ts +4 -0
- package/dist/security/credentialRejection.js +24 -0
- package/dist/security/credentialRejection.js.map +1 -0
- package/dist/security/deferredAuthentication.d.ts +36 -0
- package/dist/security/deferredAuthentication.js +70 -0
- package/dist/security/deferredAuthentication.js.map +1 -0
- package/dist/security/impersonation.d.ts +21 -0
- package/dist/security/impersonation.js +108 -9
- package/dist/security/impersonation.js.map +1 -1
- package/dist/security/jsLoader.d.ts +6 -0
- package/dist/security/jsLoader.js +77 -15
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +301 -71
- package/dist/security/keys.js.map +1 -1
- package/dist/security/operationScope.d.ts +21 -0
- package/dist/security/operationScope.js +36 -0
- package/dist/security/operationScope.js.map +1 -0
- package/dist/security/permissionsTranslator.js +21 -0
- package/dist/security/permissionsTranslator.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +19 -1
- package/dist/security/tokenAuthentication.js +191 -10
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/security/user.js +4 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
- package/dist/server/DurableSubscriptionsSession.js +67 -10
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +106 -2
- package/dist/server/REST.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -0
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +16 -1
- package/dist/server/http.js +122 -17
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +8 -1
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/jobs/jobProcess.js +6 -2
- package/dist/server/jobs/jobProcess.js.map +1 -1
- package/dist/server/jobs/jobs.js +4 -1
- package/dist/server/jobs/jobs.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +26 -4
- package/dist/server/liveSubscriptionAuth.js +105 -39
- package/dist/server/liveSubscriptionAuth.js.map +1 -1
- package/dist/server/loadRootComponents.js +49 -10
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/mqtt.d.ts +2 -0
- package/dist/server/mqtt.js +165 -30
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/nodeName.d.ts +2 -0
- package/dist/server/nodeName.js +107 -23
- package/dist/server/nodeName.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +27 -0
- package/dist/server/serverHelpers/Headers.js +145 -1
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
- package/dist/server/serverHelpers/Request.d.ts +5 -10
- package/dist/server/serverHelpers/Request.js +38 -136
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.d.ts +11 -0
- package/dist/server/serverHelpers/contentTypes.js +221 -40
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
- package/dist/server/serverHelpers/deployValidationState.js +9 -19
- package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
- package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
- package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
- package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
- package/dist/server/serverHelpers/registeredOperations.js +74 -21
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
- package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
- package/dist/server/serverHelpers/requestSanitization.js +20 -0
- package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +6 -3
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
- package/dist/server/serverHelpers/serverUtilities.js +178 -29
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
- package/dist/server/serverHelpers/uwsServer.js +19 -1
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/static.js +24 -28
- package/dist/server/static.js.map +1 -1
- package/dist/server/storageReclamation.d.ts +5 -0
- package/dist/server/storageReclamation.js +17 -1
- package/dist/server/storageReclamation.js.map +1 -1
- package/dist/server/threads/isolatedApplications.d.ts +47 -0
- package/dist/server/threads/isolatedApplications.js +171 -0
- package/dist/server/threads/isolatedApplications.js.map +1 -0
- package/dist/server/threads/itc.d.ts +7 -2
- package/dist/server/threads/itc.js +5 -1
- package/dist/server/threads/itc.js.map +1 -1
- package/dist/server/threads/logRotationTransport.d.ts +1 -0
- package/dist/server/threads/logRotationTransport.js +33 -0
- package/dist/server/threads/logRotationTransport.js.map +1 -0
- package/dist/server/threads/manageThreads.d.ts +83 -3
- package/dist/server/threads/manageThreads.js +769 -75
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.d.ts +1 -0
- package/dist/server/threads/socketRouter.js +277 -31
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadHeapMemory.d.ts +2 -0
- package/dist/server/threads/threadHeapMemory.js +31 -0
- package/dist/server/threads/threadHeapMemory.js.map +1 -0
- package/dist/server/threads/threadServer.js +76 -22
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/sqlEngine/config.d.ts +1 -3
- package/dist/sqlEngine/config.js +19 -16
- package/dist/sqlEngine/config.js.map +1 -1
- package/dist/sqlTranslator/index.d.ts +1 -1
- package/dist/sqlTranslator/index.js +30 -7
- package/dist/sqlTranslator/index.js.map +1 -1
- package/dist/upgrade/directives/5-3-0.d.ts +7 -0
- package/dist/upgrade/directives/5-3-0.js +148 -0
- package/dist/upgrade/directives/5-3-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/OperationFunctionCaller.js +2 -1
- package/dist/utility/OperationFunctionCaller.js.map +1 -1
- package/dist/utility/common_utils.d.ts +16 -0
- package/dist/utility/common_utils.js +32 -6
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/componentNames.d.ts +8 -0
- package/dist/utility/componentNames.js +12 -1
- package/dist/utility/componentNames.js.map +1 -1
- package/dist/utility/environment/environmentManager.js +3 -6
- package/dist/utility/environment/environmentManager.js.map +1 -1
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/environment/systemInformation.js +1 -0
- package/dist/utility/environment/systemInformation.js.map +1 -1
- package/dist/utility/errors/commonErrors.d.ts +2 -0
- package/dist/utility/errors/commonErrors.js +2 -0
- package/dist/utility/errors/commonErrors.js.map +1 -1
- package/dist/utility/errors/hdbError.d.ts +33 -0
- package/dist/utility/errors/hdbError.js +58 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/globalSchema.d.ts +18 -0
- package/dist/utility/hdbTerms.d.ts +18 -0
- package/dist/utility/hdbTerms.js +20 -2
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +2 -0
- package/dist/utility/logging/harper_logger.js +286 -30
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
- package/dist/utility/logging/logGenerationCoordinator.js +184 -0
- package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
- package/dist/utility/logging/logRotation.d.ts +46 -0
- package/dist/utility/logging/logRotation.js +365 -0
- package/dist/utility/logging/logRotation.js.map +1 -0
- package/dist/utility/logging/logRotator.d.ts +1 -1
- package/dist/utility/logging/logRotator.js +192 -85
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/nodeIdentity.d.ts +9 -0
- package/dist/utility/nodeIdentity.js +58 -0
- package/dist/utility/nodeIdentity.js.map +1 -0
- package/dist/utility/npmUtilities.js +11 -7
- package/dist/utility/npmUtilities.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +3 -1
- package/dist/utility/operationPermissions.js +16 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +10 -7
- package/dist/utility/operation_authorization.js +212 -41
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/watchPath.d.ts +29 -0
- package/dist/utility/watchPath.js +68 -0
- package/dist/utility/watchPath.js.map +1 -0
- package/dist/utility/watcherFallback.d.ts +41 -0
- package/dist/utility/watcherFallback.js +153 -0
- package/dist/utility/watcherFallback.js.map +1 -1
- package/dist/validation/configValidator.d.ts +12 -0
- package/dist/validation/configValidator.js +205 -75
- package/dist/validation/configValidator.js.map +1 -1
- package/dist/validation/installValidator.js +12 -0
- package/dist/validation/installValidator.js.map +1 -1
- package/dist/validation/validationWrapper.d.ts +11 -0
- package/dist/validation/validationWrapper.js +16 -3
- package/dist/validation/validationWrapper.js.map +1 -1
- package/index.ts +8 -0
- package/json/systemSchema.json +55 -0
- package/npm-shrinkwrap.json +286 -194
- package/package.json +15 -7
- package/resources/DESIGN.md +219 -52
- package/resources/DatabaseTransaction.ts +657 -52
- package/resources/LMDBTransaction.ts +46 -6
- package/resources/PrimaryRocksDatabase.ts +46 -7
- package/resources/RecordEncoder.ts +124 -9
- package/resources/RequestTarget.ts +2 -0
- package/resources/Resource.ts +114 -22
- package/resources/ResourceInterface.ts +31 -0
- package/resources/RocksIndexStore.ts +30 -9
- package/resources/RocksTransactionLogStore.ts +188 -51
- package/resources/Table.ts +1960 -301
- package/resources/analytics/write.ts +10 -3
- package/resources/auditStore.ts +647 -32
- package/resources/blob.ts +1029 -110
- package/resources/branchDatabase.ts +941 -0
- package/resources/crdt.ts +76 -12
- package/resources/dataLoader.ts +3 -4
- package/resources/databases.ts +1584 -320
- package/resources/defineTable.ts +18 -2
- package/resources/derivedIndexRegistry.ts +56 -0
- package/resources/derivedIndexRuntime.ts +2292 -0
- package/resources/graphql.ts +72 -17
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +911 -48
- package/resources/indexes/hnswDerivedIndex.ts +531 -0
- package/resources/indexes/hnswPlaneBinding.ts +174 -0
- package/resources/longLivedTransactions.ts +360 -0
- package/resources/models/backendRegistry.ts +84 -2
- package/resources/models/bootstrap.ts +473 -28
- package/resources/nodeIdMapping.ts +50 -0
- package/resources/recordLock.ts +419 -0
- package/resources/recordLockCoordinator.ts +3043 -0
- package/resources/replayLogs.ts +152 -26
- package/resources/replayLogsGuards.ts +171 -8
- package/resources/replicatedApplyFailure.ts +77 -0
- package/resources/scheduler/scheduler.ts +4 -4
- package/resources/search.ts +288 -46
- package/resources/tracked.ts +73 -22
- package/resources/transactionBroadcast.ts +3 -3
- package/security/auth.ts +68 -29
- package/security/authn/oidc/claims.ts +72 -0
- package/security/authn/oidc/identityToken.ts +129 -0
- package/security/authn/oidc/jwks.ts +260 -0
- package/security/authn/oidc/providers/generic.ts +40 -0
- package/security/authn/oidc/providers/githubActions.ts +137 -0
- package/security/authn/oidc/providers/index.ts +52 -0
- package/security/authn/oidc/tokenExchange.ts +300 -0
- package/security/authn/oidc/trustPolicyOperations.ts +343 -0
- package/security/authn/oidc/types.ts +41 -0
- package/security/certificateVerification/index.ts +54 -13
- package/security/certificateVerification/trustedIssuers.ts +76 -0
- package/security/certificateVerification/types.ts +1 -0
- package/security/credentialProvenance.ts +47 -0
- package/security/credentialRejection.ts +22 -0
- package/security/deferredAuthentication.ts +71 -0
- package/security/impersonation.ts +117 -12
- package/security/jsLoader.ts +83 -18
- package/security/keys.ts +298 -72
- package/security/operationScope.ts +33 -0
- package/security/permissionsTranslator.js +23 -0
- package/security/tokenAuthentication.ts +233 -12
- package/security/user.ts +4 -3
- package/server/DESIGN.md +194 -16
- package/server/DurableSubscriptionsSession.ts +71 -11
- package/server/REST.ts +115 -4
- package/server/graphqlQuerying.ts +4 -0
- package/server/http.ts +133 -20
- package/server/itc/serverHandlers.js +8 -1
- package/server/jobs/jobProcess.ts +8 -2
- package/server/jobs/jobs.ts +4 -1
- package/server/liveSubscriptionAuth.ts +129 -46
- package/server/loadRootComponents.js +50 -8
- package/server/mqtt.ts +179 -38
- package/server/nodeName.ts +103 -21
- package/server/serverHelpers/Headers.ts +135 -0
- package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
- package/server/serverHelpers/Request.ts +33 -131
- package/server/serverHelpers/contentTypes.ts +217 -36
- package/server/serverHelpers/deployValidationState.ts +24 -13
- package/server/serverHelpers/operationAuthorizationState.ts +34 -3
- package/server/serverHelpers/registeredOperations.ts +79 -22
- package/server/serverHelpers/requestSanitization.ts +15 -0
- package/server/serverHelpers/serverHandlers.js +6 -3
- package/server/serverHelpers/serverUtilities.ts +232 -40
- package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
- package/server/serverHelpers/uwsServer.ts +17 -2
- package/server/static.ts +23 -29
- package/server/storageReclamation.ts +15 -2
- package/server/threads/isolatedApplications.ts +157 -0
- package/server/threads/itc.js +11 -1
- package/server/threads/logRotationTransport.ts +40 -0
- package/server/threads/manageThreads.js +771 -66
- package/server/threads/socketRouter.ts +291 -30
- package/server/threads/threadHeapMemory.ts +26 -0
- package/server/threads/threadServer.js +73 -22
- package/sqlTranslator/index.ts +31 -8
- package/studio/web/assets/{Chat-4RrB5134.js → Chat-D3j-1yY1.js} +1 -1
- package/studio/web/assets/{FloatingChat-omlNMDcJ.js → FloatingChat-BxJGYcfB.js} +3 -3
- package/studio/web/assets/{apiToken-Bke3wvfZ.js → apiToken-CT55oWOe.js} +1 -1
- package/studio/web/assets/{applications-DvFDYJgK.js → applications-D9Ct9_vm.js} +1 -1
- package/studio/web/assets/{cssMode-C1vRa7zh.js → cssMode-DV8H7VwA.js} +1 -1
- package/studio/web/assets/{editor-3XRWEDWX.js → editor-uatc0unt.js} +1 -1
- package/studio/web/assets/{html-CTY5tdMr.js → html-Bm6D6paN.js} +1 -1
- package/studio/web/assets/{htmlMode-CG1vSD9t.js → htmlMode-CEn7tpLG.js} +1 -1
- package/studio/web/assets/{index-Dfpeofdu.js → index-BIXW6Pu4.js} +5 -5
- package/studio/web/assets/{index.lazy-B1VOIv-t.js → index.lazy-UI7L-Vrk.js} +1 -1
- package/studio/web/assets/{javascript-CJeJzGnI.js → javascript-CJ0G3AFZ.js} +1 -1
- package/studio/web/assets/{jsonMode-CYPBwM82.js → jsonMode-DQADAYEa.js} +1 -1
- package/studio/web/assets/{languageServices-IKH4GUHl.js → languageServices-CAQJXWcI.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-BC8_gxKG.js → lspLanguageFeatures-CCQ8P5sY.js} +1 -1
- package/studio/web/assets/{notifications-BfKBpYcq.js → notifications-BbxTU6Aw.js} +1 -1
- package/studio/web/assets/{notifications-DFbArTfC.js → notifications-Cvb3P1lB.js} +1 -1
- package/studio/web/assets/{profile-Q4-T6c-S.js → profile-Yyb7gsvL.js} +1 -1
- package/studio/web/assets/{regions-CtkV0xje.js → regions-OgjGHlU5.js} +1 -1
- package/studio/web/assets/{register-1ZZuMsiA.js → register-6qwNEOY3.js} +2 -2
- package/studio/web/assets/{setComponentFile-ZWLcWv5X.js → setComponentFile-BilDMtgB.js} +1 -1
- package/studio/web/assets/{setup-DWprJyJy.js → setup-J6qJ7OIU.js} +2 -2
- package/studio/web/assets/{status--aNm8isn.js → status-0RWGcfyD.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-Cq_lt3XD.js → toggleHighContrast-BIn-vErT.js} +1 -1
- package/studio/web/assets/{tsMode-pjgytARx.js → tsMode-DgUXku4d.js} +1 -1
- package/studio/web/assets/{typescript-Cdg0mqUh.js → typescript-C9orXcsM.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-RZhaY8Rn.js → useEntityRestURL-BEoXXbUB.js} +1 -1
- package/studio/web/assets/{workers-BgoXIqQe.js → workers-JVzSDmgx.js} +1 -1
- package/studio/web/assets/{xml-BwSeDMiP.js → xml-Cq-S8S4X.js} +1 -1
- package/studio/web/assets/{yaml-DotCUG5l.js → yaml-sfoRdh1M.js} +1 -1
- package/studio/web/index.html +1 -1
- package/upgrade/directives/5-3-0.ts +132 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/OperationFunctionCaller.ts +2 -1
- package/utility/common_utils.ts +30 -5
- package/utility/componentNames.ts +12 -0
- package/utility/environment/environmentManager.ts +3 -7
- package/utility/environment/systemInformation.ts +7 -0
- package/utility/errors/commonErrors.ts +4 -0
- package/utility/errors/hdbError.ts +57 -0
- package/utility/hdbTerms.ts +19 -0
- package/utility/logging/harper_logger.ts +278 -26
- package/utility/logging/logGenerationCoordinator.ts +196 -0
- package/utility/logging/logRotation.ts +367 -0
- package/utility/logging/logRotator.ts +213 -81
- package/utility/nodeIdentity.ts +45 -0
- package/utility/npmUtilities.ts +12 -8
- package/utility/operationPermissions.ts +18 -1
- package/utility/operation_authorization.ts +231 -42
- package/utility/watchPath.ts +63 -0
- package/utility/watcherFallback.ts +148 -0
- package/validation/configValidator.ts +215 -75
- package/validation/installValidator.ts +15 -0
- package/validation/validationWrapper.ts +18 -4
|
@@ -33,24 +33,31 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.RELEASED_TRANSACTION = exports.ImmediateTransaction = exports.DatabaseTransaction = exports.TRANSACTION_STATE = void 0;
|
|
36
|
+
exports.RELEASED_TRANSACTION = exports.ImmediateTransaction = exports.DatabaseTransaction = exports.COMMIT_PHASE_GRACE = exports.TRANSACTION_STATE = void 0;
|
|
37
37
|
exports.trackOutstandingCommit = trackOutstandingCommit;
|
|
38
38
|
exports.getOutstandingCommits = getOutstandingCommits;
|
|
39
39
|
exports.setCommitLatencyRecorder = setCommitLatencyRecorder;
|
|
40
40
|
exports.getTransactionQueueDepths = getTransactionQueueDepths;
|
|
41
41
|
exports.replicationConfirmation = replicationConfirmation;
|
|
42
42
|
exports.transactionOpenTooLongError = transactionOpenTooLongError;
|
|
43
|
+
exports.getReadTransactionGuard = getReadTransactionGuard;
|
|
43
44
|
exports.trackReadRange = trackReadRange;
|
|
45
|
+
exports.closeWriteInstance = closeWriteInstance;
|
|
46
|
+
exports.validateWrite = validateWrite;
|
|
47
|
+
exports.getAppliedWriteVersion = getAppliedWriteVersion;
|
|
44
48
|
exports.priorStagedWrite = priorStagedWrite;
|
|
45
49
|
exports.writeKeyId = writeKeyId;
|
|
50
|
+
exports.shouldSpareCommitPhase = shouldSpareCommitPhase;
|
|
46
51
|
exports.isReleasedTransaction = isReleasedTransaction;
|
|
47
52
|
exports.isJoinableScope = isJoinableScope;
|
|
48
53
|
exports.resetReplayedWritesWarning = resetReplayedWritesWarning;
|
|
54
|
+
exports.setMaxOutstandingTxnDuration = setMaxOutstandingTxnDuration;
|
|
49
55
|
exports.isWriteSupervised = isWriteSupervised;
|
|
50
56
|
exports.setTxnExpiration = setTxnExpiration;
|
|
51
57
|
const blob_ts_1 = require("./blob.js");
|
|
52
58
|
const commonUtility_ts_1 = require("../utility/lmdb/commonUtility.js");
|
|
53
59
|
const hdbError_ts_1 = require("../utility/errors/hdbError.js");
|
|
60
|
+
const recordLock_ts_1 = require("./recordLock.js");
|
|
54
61
|
const harperLogger = __importStar(require("../utility/logging/harper_logger.js"));
|
|
55
62
|
const envMngr = __importStar(require("../utility/environment/environmentManager.js"));
|
|
56
63
|
const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
|
|
@@ -60,6 +67,7 @@ const promises_1 = require("node:timers/promises");
|
|
|
60
67
|
const rocksdb_js_1 = require("@harperfast/rocksdb-js");
|
|
61
68
|
const RETRY_NOW_VALUE = rocksdb_js_1.constants.RETRY_NOW_VALUE;
|
|
62
69
|
const ordered_binary_1 = require("ordered-binary");
|
|
70
|
+
const longLivedTransactions_ts_1 = require("./longLivedTransactions.js");
|
|
63
71
|
const trackedTxns = new Set();
|
|
64
72
|
const readTransactionOwners = new WeakMap();
|
|
65
73
|
// Read options for a rotated generation's native transactions; shared because they never vary.
|
|
@@ -69,7 +77,7 @@ const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
|
|
|
69
77
|
// is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the
|
|
70
78
|
// chain root — so a chain child can never become its own timeout root (issue #2231).
|
|
71
79
|
const supervisedWriteRoots = new Set();
|
|
72
|
-
|
|
80
|
+
let MAX_OUTSTANDING_TXN_DURATION = (0, common_utils_ts_1.convertToMS)(envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them
|
|
73
81
|
const DEBUG_LONG_TXNS = envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS);
|
|
74
82
|
exports.TRANSACTION_STATE = {
|
|
75
83
|
CLOSED: 0, // the transaction has been committed or aborted and can no longer be used for writes (if read txn is active, it can be used for reads)
|
|
@@ -81,6 +89,12 @@ exports.TRANSACTION_STATE = {
|
|
|
81
89
|
LINGERING: 2,
|
|
82
90
|
};
|
|
83
91
|
const MAX_RETRIES = 40;
|
|
92
|
+
// Over-limit monitor ticks a transaction parked in its commit phase is spared before it is aborted
|
|
93
|
+
// like any other over-time transaction. Sparing re-arms `timeout`, so each of these costs two
|
|
94
|
+
// monitor intervals — roughly 10 minutes at the 30s default. Generous, so a legitimately large blob
|
|
95
|
+
// write finishes, but bounded, so a pre-commit source that stalls instead of finishing can't pin its
|
|
96
|
+
// read snapshot forever (issue #2062).
|
|
97
|
+
exports.COMMIT_PHASE_GRACE = 10;
|
|
84
98
|
// Cap the per-retry backoff so replication-applied transactions, which retry conflicts without a
|
|
85
99
|
// cap (see the commit rejection handler), don't grow the delay unbounded.
|
|
86
100
|
const MAX_RETRY_DELAY_MS = 1000;
|
|
@@ -91,7 +105,27 @@ let outstandingCommitCount = 0;
|
|
|
91
105
|
// the whole thread, regardless of how many distinct commits individually cross the threshold — see
|
|
92
106
|
// the comment at the log site for why a per-commit-only dedup isn't enough under sustained overload.
|
|
93
107
|
const OVERLOAD_LOG_MIN_INTERVAL_MS = 1000;
|
|
94
|
-
|
|
108
|
+
// One cooldown per reporting site, not one shared: `shed` fires on every bystander write during a
|
|
109
|
+
// wedge and would starve `abandon`, which names a different transaction and is the only server-side
|
|
110
|
+
// record of why a request was failed. Mutates only when it grants, so a caller with its own
|
|
111
|
+
// suppression as well (checkOverloaded's per-node `logged`) must test that first.
|
|
112
|
+
const lastStuckCommitLogAt = { shed: -Infinity, abandon: -Infinity };
|
|
113
|
+
function allowStuckCommitLog(site, now) {
|
|
114
|
+
if (now - lastStuckCommitLogAt[site] <= OVERLOAD_LOG_MIN_INTERVAL_MS)
|
|
115
|
+
return false;
|
|
116
|
+
lastStuckCommitLogAt[site] = now;
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
119
|
+
// Which database/table, which native transaction, and which request a stuck commit belongs to —
|
|
120
|
+
// without it a wedge gives no indication of what to investigate (harper#2001).
|
|
121
|
+
function describeCommitIdentity(store, startedFrom, nativeTransaction) {
|
|
122
|
+
const nativeTransactionId = nativeTransaction?.id;
|
|
123
|
+
return (`from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
|
|
124
|
+
(nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
|
|
125
|
+
(startedFrom?.resourceName
|
|
126
|
+
? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
|
|
127
|
+
: ''));
|
|
128
|
+
}
|
|
95
129
|
// Track a submitted commit until it settles. Every attempt is tracked unconditionally: a
|
|
96
130
|
// coordinated retry round and a chained second-store commit are both issued from inside the
|
|
97
131
|
// preceding commit's own resolve handler, which runs before any reaction that could release a
|
|
@@ -169,6 +203,21 @@ function getOutstandingCommits() {
|
|
|
169
203
|
// Once per process: committing under open read iterators forces a write replay, so the warning is
|
|
170
204
|
// about the caller's pattern, not the individual commit.
|
|
171
205
|
let replayedWritesWarned = false;
|
|
206
|
+
/**
|
|
207
|
+
* Abort a detached native handle. RocksTransaction.abort() throws on one that was already
|
|
208
|
+
* committed or aborted, and every caller is a cleanup path whose own callers have no handler — a
|
|
209
|
+
* throw there would abandon the rest of the cleanup.
|
|
210
|
+
*/
|
|
211
|
+
function abortNativeTransaction(transaction, context) {
|
|
212
|
+
if (transaction == null)
|
|
213
|
+
return;
|
|
214
|
+
try {
|
|
215
|
+
transaction.abort();
|
|
216
|
+
}
|
|
217
|
+
catch (error) {
|
|
218
|
+
harperLogger.debug?.(context, error);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
172
221
|
// The analytics module registers a recorder here at load (dependency inversion, mirroring
|
|
173
222
|
// `replicationConfirmation` below) so the storage layer doesn't statically import the analytics/server
|
|
174
223
|
// modules. Unset until analytics loads, and when analytics is disabled the recorder call is cheap.
|
|
@@ -259,17 +308,23 @@ class ReadSnapshotExpiredError extends hdbError_ts_1.ServerError {
|
|
|
259
308
|
this.name = 'ReadSnapshotExpiredError';
|
|
260
309
|
}
|
|
261
310
|
}
|
|
262
|
-
function
|
|
311
|
+
function getReadTransactionGuard(transaction) {
|
|
263
312
|
const owner = readTransactionOwners.get(transaction);
|
|
264
313
|
if (!owner)
|
|
265
|
-
return
|
|
266
|
-
function checkActive() {
|
|
314
|
+
return;
|
|
315
|
+
return function checkActive() {
|
|
267
316
|
if (owner.timedOut)
|
|
268
317
|
throw transactionOpenTooLongError();
|
|
269
318
|
if (owner.transaction !== transaction) {
|
|
270
319
|
throw new ReadSnapshotExpiredError();
|
|
271
320
|
}
|
|
272
|
-
}
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
function trackReadRange(transaction, createRange) {
|
|
324
|
+
const owner = readTransactionOwners.get(transaction);
|
|
325
|
+
if (!owner)
|
|
326
|
+
return createRange();
|
|
327
|
+
const checkActive = getReadTransactionGuard(transaction);
|
|
273
328
|
checkActive();
|
|
274
329
|
const range = createRange();
|
|
275
330
|
const iterate = range.iterate;
|
|
@@ -326,6 +381,28 @@ function trackReadRange(transaction, createRange) {
|
|
|
326
381
|
};
|
|
327
382
|
return range;
|
|
328
383
|
}
|
|
384
|
+
function closeWriteInstance(operation) {
|
|
385
|
+
if (operation && !operation.instanceClosed) {
|
|
386
|
+
operation.instanceClosed = true;
|
|
387
|
+
if (operation.writeGeneration)
|
|
388
|
+
operation.writeGeneration.closed = true;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
function validateWrite(operation, txnTime, transaction) {
|
|
392
|
+
const generation = operation.writeGeneration;
|
|
393
|
+
if (generation)
|
|
394
|
+
generation.internalWrites++;
|
|
395
|
+
try {
|
|
396
|
+
return operation.validate?.(txnTime, transaction);
|
|
397
|
+
}
|
|
398
|
+
finally {
|
|
399
|
+
if (generation)
|
|
400
|
+
generation.internalWrites--;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
function getAppliedWriteVersion(recordVersion, txnLogKey) {
|
|
404
|
+
return recordVersion == null ? txnLogKey : Math.min(recordVersion, txnLogKey);
|
|
405
|
+
}
|
|
329
406
|
/**
|
|
330
407
|
* The state a preceding write in this transaction left for `operation`'s key, or undefined if this
|
|
331
408
|
* is the first write to it. Within a transaction the writes are ordered by program order, and
|
|
@@ -368,6 +445,7 @@ class DatabaseTransaction {
|
|
|
368
445
|
this.#scopeOwned = options?.scopeOwned === true;
|
|
369
446
|
}
|
|
370
447
|
writes = []; // the set of writes to commit if the conditions are met
|
|
448
|
+
ownedWrites;
|
|
371
449
|
completions = []; // the set of outstanding async operations to complete
|
|
372
450
|
db;
|
|
373
451
|
transaction;
|
|
@@ -375,7 +453,11 @@ class DatabaseTransaction {
|
|
|
375
453
|
readTxnRefCount;
|
|
376
454
|
readTxnsUsed;
|
|
377
455
|
timeout;
|
|
456
|
+
writeTick = -1;
|
|
378
457
|
timeoutBudget = 0;
|
|
458
|
+
// Initialized rather than `declare`d so the class shape stays monomorphic on the write path. One clock
|
|
459
|
+
// read per handle is what lets the monitor age a handle without calling into the registry every tick.
|
|
460
|
+
handleOpenedAt = 0;
|
|
379
461
|
// save() only stages here; ImmediateTransaction overrides it to commit, which addWrite must not defer
|
|
380
462
|
saveCommits = false;
|
|
381
463
|
// True where save() puts the write into this transaction's native handle, which is what lets a scope
|
|
@@ -394,6 +476,22 @@ class DatabaseTransaction {
|
|
|
394
476
|
// asks to stop reading a pinned snapshot, so re-pinning one for the rest of the scope would take
|
|
395
477
|
// back what it asked for.
|
|
396
478
|
snapshotFree = false;
|
|
479
|
+
// Set while commit() is parked in its pre-commit await (`before`/`beforeIntermediate` completions —
|
|
480
|
+
// in practice a blob's durable file write). The write set is sealed and the caller is awaiting the
|
|
481
|
+
// commit, so this is core's own I/O rather than an application holding a transaction open, which is
|
|
482
|
+
// what the open-transaction limit polices: the monitor spares it for COMMIT_PHASE_GRACE ticks
|
|
483
|
+
// instead of poisoning it (issue #2062).
|
|
484
|
+
committing = false;
|
|
485
|
+
commitPhaseTicks = 0;
|
|
486
|
+
setCommitPhase(committing) {
|
|
487
|
+
// A commit phase covers the sealed write set across the whole multi-store chain.
|
|
488
|
+
for (let txn = this; txn; txn = txn.next) {
|
|
489
|
+
txn.committing = committing;
|
|
490
|
+
txn.commitChainHead = committing ? this : undefined;
|
|
491
|
+
if (committing)
|
|
492
|
+
txn.commitPhaseTicks = 0;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
397
495
|
rangeReadActive = false;
|
|
398
496
|
renewReadTimeout() {
|
|
399
497
|
// The limit is an IDLE limit. Writes always re-arm it (see addWrite), but reads only do so
|
|
@@ -409,6 +507,19 @@ class DatabaseTransaction {
|
|
|
409
507
|
this.timeout = Math.max(txnExpiration, this.timeoutBudget);
|
|
410
508
|
}
|
|
411
509
|
}
|
|
510
|
+
// Each engine keeps its own expiration; LMDBTransaction overrides this with its own.
|
|
511
|
+
renewIdleTimeout() {
|
|
512
|
+
this.timeout = Math.max(txnExpiration, this.timeoutBudget ?? 0);
|
|
513
|
+
}
|
|
514
|
+
// The links after this one wait for its native commit before their own commit() is entered; each
|
|
515
|
+
// hop starts with a full idle window and a stalled native commit is still bounded by one.
|
|
516
|
+
renewChainForNativeCommit() {
|
|
517
|
+
for (let txn = this; txn; txn = txn.next) {
|
|
518
|
+
if (txn.timedOut || txn.open === exports.TRANSACTION_STATE.CLOSED)
|
|
519
|
+
continue;
|
|
520
|
+
txn.renewIdleTimeout();
|
|
521
|
+
}
|
|
522
|
+
}
|
|
412
523
|
getReadTxn(disableSnapshot) {
|
|
413
524
|
this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
|
|
414
525
|
this.renewReadTimeout();
|
|
@@ -448,6 +559,7 @@ class DatabaseTransaction {
|
|
|
448
559
|
this.rangeReadActive = false;
|
|
449
560
|
this.readTxnsUsed = 1;
|
|
450
561
|
this.baseReadRefConsumed = false;
|
|
562
|
+
this.handleOpenedAt = performance.now();
|
|
451
563
|
}
|
|
452
564
|
/**
|
|
453
565
|
* Drop this link's supervision claim, and the root's with it once no link in the chain still holds
|
|
@@ -489,6 +601,7 @@ class DatabaseTransaction {
|
|
|
489
601
|
this.rangeReadActive = false;
|
|
490
602
|
this.readTxnsUsed = 0;
|
|
491
603
|
this.readTxnRefCount = 0;
|
|
604
|
+
this.handleOpenedAt = 0;
|
|
492
605
|
return transaction;
|
|
493
606
|
}
|
|
494
607
|
useReadTxn(disableSnapshot) {
|
|
@@ -535,12 +648,7 @@ class DatabaseTransaction {
|
|
|
535
648
|
*/
|
|
536
649
|
releaseReadTxn() {
|
|
537
650
|
const transaction = this.detachOwnedTransaction();
|
|
538
|
-
|
|
539
|
-
transaction?.abort();
|
|
540
|
-
}
|
|
541
|
-
catch (error) {
|
|
542
|
-
harperLogger.debug?.('releasing timed-out read transaction', error);
|
|
543
|
-
}
|
|
651
|
+
abortNativeTransaction(transaction, 'releasing timed-out read transaction');
|
|
544
652
|
this.completeDeferredContextRelease();
|
|
545
653
|
}
|
|
546
654
|
/**
|
|
@@ -592,6 +700,7 @@ class DatabaseTransaction {
|
|
|
592
700
|
const index = this.writes.indexOf(operation);
|
|
593
701
|
if (index > -1)
|
|
594
702
|
this.writes[index] = null;
|
|
703
|
+
this.ownedWrites?.delete(operation);
|
|
595
704
|
if (operation.key === undefined)
|
|
596
705
|
return;
|
|
597
706
|
const writesForStore = this.writesByKey?.get(operation.store);
|
|
@@ -601,7 +710,7 @@ class DatabaseTransaction {
|
|
|
601
710
|
// Membership, not `stagedIn`, which every commit handler clears and so cannot tell a takeover from a
|
|
602
711
|
// write done in place; a prior already taken over must not become this transaction's basis again.
|
|
603
712
|
let prior = operation.priorWrite;
|
|
604
|
-
while (prior && !this.
|
|
713
|
+
while (prior && !this.ownedWrites?.has(prior))
|
|
605
714
|
prior = prior.priorWrite;
|
|
606
715
|
const tail = writesForStore.get(keyId);
|
|
607
716
|
if (tail === operation) {
|
|
@@ -620,6 +729,77 @@ class DatabaseTransaction {
|
|
|
620
729
|
}
|
|
621
730
|
}
|
|
622
731
|
}
|
|
732
|
+
registerRecordLock(handle) {
|
|
733
|
+
if (!this.recordLocks)
|
|
734
|
+
this.recordLocks = new Map();
|
|
735
|
+
let storeMap = this.recordLocks.get(handle.store);
|
|
736
|
+
if (!storeMap)
|
|
737
|
+
this.recordLocks.set(handle.store, (storeMap = new Map()));
|
|
738
|
+
storeMap.set(handle.keyId, handle);
|
|
739
|
+
}
|
|
740
|
+
recordLockFor(store, keyId) {
|
|
741
|
+
const storeMap = this.recordLocks?.get(store);
|
|
742
|
+
if (!storeMap)
|
|
743
|
+
return undefined;
|
|
744
|
+
const h = storeMap.get(keyId);
|
|
745
|
+
if (!h)
|
|
746
|
+
return undefined;
|
|
747
|
+
if (!h.isExpired())
|
|
748
|
+
return h;
|
|
749
|
+
// Prune released/expired handles so they don't accumulate; a handle past its deadline checking
|
|
750
|
+
// re-entrancy would otherwise be seen as the holder and incorrectly granted access.
|
|
751
|
+
storeMap.delete(keyId);
|
|
752
|
+
return undefined;
|
|
753
|
+
}
|
|
754
|
+
unregisterRecordLock(handle) {
|
|
755
|
+
const storeMap = this.recordLocks?.get(handle.store);
|
|
756
|
+
if (storeMap?.get(handle.keyId) === handle)
|
|
757
|
+
storeMap.delete(handle.keyId);
|
|
758
|
+
}
|
|
759
|
+
registerPendingLock(store, keyId, pending) {
|
|
760
|
+
if (!this.pendingLocks)
|
|
761
|
+
this.pendingLocks = new Map();
|
|
762
|
+
let storeMap = this.pendingLocks.get(store);
|
|
763
|
+
if (!storeMap)
|
|
764
|
+
this.pendingLocks.set(store, (storeMap = new Map()));
|
|
765
|
+
storeMap.set(keyId, pending);
|
|
766
|
+
}
|
|
767
|
+
pendingLockFor(store, keyId) {
|
|
768
|
+
return this.pendingLocks?.get(store)?.get(keyId);
|
|
769
|
+
}
|
|
770
|
+
unregisterPendingLock(store, keyId) {
|
|
771
|
+
this.pendingLocks?.get(store)?.delete(keyId);
|
|
772
|
+
}
|
|
773
|
+
/** Release every transaction-scoped lock() handle this link owns. */
|
|
774
|
+
releaseRecordLocks() {
|
|
775
|
+
const recordLocks = this.recordLocks;
|
|
776
|
+
if (!recordLocks)
|
|
777
|
+
return;
|
|
778
|
+
for (const [store, storeMap] of recordLocks) {
|
|
779
|
+
for (const [keyId, handle] of storeMap) {
|
|
780
|
+
if (handle.hold)
|
|
781
|
+
continue; // hold handles outlive the transaction; released by unlock()
|
|
782
|
+
handle.release();
|
|
783
|
+
storeMap.delete(keyId);
|
|
784
|
+
}
|
|
785
|
+
if (storeMap.size === 0)
|
|
786
|
+
recordLocks.delete(store);
|
|
787
|
+
}
|
|
788
|
+
if (recordLocks.size === 0)
|
|
789
|
+
this.recordLocks = undefined;
|
|
790
|
+
}
|
|
791
|
+
noteCommittedLockVersions() {
|
|
792
|
+
const recordLocks = this.recordLocks;
|
|
793
|
+
if (!recordLocks)
|
|
794
|
+
return;
|
|
795
|
+
for (const write of this.writes) {
|
|
796
|
+
if (write?.appliedRecordVersion == null)
|
|
797
|
+
continue;
|
|
798
|
+
const handle = write.lockHandle ?? recordLocks.get(write.store)?.get(writeKeyId(write.key));
|
|
799
|
+
if (handle && !handle.released)
|
|
800
|
+
handle.noteHolderVersion(write.appliedRecordVersion);
|
|
801
|
+
}
|
|
802
|
+
}
|
|
623
803
|
/**
|
|
624
804
|
* Discard the staged write set (committed or aborted); the per-key chain must go with it so a
|
|
625
805
|
* reused transaction never bases a write on a previous batch's staged state.
|
|
@@ -629,11 +809,19 @@ class DatabaseTransaction {
|
|
|
629
809
|
// fire after a commit or abort, and routing it back here would revive a write this transaction
|
|
630
810
|
// already rolled back — whose blobs abort() has reclaimed. Cleared here, save() resolves the
|
|
631
811
|
// context's current transaction as it did before `stagedIn` existed.
|
|
632
|
-
for (const write of this.writes)
|
|
812
|
+
for (const write of this.writes) {
|
|
633
813
|
if (write?.stagedIn === this)
|
|
634
814
|
write.stagedIn = undefined;
|
|
815
|
+
if (write)
|
|
816
|
+
this.ownedWrites?.delete(write);
|
|
817
|
+
}
|
|
635
818
|
this.writes = [];
|
|
636
819
|
this.writesByKey = undefined;
|
|
820
|
+
// The commit-fence latch belongs to the discarded batch. A reused transaction that once staged
|
|
821
|
+
// a locked write would otherwise re-scan every write of every later unlocked batch. Assigned
|
|
822
|
+
// only when it was set, so an ordinary transaction never gains the property at all.
|
|
823
|
+
if (this.hasLeaseProtectedWrite)
|
|
824
|
+
this.hasLeaseProtectedWrite = false;
|
|
637
825
|
}
|
|
638
826
|
/**
|
|
639
827
|
* Drop this transaction's back-reference from its context once completed (commit or abort),
|
|
@@ -679,25 +867,20 @@ class DatabaseTransaction {
|
|
|
679
867
|
// original per-request log was fixed to avoid, just shifted from per-request to per-commit.
|
|
680
868
|
// A commit skipped by the cooldown is NOT marked `logged`, so it still gets a log later if
|
|
681
869
|
// it's still the oldest once the cooldown clears, rather than going silent forever.
|
|
682
|
-
if (!oldestOutstandingCommit.logged && now
|
|
870
|
+
if (!oldestOutstandingCommit.logged && allowStuckCommitLog('shed', now)) {
|
|
683
871
|
// Log once per stuck commit (not once per rejected request, harper#2001): a wedged
|
|
684
872
|
// thread otherwise logs nothing at all server-side while rejecting every write with a
|
|
685
873
|
// 503, which was the single biggest obstacle to root-causing a recurrence. The flag lives
|
|
686
874
|
// on the node itself, so if THIS commit settles while still over the limit and a
|
|
687
875
|
// different one is now oldest, that one logs too instead of staying silent forever.
|
|
688
876
|
oldestOutstandingCommit.logged = true;
|
|
689
|
-
lastOverloadLogAt = now;
|
|
690
|
-
const nativeTransactionId = oldestOutstandingCommit.nativeTransaction?.id;
|
|
691
|
-
const store = oldestOutstandingCommit.store;
|
|
692
|
-
const startedFrom = oldestOutstandingCommit.startedFrom;
|
|
693
877
|
harperLogger.error(`Rejecting writes on this thread: a commit has been outstanding for ` +
|
|
694
878
|
`${Math.round(now - oldestOutstandingCommit.start)}ms (exceeds the ` +
|
|
695
|
-
`${MAX_OUTSTANDING_TXN_DURATION}ms limit),
|
|
696
|
-
(
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
`. Further record updates and publishes from new application requests on this thread ` +
|
|
879
|
+
`${MAX_OUTSTANDING_TXN_DURATION}ms limit), ` +
|
|
880
|
+
describeCommitIdentity(oldestOutstandingCommit.store, oldestOutstandingCommit.startedFrom, oldestOutstandingCommit.nativeTransaction) +
|
|
881
|
+
`.` +
|
|
882
|
+
(0, longLivedTransactions_ts_1.describeHolderCandidates)(oldestOutstandingCommit.store?.rootStore?.path, oldestOutstandingCommit.nativeTransaction?.id) +
|
|
883
|
+
` Further record updates and publishes from new application requests on this thread ` +
|
|
701
884
|
`will be rejected with 503 until the commit settles or the process is restarted (deletes, ` +
|
|
702
885
|
`and writes applied from a canonical source, e.g. replication or a caching source, bypass this check).`);
|
|
703
886
|
}
|
|
@@ -758,8 +941,10 @@ class DatabaseTransaction {
|
|
|
758
941
|
// Independent write-recency signal for chainStillActive (see the field comment) — reads never
|
|
759
942
|
// touch this, only writes do.
|
|
760
943
|
this.writeTimeout = this.timeout;
|
|
944
|
+
this.writeTick = monitorTick;
|
|
761
945
|
this.linkWrite(operation);
|
|
762
946
|
this.writes.push(operation);
|
|
947
|
+
(this.ownedWrites ??= new WeakSet()).add(operation);
|
|
763
948
|
operation.stagedIn = this;
|
|
764
949
|
// Hold this write back while any earlier same-key write has not run — out of staging order both
|
|
765
950
|
// diff against the pre-transaction record (harper#2211, DESIGN.md). The whole chain, not just the
|
|
@@ -787,7 +972,44 @@ class DatabaseTransaction {
|
|
|
787
972
|
return operation;
|
|
788
973
|
}
|
|
789
974
|
save(operation, transaction, reloadEntry = false, options) {
|
|
790
|
-
|
|
975
|
+
const lockHandle = operation.lockHandle;
|
|
976
|
+
// Guard: a write staged through an expired or released lock handle must not land.
|
|
977
|
+
// The handle's lease timer already unlocked the native key; another holder may have taken it.
|
|
978
|
+
// isExpired() re-evaluates the deadline here rather than trusting the timer to have run: a
|
|
979
|
+
// holder whose event loop stalled past its lease would otherwise commit in the window between
|
|
980
|
+
// the deadline and its own timer callback, after peers had already granted the key onward.
|
|
981
|
+
//
|
|
982
|
+
// A RE-save is judged by the lease alone. `released` belongs to the first stage — a caller that
|
|
983
|
+
// unlocked and then saved is writing through a handle it gave back — but an operation already
|
|
984
|
+
// staged was staged while the lock was held, and an `unlock()` inside the lease leaves it valid
|
|
985
|
+
// (the same rule the pre-submit fence applies below). Without the distinction,
|
|
986
|
+
// `lock(); save(); unlock(); commit()` succeeded normally and threw 409 only when a conflict
|
|
987
|
+
// retry or an open read iterator forced the replay to re-save it.
|
|
988
|
+
if (operation.saved ? lockHandle?.isLeaseExpired() : lockHandle?.isExpired()) {
|
|
989
|
+
// Through `detachWrite`, which also repairs the per-key chain: dropping it from `writes` alone
|
|
990
|
+
// leaves `writesByKey` pointing at a write that was refused, so a caller that catches this 409
|
|
991
|
+
// and stages the same key again would take the rejected operation as its merge basis.
|
|
992
|
+
this.detachWrite(operation);
|
|
993
|
+
throw (0, recordLock_ts_1.lockNotHeldError)(lockHandle);
|
|
994
|
+
}
|
|
995
|
+
// Lock-write timestamp rules.
|
|
996
|
+
if (lockHandle) {
|
|
997
|
+
this.hasLeaseProtectedWrite = true;
|
|
998
|
+
if (this.open === exports.TRANSACTION_STATE.CLOSED || this.saveCommits) {
|
|
999
|
+
// CLOSED path (second+ write per ImmediateTransaction cycle) OR the first write in
|
|
1000
|
+
// an ImmediateTransaction (open=OPEN until commit sets it CLOSED, but saveCommits
|
|
1001
|
+
// signals the per-write-commit semantics): stamp with nextHolderVersion() so that
|
|
1002
|
+
// each sequential save() gets its own monotonically increasing stamp — scoped or hold.
|
|
1003
|
+
// The stamp stays on the operation and is consumed below rather than assigned to
|
|
1004
|
+
// this.timestamp: pinning the link's clock would stamp every OTHER write staged on
|
|
1005
|
+
// the same context before the commit resets it — a concurrent write in the caller's
|
|
1006
|
+
// own Promise.all, or the next operation in a retry/replay save loop — with the
|
|
1007
|
+
// lock's version, which LWW then silently drops against a newer record version.
|
|
1008
|
+
if (!operation.lockStamp)
|
|
1009
|
+
operation.lockStamp = lockHandle.holderVersionCandidate();
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
let txnTime = operation.lockStamp ?? this.timestamp;
|
|
791
1013
|
// Only an OPEN transaction accepts new staged writes. After commit, this.transaction may still
|
|
792
1014
|
// be retained for outstanding read iterators; staging into it would silently discard the write
|
|
793
1015
|
// when doneReadTxn() aborts the handle, so such writes commit immediately on a fresh
|
|
@@ -832,16 +1054,44 @@ class DatabaseTransaction {
|
|
|
832
1054
|
}
|
|
833
1055
|
if (!txnTime)
|
|
834
1056
|
txnTime = this.timestamp = transaction.getTimestamp();
|
|
835
|
-
if (
|
|
836
|
-
|
|
1057
|
+
if (!operation.saved && operation.pendingPriorWrite) {
|
|
1058
|
+
const pendingWrites = [];
|
|
1059
|
+
for (let pending = operation.pendingPriorWrite; pending;) {
|
|
1060
|
+
if (!pending.saved && this.ownedWrites?.has(pending))
|
|
1061
|
+
pendingWrites.push(pending);
|
|
1062
|
+
pending = pending.pendingPriorWrite !== undefined ? pending.pendingPriorWrite : pending.priorWrite;
|
|
1063
|
+
}
|
|
1064
|
+
for (let index = pendingWrites.length - 1; index >= 0; index--)
|
|
1065
|
+
this.save(pendingWrites[index], transaction, false, options);
|
|
1066
|
+
operation.pendingPriorWrite = null;
|
|
1067
|
+
}
|
|
1068
|
+
// `txnTime` is this transaction's timestamp — the key its entries take in the per-origin log.
|
|
1069
|
+
// A write applied from elsewhere carries the origin's record version too, and that is what the
|
|
1070
|
+
// record is stored at; the two coincide for every locally-originated write. Gated on the apply
|
|
1071
|
+
// flags so an ordinary write never reads the property (harper#2412).
|
|
1072
|
+
const writeVersion = this.sourceApply || this.isReplay ? getAppliedWriteVersion(operation.recordVersion, txnTime) : txnTime;
|
|
1073
|
+
// A base that feeds stored state must come from this transaction's snapshot, never the
|
|
1074
|
+
// cross-worker cache vouch (stale when a resequenced write reused a version). That closes the
|
|
1075
|
+
// lost-update window only when this transaction holds a snapshot to validate the later Put
|
|
1076
|
+
// against; a snapshot-free transaction (this.snapshotFree, after a mid-scope-commit rotation)
|
|
1077
|
+
// has no snapshot for rocksdb-js to validate the Put against, so it only narrows the window to
|
|
1078
|
+
// the read-to-put span rather than closing it (open follow-up, tracked in the PR description).
|
|
1079
|
+
// Replays keep their pre-read base — their convergence contract is the replay pass itself.
|
|
1080
|
+
const reloadsCommitBase = operation.reloadCommitBase && !operation.saved && !this.isReplay;
|
|
1081
|
+
if (reloadEntry || operation.entry === undefined || reloadsCommitBase) {
|
|
1082
|
+
const uncachedRead = (!!operation.reloadCommitBase && !this.isReplay) || reloadEntry;
|
|
1083
|
+
operation.entry = operation.store.getEntry(operation.key, { transaction, uncachedRead });
|
|
837
1084
|
}
|
|
838
1085
|
if (!operation.saved) {
|
|
839
|
-
operation.saved = true;
|
|
840
1086
|
// immediately execute in this transaction
|
|
841
|
-
|
|
1087
|
+
const validated = validateWrite(operation, writeVersion, this);
|
|
1088
|
+
if (validated === false) {
|
|
1089
|
+
operation.saved = true;
|
|
842
1090
|
operation.commit = () => { }; // noop if we try again
|
|
1091
|
+
closeWriteInstance(operation);
|
|
843
1092
|
return;
|
|
844
1093
|
}
|
|
1094
|
+
operation.saved = true;
|
|
845
1095
|
let result = operation.before?.();
|
|
846
1096
|
if (result?.then)
|
|
847
1097
|
this.stageCompletion(result);
|
|
@@ -849,7 +1099,19 @@ class DatabaseTransaction {
|
|
|
849
1099
|
if (result?.then)
|
|
850
1100
|
this.stageCompletion(result);
|
|
851
1101
|
}
|
|
852
|
-
|
|
1102
|
+
if (lockHandle || this.recordLocks)
|
|
1103
|
+
operation.trackRecordVersion = true;
|
|
1104
|
+
if (operation.trackRecordVersion)
|
|
1105
|
+
operation.recordVersionApplied = false;
|
|
1106
|
+
let completion;
|
|
1107
|
+
try {
|
|
1108
|
+
completion = operation.commit(writeVersion, operation.entry, this.retries > 0, transaction);
|
|
1109
|
+
}
|
|
1110
|
+
finally {
|
|
1111
|
+
closeWriteInstance(operation);
|
|
1112
|
+
}
|
|
1113
|
+
if (operation.trackRecordVersion)
|
|
1114
|
+
operation.appliedRecordVersion = operation.recordVersionApplied ? writeVersion : undefined;
|
|
853
1115
|
if (typeof completion?.then === 'function')
|
|
854
1116
|
this.stageCompletion(completion);
|
|
855
1117
|
// Sticky record that THIS write staged with its audit entry appended (log entries batch on the
|
|
@@ -862,7 +1124,13 @@ class DatabaseTransaction {
|
|
|
862
1124
|
operation.appendedAuditEntry = true;
|
|
863
1125
|
}
|
|
864
1126
|
if (immediateCommit) {
|
|
865
|
-
|
|
1127
|
+
// immediately commit if the harper transaction is closed
|
|
1128
|
+
const innerCommit = this.commit({ ...options, transaction });
|
|
1129
|
+
// Expose on the operation so the lock-writable Table.save() path can await the real
|
|
1130
|
+
// native commit — without this the ImmediateTransaction outer commit resolves before
|
|
1131
|
+
// the native transaction.commit() settles (fire-and-forget from the if-branch).
|
|
1132
|
+
operation.innerCommit = innerCommit;
|
|
1133
|
+
return innerCommit;
|
|
866
1134
|
}
|
|
867
1135
|
}
|
|
868
1136
|
/**
|
|
@@ -874,17 +1142,38 @@ class DatabaseTransaction {
|
|
|
874
1142
|
// reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain —
|
|
875
1143
|
// but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction
|
|
876
1144
|
let transaction = options.transaction ?? this.transaction;
|
|
877
|
-
|
|
878
|
-
let
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
1145
|
+
try {
|
|
1146
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1147
|
+
let operation = this.writes[i];
|
|
1148
|
+
if (!operation || (this.retries === 0 && operation.saved))
|
|
1149
|
+
continue;
|
|
1150
|
+
this.save(operation, transaction, i < this.validated, options);
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
catch (error) {
|
|
1154
|
+
this.abort();
|
|
1155
|
+
throw error;
|
|
882
1156
|
}
|
|
883
1157
|
this.validated = this.writes.length;
|
|
884
1158
|
const completions = this.completions;
|
|
885
1159
|
if (completions.length > 0)
|
|
886
1160
|
this.completions = []; // reset
|
|
1161
|
+
const stagedWrites = this.writes.length;
|
|
1162
|
+
if (completions.length > 0) {
|
|
1163
|
+
this.setCommitPhase(true);
|
|
1164
|
+
}
|
|
887
1165
|
return (0, when_ts_1.when)(completions.length > 0 ? Promise.all(completions) : null, () => {
|
|
1166
|
+
if (completions.length > 0)
|
|
1167
|
+
this.setCommitPhase(false);
|
|
1168
|
+
// The transaction can be aborted underneath us while we are parked in the await above — by the
|
|
1169
|
+
// monitor once the commit phase outlives its grace, or through the multi-store poison chain.
|
|
1170
|
+
// abort() cleared the write set and released the handle, so resuming would commit nothing and
|
|
1171
|
+
// resolve as SUCCESS: the caller is told its write landed when it was dropped, and a write
|
|
1172
|
+
// carrying a blob is left holding an instance whose file was unlinked (issue #2062).
|
|
1173
|
+
if (this.timedOut)
|
|
1174
|
+
throw transactionOpenTooLongError();
|
|
1175
|
+
if (stagedWrites > 0 && this.writes.length === 0 && this.open === exports.TRANSACTION_STATE.CLOSED)
|
|
1176
|
+
throw new hdbError_ts_1.ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
|
|
888
1177
|
if (this.writes.length > this.validated) {
|
|
889
1178
|
// check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
|
|
890
1179
|
return this.commit(options);
|
|
@@ -897,6 +1186,8 @@ class DatabaseTransaction {
|
|
|
897
1186
|
// and replay branches below deliberately commit a handle other than this.transaction.
|
|
898
1187
|
if (!transaction)
|
|
899
1188
|
transaction = this.transaction;
|
|
1189
|
+
if (!options.transaction && this.writes.some((write) => write))
|
|
1190
|
+
this.renewChainForNativeCommit();
|
|
900
1191
|
this.open = exports.TRANSACTION_STATE.CLOSED;
|
|
901
1192
|
// RocksTransaction.commit() resolves with RETRY_NOW_VALUE (a number) under
|
|
902
1193
|
// coordinatedRetry, or void on a normal commit/abort.
|
|
@@ -982,6 +1273,31 @@ class DatabaseTransaction {
|
|
|
982
1273
|
if (this.writes.length > 0) {
|
|
983
1274
|
// Commit retries can construct fresh ranges on this live handle after read ownership ends.
|
|
984
1275
|
readTransactionOwners.delete(transaction);
|
|
1276
|
+
// Re-fence before submitting: the loop above skips operations already marked saved, so
|
|
1277
|
+
// save()'s own check cannot see a holder that stalled between staging and commit. The
|
|
1278
|
+
// retry/replay path re-saves every operation and is fenced there instead. Lease expiry
|
|
1279
|
+
// only: an unlock() inside the lease leaves the staged write valid, an elapsed lease
|
|
1280
|
+
// does not, whether or not the caller also unlocked.
|
|
1281
|
+
for (let i = 0; this.hasLeaseProtectedWrite && i < this.writes.length; i++) {
|
|
1282
|
+
const lapsed = this.writes[i].lockHandle;
|
|
1283
|
+
if (!lapsed?.isLeaseExpired())
|
|
1284
|
+
continue;
|
|
1285
|
+
try {
|
|
1286
|
+
transaction.abort();
|
|
1287
|
+
}
|
|
1288
|
+
catch { }
|
|
1289
|
+
// Every other terminal exit from commit() runs the logical cleanup too. Throwing
|
|
1290
|
+
// straight out would strand the OTHER locks this transaction holds, its staged
|
|
1291
|
+
// blobs, and the context's back-reference to a CLOSED transaction — and
|
|
1292
|
+
// transaction()'s onComplete has no rejection path to run it later.
|
|
1293
|
+
try {
|
|
1294
|
+
this.abort();
|
|
1295
|
+
}
|
|
1296
|
+
catch (error) {
|
|
1297
|
+
harperLogger.debug?.('cleaning up a transaction whose record lock lapsed', error);
|
|
1298
|
+
}
|
|
1299
|
+
throw (0, recordLock_ts_1.lockNotHeldError)(lapsed);
|
|
1300
|
+
}
|
|
985
1301
|
// The transaction was created with coordinatedRetry:true (see
|
|
986
1302
|
// getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
|
|
987
1303
|
// sentinel (a number) is why commitResolution is typed
|
|
@@ -1020,14 +1336,22 @@ class DatabaseTransaction {
|
|
|
1020
1336
|
// claimed this per-database transaction in txnForContext and so can name the wrong
|
|
1021
1337
|
// table when a transaction spans more than one table in the same database.
|
|
1022
1338
|
trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
|
|
1339
|
+
// Every retry round and every chained store re-enters here and must inherit the chain
|
|
1340
|
+
// root's clock rather than restart it, so only the first submission stamps.
|
|
1341
|
+
const chainRoot = this.root ?? this;
|
|
1342
|
+
if (chainRoot.commitStartedAt == null)
|
|
1343
|
+
chainRoot.commitStartedAt = performance.now();
|
|
1023
1344
|
const completions = [];
|
|
1024
|
-
|
|
1345
|
+
const commitOutcome = commitResolution.then((commitResult) => {
|
|
1025
1346
|
if (commitResult === RETRY_NOW_VALUE) {
|
|
1026
1347
|
this.retries++;
|
|
1027
1348
|
harperLogger.debug?.('coordinated retry', transaction.id, this.retries);
|
|
1028
1349
|
// Mark this specific native transaction as a retry so RocksTransactionLogStore
|
|
1029
1350
|
// skips re-writing its already-staged txn-log entries (#2).
|
|
1030
1351
|
transaction.isRetry = true;
|
|
1352
|
+
const pastBudget = this.elapsedPastCommitBudget();
|
|
1353
|
+
if (pastBudget)
|
|
1354
|
+
this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
1031
1355
|
// Mirror the ERR_BUSY cap/warn policy: non-sourceApply transactions abort
|
|
1032
1356
|
// at MAX_RETRIES; sourceApply transactions keep retrying with periodic warn.
|
|
1033
1357
|
if (this.retries > MAX_RETRIES) {
|
|
@@ -1080,9 +1404,12 @@ class DatabaseTransaction {
|
|
|
1080
1404
|
if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
|
|
1081
1405
|
(0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
|
|
1082
1406
|
}
|
|
1407
|
+
if (this.recordLocks)
|
|
1408
|
+
this.noteCommittedLockVersions();
|
|
1083
1409
|
// now reset transactions tracking; this transaction be reused and committed again
|
|
1084
1410
|
this.retries = 0; // reset per-native-transaction retry counter so a reused DatabaseTransaction's next batch starts fresh
|
|
1085
1411
|
this.clearWrites();
|
|
1412
|
+
this.releaseRecordLocks();
|
|
1086
1413
|
if (options.doneWriting)
|
|
1087
1414
|
this.endScopeOwnership();
|
|
1088
1415
|
this.releaseContext(!!options.doneWriting);
|
|
@@ -1134,6 +1461,11 @@ class DatabaseTransaction {
|
|
|
1134
1461
|
// premature publish, and no fresh-transaction replay that would drop the entry.
|
|
1135
1462
|
// Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
|
|
1136
1463
|
transaction.isRetry = true;
|
|
1464
|
+
// Before the backoff gate below: the budget can already be spent on the first
|
|
1465
|
+
// retry when an earlier store in this chain consumed it.
|
|
1466
|
+
const pastBudget = this.elapsedPastCommitBudget();
|
|
1467
|
+
if (pastBudget)
|
|
1468
|
+
this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
1137
1469
|
if (this.retries > 2) {
|
|
1138
1470
|
// Transactions applying data from a canonical source of truth (replication peer or
|
|
1139
1471
|
// external caching source) must never drop a write on a transient conflict: there is no
|
|
@@ -1178,24 +1510,43 @@ class DatabaseTransaction {
|
|
|
1178
1510
|
// finished and its durability is unknown, so ownership goes with it.
|
|
1179
1511
|
this.endScopeOwnership();
|
|
1180
1512
|
this.releaseContext(!!options.doneWriting);
|
|
1513
|
+
this.releaseRecordLocks();
|
|
1514
|
+
this.timestamp = 0;
|
|
1181
1515
|
throw error;
|
|
1182
1516
|
}
|
|
1183
1517
|
});
|
|
1518
|
+
// `commitOutcome` settles when the LOGICAL commit ends — its success branch awaits the
|
|
1519
|
+
// chained stores' own commits — so releasing here covers every terminal exit (success,
|
|
1520
|
+
// retry exhaustion, abandonment, terminal failure) in one place rather than five.
|
|
1521
|
+
// Released through the RETURNED promise rather than a second subscriber on
|
|
1522
|
+
// `commitOutcome`: a subscriber would mark a dropped commit rejection as handled and
|
|
1523
|
+
// silence the unhandled-rejection that surfaces it.
|
|
1524
|
+
return commitOutcome.then((resolution) => {
|
|
1525
|
+
chainRoot.commitStartedAt = undefined;
|
|
1526
|
+
return resolution;
|
|
1527
|
+
}, (error) => {
|
|
1528
|
+
chainRoot.commitStartedAt = undefined;
|
|
1529
|
+
throw error;
|
|
1530
|
+
});
|
|
1184
1531
|
}
|
|
1185
1532
|
for (const write of this.writes) {
|
|
1186
1533
|
if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
|
|
1187
1534
|
(0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
|
|
1188
1535
|
}
|
|
1536
|
+
if (this.recordLocks)
|
|
1537
|
+
this.noteCommittedLockVersions();
|
|
1189
1538
|
this.clearWrites();
|
|
1539
|
+
this.releaseRecordLocks();
|
|
1190
1540
|
if (options.doneWriting)
|
|
1191
1541
|
this.endScopeOwnership();
|
|
1192
1542
|
this.releaseContext(!!options.doneWriting);
|
|
1193
1543
|
const txnResolution = {
|
|
1194
1544
|
txnTime: this.timestamp,
|
|
1195
1545
|
};
|
|
1546
|
+
this.timestamp = 0; // reset like the async path (~1279) so stale lock stamps don't persist
|
|
1196
1547
|
if (this.next) {
|
|
1197
1548
|
// now run any other transactions
|
|
1198
|
-
options.timestamp =
|
|
1549
|
+
options.timestamp = txnResolution.txnTime;
|
|
1199
1550
|
// as above: the next store must not inherit this store's explicit native transaction
|
|
1200
1551
|
let nextResolution;
|
|
1201
1552
|
try {
|
|
@@ -1214,7 +1565,7 @@ class DatabaseTransaction {
|
|
|
1214
1565
|
// scope resumable after a partially failed mid-scope commit.
|
|
1215
1566
|
this.completeMidScopeCommit(options);
|
|
1216
1567
|
return {
|
|
1217
|
-
txnTime:
|
|
1568
|
+
txnTime: txnResolution.txnTime,
|
|
1218
1569
|
next: nextResolution,
|
|
1219
1570
|
};
|
|
1220
1571
|
}, (error) => {
|
|
@@ -1230,6 +1581,7 @@ class DatabaseTransaction {
|
|
|
1230
1581
|
this.completeMidScopeCommit(options);
|
|
1231
1582
|
return txnResolution;
|
|
1232
1583
|
}, (error) => {
|
|
1584
|
+
this.setCommitPhase(false);
|
|
1233
1585
|
this.abort();
|
|
1234
1586
|
throw error;
|
|
1235
1587
|
});
|
|
@@ -1277,6 +1629,7 @@ class DatabaseTransaction {
|
|
|
1277
1629
|
if (this.transaction)
|
|
1278
1630
|
this.releaseReadTxn();
|
|
1279
1631
|
this.open = exports.TRANSACTION_STATE.CLOSED;
|
|
1632
|
+
this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
|
|
1280
1633
|
this.drainCompletions();
|
|
1281
1634
|
try {
|
|
1282
1635
|
for (const write of this.writes) {
|
|
@@ -1287,6 +1640,7 @@ class DatabaseTransaction {
|
|
|
1287
1640
|
finally {
|
|
1288
1641
|
this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
|
|
1289
1642
|
this.clearWrites();
|
|
1643
|
+
this.releaseRecordLocks();
|
|
1290
1644
|
// A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
|
|
1291
1645
|
// Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
|
|
1292
1646
|
// starting a fresh one) so the rest of the logical operation fails atomically via the
|
|
@@ -1306,6 +1660,56 @@ class DatabaseTransaction {
|
|
|
1306
1660
|
}
|
|
1307
1661
|
}
|
|
1308
1662
|
}
|
|
1663
|
+
/** How long this logical commit may keep retrying: the thread-wide queue limit, or its own larger explicit budget. */
|
|
1664
|
+
commitConflictBudget() {
|
|
1665
|
+
return Math.max(MAX_OUTSTANDING_TXN_DURATION, (this.root ?? this).timeoutBudget || 0);
|
|
1666
|
+
}
|
|
1667
|
+
/**
|
|
1668
|
+
* How long this logical commit has been retrying once it is past its budget, else 0. rocksdb-js
|
|
1669
|
+
* returns control from a parked commit every `ROCKSDB_JS_PARK_TIMEOUT_MS` even when the intent
|
|
1670
|
+
* holder never releases, so without this bound a request-path commit retries the attempt cap out
|
|
1671
|
+
* — minutes past the queue limit an operator configured (issue #2450).
|
|
1672
|
+
*
|
|
1673
|
+
* Source-applied writes are exempt for the same reason they are exempt from the attempt cap:
|
|
1674
|
+
* there is no resubscribe/sequence-resume path, so dropping one permanently diverges this node.
|
|
1675
|
+
*/
|
|
1676
|
+
elapsedPastCommitBudget() {
|
|
1677
|
+
if (this.sourceApply)
|
|
1678
|
+
return 0;
|
|
1679
|
+
const startedAt = (this.root ?? this).commitStartedAt;
|
|
1680
|
+
if (startedAt == null)
|
|
1681
|
+
return 0;
|
|
1682
|
+
const elapsed = performance.now() - startedAt;
|
|
1683
|
+
return elapsed > this.commitConflictBudget() ? elapsed : 0;
|
|
1684
|
+
}
|
|
1685
|
+
/**
|
|
1686
|
+
* Abandon a logical commit that stayed in write-intent conflict past its budget. Cleanup is
|
|
1687
|
+
* retry exhaustion's, so no link leaks a native handle or read snapshot; the error is distinct
|
|
1688
|
+
* because the condition is distinct — every attempt reported transient contention, so a later
|
|
1689
|
+
* request can succeed once the holder releases, which the generic exhaustion 500 does not say.
|
|
1690
|
+
*
|
|
1691
|
+
* Only a chain root may report `retryable`: a link commits solely from its predecessor's success
|
|
1692
|
+
* handler, so anywhere else in the chain an earlier store has already landed durable audit
|
|
1693
|
+
* entries and hooks that a replayed request would run twice. A head whose scope already rotated
|
|
1694
|
+
* through a mid-scope commit is in the same position.
|
|
1695
|
+
*/
|
|
1696
|
+
abandonCommitAfterDeadline(headTransaction, elapsedMs) {
|
|
1697
|
+
const elapsed = Math.round(elapsedMs);
|
|
1698
|
+
const budget = this.commitConflictBudget();
|
|
1699
|
+
const retryable = !this.root && !this.snapshotFree;
|
|
1700
|
+
if (allowStuckCommitLog('abandon', performance.now())) {
|
|
1701
|
+
harperLogger.error(`Abandoning a write transaction: its commit has been in write-intent conflict for ${elapsed}ms ` +
|
|
1702
|
+
`(exceeds the ${budget}ms limit) across ${this.retries} retries, ` +
|
|
1703
|
+
describeCommitIdentity(this.writes[0]?.store, this.startedFrom, headTransaction) +
|
|
1704
|
+
`.` +
|
|
1705
|
+
(0, longLivedTransactions_ts_1.describeHolderCandidates)(this.writes[0]?.store?.rootStore?.path, headTransaction?.id) +
|
|
1706
|
+
` Another transaction holds a conflicting write intent and has not completed; the request is ` +
|
|
1707
|
+
`failed with a 503${retryable ? '' : ' (not retryable — an earlier store in this transaction already committed)'} ` +
|
|
1708
|
+
`rather than waiting further.`);
|
|
1709
|
+
}
|
|
1710
|
+
this.abortChainAfterRetries(headTransaction);
|
|
1711
|
+
throw new hdbError_ts_1.TransactionCommitConflictTimeoutError(`Commit was in conflict with ongoing writes for ${elapsed}ms, exceeding the ${budget}ms limit; transaction abandoned after ${this.retries} retries`, retryable);
|
|
1712
|
+
}
|
|
1309
1713
|
/**
|
|
1310
1714
|
* Give up on a chain of linked transactions after exhausting conflict retries: poison every link
|
|
1311
1715
|
* first, then abort each link's native transaction and release its DatabaseTransaction-level
|
|
@@ -1401,12 +1805,7 @@ class DatabaseTransaction {
|
|
|
1401
1805
|
// throws. abort() rather than the native abort alone: it reclaims blobs a replayed write
|
|
1402
1806
|
// staged.
|
|
1403
1807
|
this.detachOwnedTransaction();
|
|
1404
|
-
|
|
1405
|
-
transaction?.abort();
|
|
1406
|
-
}
|
|
1407
|
-
catch (abortError) {
|
|
1408
|
-
harperLogger.debug?.('aborting a transaction whose synchronous commit failed', abortError);
|
|
1409
|
-
}
|
|
1808
|
+
abortNativeTransaction(transaction, 'aborting a transaction whose synchronous commit failed');
|
|
1410
1809
|
try {
|
|
1411
1810
|
this.abort();
|
|
1412
1811
|
}
|
|
@@ -1425,6 +1824,18 @@ class DatabaseTransaction {
|
|
|
1425
1824
|
}
|
|
1426
1825
|
}
|
|
1427
1826
|
exports.DatabaseTransaction = DatabaseTransaction;
|
|
1827
|
+
function shouldSpareCommitPhase(txn, checkedCommitPhaseChains) {
|
|
1828
|
+
if (!txn.committing)
|
|
1829
|
+
return false;
|
|
1830
|
+
if (txn.sourceApply || txn.isReplay)
|
|
1831
|
+
return true;
|
|
1832
|
+
const commitChainHead = txn.commitChainHead ?? txn;
|
|
1833
|
+
if (!checkedCommitPhaseChains.has(commitChainHead)) {
|
|
1834
|
+
checkedCommitPhaseChains.add(commitChainHead);
|
|
1835
|
+
commitChainHead.commitPhaseTicks++;
|
|
1836
|
+
}
|
|
1837
|
+
return commitChainHead.commitPhaseTicks <= exports.COMMIT_PHASE_GRACE;
|
|
1838
|
+
}
|
|
1428
1839
|
class ImmediateTransaction extends DatabaseTransaction {
|
|
1429
1840
|
isCommitting = false;
|
|
1430
1841
|
saveCommits = true;
|
|
@@ -1433,18 +1844,50 @@ class ImmediateTransaction extends DatabaseTransaction {
|
|
|
1433
1844
|
this.db = db;
|
|
1434
1845
|
}
|
|
1435
1846
|
save(...args) {
|
|
1436
|
-
const
|
|
1847
|
+
const operation = args[0]; // the staged write, not a transaction — commit() re-enters here with it
|
|
1437
1848
|
if (this.isCommitting) {
|
|
1438
1849
|
// if we are in the commit, do the save and force a reload so we get a read within the transaction
|
|
1439
|
-
super.save(
|
|
1850
|
+
super.save(operation, null, true);
|
|
1440
1851
|
}
|
|
1441
1852
|
else {
|
|
1442
1853
|
this.isCommitting = true;
|
|
1443
|
-
|
|
1854
|
+
// A synchronous throw from commit() (e.g. a 409 from an expired lock handle) would
|
|
1855
|
+
// otherwise leave isCommitting latched at true, causing every subsequent save() in this
|
|
1856
|
+
// context to take the fire-and-forget if-branch and silently drop writes.
|
|
1857
|
+
let commitResult;
|
|
1858
|
+
try {
|
|
1859
|
+
commitResult = this.commit();
|
|
1860
|
+
}
|
|
1861
|
+
catch (err) {
|
|
1862
|
+
this.isCommitting = false;
|
|
1863
|
+
throw err;
|
|
1864
|
+
}
|
|
1865
|
+
return (0, when_ts_1.when)(commitResult, () => {
|
|
1444
1866
|
this.isCommitting = false;
|
|
1867
|
+
}, (err) => {
|
|
1868
|
+
// Async rejection (e.g. a rejected rocksdb commit promise) must also clear the
|
|
1869
|
+
// latch; without this, every subsequent save() silently fire-and-forgets.
|
|
1870
|
+
this.isCommitting = false;
|
|
1871
|
+
throw err;
|
|
1445
1872
|
});
|
|
1446
1873
|
}
|
|
1447
1874
|
}
|
|
1875
|
+
// Without an explicit transaction() a live lock is released only by unlock() or its lease, never
|
|
1876
|
+
// by the per-write commit. Expired handles are pruned so a reused context does not retain every key.
|
|
1877
|
+
releaseRecordLocks() {
|
|
1878
|
+
const recordLocks = this.recordLocks;
|
|
1879
|
+
if (!recordLocks)
|
|
1880
|
+
return;
|
|
1881
|
+
for (const [store, storeMap] of recordLocks) {
|
|
1882
|
+
for (const [keyId, handle] of storeMap)
|
|
1883
|
+
if (handle.released)
|
|
1884
|
+
storeMap.delete(keyId);
|
|
1885
|
+
if (storeMap.size === 0)
|
|
1886
|
+
recordLocks.delete(store);
|
|
1887
|
+
}
|
|
1888
|
+
if (recordLocks.size === 0)
|
|
1889
|
+
this.recordLocks = undefined;
|
|
1890
|
+
}
|
|
1448
1891
|
// @ts-expect-error accessor overriding property
|
|
1449
1892
|
get timestamp() {
|
|
1450
1893
|
return this._timestamp || (this._timestamp = (0, commonUtility_ts_1.getNextMonotonicTime)());
|
|
@@ -1517,6 +1960,7 @@ function isJoinableScope(transaction) {
|
|
|
1517
1960
|
return transaction?.open === exports.TRANSACTION_STATE.OPEN && !transaction.saveCommits;
|
|
1518
1961
|
}
|
|
1519
1962
|
let timer;
|
|
1963
|
+
let monitorTick = 0;
|
|
1520
1964
|
/**
|
|
1521
1965
|
* True when a link other than `txn` in the same multi-store chain was written recently enough to
|
|
1522
1966
|
* still be active — i.e. its `writeTimeout` (set only by addWrite, see the field comment) hasn't
|
|
@@ -1536,22 +1980,90 @@ function chainStillActive(txn) {
|
|
|
1536
1980
|
}
|
|
1537
1981
|
return false;
|
|
1538
1982
|
}
|
|
1983
|
+
/**
|
|
1984
|
+
* Name a chain link still holding its native handle past the reporting threshold, and why the
|
|
1985
|
+
* branches below are not reaping it (harper#2471). Attribution only: it reads the recency clocks
|
|
1986
|
+
* those branches maintain rather than calling `chainStillActive`, which would decay `writeTimeout`
|
|
1987
|
+
* as a side effect, and it runs before them so it cannot reorder them.
|
|
1988
|
+
*/
|
|
1989
|
+
function reportLongLivedLink(link, thresholdMs, now, reportBudget) {
|
|
1990
|
+
if (!link.transaction || link.handleOpenedAt === 0)
|
|
1991
|
+
return;
|
|
1992
|
+
const ageMs = now - link.handleOpenedAt;
|
|
1993
|
+
if (ageMs < thresholdMs)
|
|
1994
|
+
return;
|
|
1995
|
+
const states = [];
|
|
1996
|
+
if (link.sourceApply)
|
|
1997
|
+
states.push('source-apply');
|
|
1998
|
+
if (link.isReplay)
|
|
1999
|
+
states.push('replay');
|
|
2000
|
+
if (link.committing)
|
|
2001
|
+
states.push('commit-phase');
|
|
2002
|
+
if (link.open === exports.TRANSACTION_STATE.CLOSED)
|
|
2003
|
+
states.push('closed-with-open-iterators');
|
|
2004
|
+
if (monitorTick - link.writeTick <= 1)
|
|
2005
|
+
states.push('active');
|
|
2006
|
+
if (states.length === 0)
|
|
2007
|
+
states.push('over-limit');
|
|
2008
|
+
(0, longLivedTransactions_ts_1.reportLongLivedHolder)({
|
|
2009
|
+
databasePath: link.db?.rootStore?.path,
|
|
2010
|
+
nativeId: link.transaction.id,
|
|
2011
|
+
openedAt: link.handleOpenedAt,
|
|
2012
|
+
ageMs,
|
|
2013
|
+
databaseName: link.db?.rootStore?.databaseName,
|
|
2014
|
+
tableName: link.db?.name,
|
|
2015
|
+
countPendingWrites: () => {
|
|
2016
|
+
let pendingWrites = 0;
|
|
2017
|
+
for (const write of link.writes)
|
|
2018
|
+
if (write)
|
|
2019
|
+
pendingWrites++;
|
|
2020
|
+
return pendingWrites;
|
|
2021
|
+
},
|
|
2022
|
+
states,
|
|
2023
|
+
timeoutBudget: link.timeoutBudget,
|
|
2024
|
+
startedFrom: link.startedFrom,
|
|
2025
|
+
}, reportBudget);
|
|
2026
|
+
}
|
|
2027
|
+
/**
|
|
2028
|
+
* Report every link of this logical transaction that holds its own native handle. A blind write to a
|
|
2029
|
+
* second database attaches a handle to a `.next` link while only the root is supervised, so reporting
|
|
2030
|
+
* the entry alone names an id the sweep's line can never be joined to when the child is the holder.
|
|
2031
|
+
* Links that read are in `trackedTxns` and get their own visit from the tick.
|
|
2032
|
+
*/
|
|
2033
|
+
function reportIfLongLived(txn, thresholdMs, now, reportBudget) {
|
|
2034
|
+
if (thresholdMs === 0)
|
|
2035
|
+
return;
|
|
2036
|
+
for (let link = txn; link; link = link.next) {
|
|
2037
|
+
if (link !== txn && trackedTxns.has(link))
|
|
2038
|
+
break;
|
|
2039
|
+
reportLongLivedLink(link, thresholdMs, now, reportBudget);
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
1539
2042
|
function startMonitoringTxns() {
|
|
1540
2043
|
timer = setInterval(function () {
|
|
2044
|
+
monitorTick++;
|
|
2045
|
+
const checkedCommitPhaseChains = new Set();
|
|
2046
|
+
const reportThresholdMs = (0, longLivedTransactions_ts_1.getReportThresholdMs)();
|
|
2047
|
+
(0, longLivedTransactions_ts_1.rebaseLongLivedTransactionReports)(reportThresholdMs);
|
|
2048
|
+
const reportBudget = (0, longLivedTransactions_ts_1.createLongLivedHolderReportBudget)(reportThresholdMs);
|
|
2049
|
+
const reportNow = performance.now();
|
|
1541
2050
|
// Both registries, in sequence rather than as a union: a root can be in each (it read, and a
|
|
1542
2051
|
// later link blind-wrote), and the membership check is cheaper than allocating per tick.
|
|
1543
2052
|
for (const txn of trackedTxns)
|
|
1544
|
-
monitorTransaction(txn);
|
|
2053
|
+
monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
|
|
1545
2054
|
for (const txn of supervisedWriteRoots)
|
|
1546
2055
|
if (!trackedTxns.has(txn))
|
|
1547
|
-
monitorTransaction(txn);
|
|
2056
|
+
monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
|
|
2057
|
+
(0, longLivedTransactions_ts_1.finishLongLivedHolderReports)(reportBudget);
|
|
1548
2058
|
}, txnExpiration).unref();
|
|
1549
|
-
function monitorTransaction(txn) {
|
|
2059
|
+
function monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget) {
|
|
1550
2060
|
if (txn.rangeReadActive) {
|
|
1551
2061
|
txn.rangeReadActive = false;
|
|
1552
2062
|
txn.renewReadTimeout();
|
|
1553
2063
|
}
|
|
2064
|
+
reportIfLongLived(txn, reportThresholdMs, reportNow, reportBudget);
|
|
1554
2065
|
{
|
|
2066
|
+
const commitChainHead = txn.commitChainHead ?? txn;
|
|
1555
2067
|
// Decay write recency once per tick for every tracked link, independent of the `timeout`
|
|
1556
2068
|
// branches below — a tracked link that keeps its own idle limit alive by reading must not
|
|
1557
2069
|
// thereby keep chainStillActive believing it was written recently too.
|
|
@@ -1577,6 +2089,21 @@ function startMonitoringTxns() {
|
|
|
1577
2089
|
harperLogger.warn?.(`Read iterators held a committed transaction's snapshot past the open-transaction limit; releasing it, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
|
|
1578
2090
|
txn.releaseReadTxn();
|
|
1579
2091
|
}
|
|
2092
|
+
else if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
|
|
2093
|
+
// Parked in commit()'s pre-commit await — a `before`/`beforeIntermediate` hook, in practice a
|
|
2094
|
+
// blob's durable file write, which for a multi-tens-of-MB payload legitimately outruns the
|
|
2095
|
+
// limit. The write set is sealed and the caller is awaiting this commit, so the limit's
|
|
2096
|
+
// premise (the application is holding a transaction open) does not hold, and poisoning here
|
|
2097
|
+
// would unlink the blobs the write still references (issue #2062). The grace is bounded
|
|
2098
|
+
// because the transaction still pins a read snapshot: a source that stalls rather than
|
|
2099
|
+
// finishing falls through to the abort below once it runs out. A canonical-source apply or
|
|
2100
|
+
// replay is spared for as long as it takes: neither aborting it (harper-pro#348) nor the
|
|
2101
|
+
// force-commit below — which would durably commit a record whose blob file is still being
|
|
2102
|
+
// written — is acceptable, and their blob sources are bounded by the receive-side idle
|
|
2103
|
+
// watchdog instead.
|
|
2104
|
+
harperLogger.warn?.(`Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work (e.g. a large blob write); letting it complete, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`, ...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : []));
|
|
2105
|
+
txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
|
|
2106
|
+
}
|
|
1580
2107
|
else if (txn.hasPendingWrites() && chainStillActive(txn)) {
|
|
1581
2108
|
// A later link in the chain was written recently (writes re-arm only the link that
|
|
1582
2109
|
// receives them, and a multi-store transaction can be writing database B while this
|
|
@@ -1595,7 +2122,7 @@ function startMonitoringTxns() {
|
|
|
1595
2122
|
// those, keep the prior force-commit behavior below.
|
|
1596
2123
|
harperLogger.error(`Transaction was open too long and has been aborted after exceeding the open-transaction limit, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`, ...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : []), ...(DEBUG_LONG_TXNS ? ['starting stack trace', txn.stackTraces] : []));
|
|
1597
2124
|
try {
|
|
1598
|
-
|
|
2125
|
+
commitChainHead.abortDueToTimeout();
|
|
1599
2126
|
}
|
|
1600
2127
|
catch (error) {
|
|
1601
2128
|
harperLogger.debug?.(`Error aborting timed out transaction: ${error.message}`);
|
|
@@ -1634,6 +2161,11 @@ startMonitoringTxns();
|
|
|
1634
2161
|
function resetReplayedWritesWarning() {
|
|
1635
2162
|
replayedWritesWarned = false;
|
|
1636
2163
|
}
|
|
2164
|
+
function setMaxOutstandingTxnDuration(ms) {
|
|
2165
|
+
const previous = MAX_OUTSTANDING_TXN_DURATION;
|
|
2166
|
+
MAX_OUTSTANDING_TXN_DURATION = ms;
|
|
2167
|
+
return previous;
|
|
2168
|
+
}
|
|
1637
2169
|
/** Test seam: whether the monitor supervises this logical transaction for its writes. */
|
|
1638
2170
|
function isWriteSupervised(txn) {
|
|
1639
2171
|
return supervisedWriteRoots.has(txn);
|