@harperfast/harper 5.2.10 → 5.3.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/mcpTools.ts +1 -1
- package/agent/session.ts +25 -14
- package/bin/cliOperations.ts +46 -9
- package/bin/copyDb.ts +282 -59
- package/bin/deploySetup.ts +16 -5
- package/bin/harper.ts +1 -1
- package/bin/help.ts +4 -1
- package/bin/lite.ts +4 -1
- package/bin/restart.ts +59 -3
- package/bin/run.ts +6 -11
- package/bin/upgrade.js +7 -3
- package/bin/workloadIdentity.ts +119 -0
- package/components/Application.ts +2414 -242
- package/components/ApplicationScope.ts +8 -0
- package/components/EntryHandler.ts +59 -39
- package/components/OptionsWatcher.ts +150 -74
- package/components/RuntimeModuleTracker.ts +38 -7
- package/components/Scope.ts +37 -15
- package/components/awaitRestart.ts +84 -0
- package/components/componentLoader.ts +340 -30
- package/components/componentPreparationLock.ts +16 -5
- package/components/mcp/adapters/harperHttp.ts +4 -0
- package/components/mcp/listChanged.ts +4 -0
- package/components/mcp/toolRegistry.ts +2 -0
- package/components/mcp/tools/operations.ts +9 -0
- package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
- package/components/operations.js +270 -78
- package/components/operationsValidation.js +49 -1
- package/components/status/ComponentStatusRegistry.ts +59 -0
- package/config/RootConfigWatcher.ts +80 -34
- package/config/configUtils.ts +291 -34
- package/config/harperConfigEnvVars.ts +170 -27
- package/config-root.schema.json +29 -0
- package/dataLayer/blobBackup.ts +160 -50
- package/dataLayer/delete.ts +6 -1
- package/dataLayer/harperBridge/ResourceBridge.ts +52 -8
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dataLayer/hdbInfoController.ts +34 -1
- package/dataLayer/insert.ts +44 -1
- package/dataLayer/rocksdbBackup.ts +53 -10
- package/dataLayer/schema.ts +11 -1
- package/dataLayer/schemaDescribe.ts +8 -1
- package/dist/agent/mcpTools.js +1 -1
- package/dist/agent/mcpTools.js.map +1 -1
- package/dist/agent/session.d.ts +22 -0
- package/dist/agent/session.js +26 -15
- package/dist/agent/session.js.map +1 -1
- package/dist/bin/cliOperations.js +48 -9
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/copyDb.d.ts +12 -1
- package/dist/bin/copyDb.js +248 -60
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/bin/deploySetup.d.ts +2 -0
- package/dist/bin/deploySetup.js +11 -3
- package/dist/bin/deploySetup.js.map +1 -1
- package/dist/bin/harper.js +1 -1
- package/dist/bin/harper.js.map +1 -1
- package/dist/bin/help.js +4 -1
- package/dist/bin/help.js.map +1 -1
- package/dist/bin/lite.js +4 -1
- package/dist/bin/lite.js.map +1 -1
- package/dist/bin/restart.js +54 -5
- package/dist/bin/restart.js.map +1 -1
- package/dist/bin/run.js +4 -10
- package/dist/bin/run.js.map +1 -1
- package/dist/bin/upgrade.js +4 -3
- package/dist/bin/upgrade.js.map +1 -1
- package/dist/bin/workloadIdentity.d.ts +18 -0
- package/dist/bin/workloadIdentity.js +100 -0
- package/dist/bin/workloadIdentity.js.map +1 -0
- package/dist/components/Application.d.ts +139 -16
- package/dist/components/Application.js +2215 -267
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +8 -0
- package/dist/components/ApplicationScope.js +7 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/EntryHandler.js +26 -10
- package/dist/components/EntryHandler.js.map +1 -1
- package/dist/components/OptionsWatcher.d.ts +1 -0
- package/dist/components/OptionsWatcher.js +141 -74
- package/dist/components/OptionsWatcher.js.map +1 -1
- package/dist/components/RuntimeModuleTracker.js +40 -6
- package/dist/components/RuntimeModuleTracker.js.map +1 -1
- package/dist/components/Scope.js +38 -13
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/awaitRestart.d.ts +33 -0
- package/dist/components/awaitRestart.js +61 -0
- package/dist/components/awaitRestart.js.map +1 -0
- package/dist/components/componentLoader.d.ts +38 -1
- package/dist/components/componentLoader.js +279 -22
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentPreparationLock.d.ts +5 -0
- package/dist/components/componentPreparationLock.js +14 -6
- package/dist/components/componentPreparationLock.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.js +4 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/listChanged.js +5 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +1 -0
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +5 -0
- package/dist/components/mcp/tools/operations.js +9 -0
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/operations.js +231 -77
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.js +49 -1
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
- package/dist/components/status/ComponentStatusRegistry.js +63 -0
- package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
- package/dist/config/RootConfigWatcher.d.ts +7 -1
- package/dist/config/RootConfigWatcher.js +64 -27
- package/dist/config/RootConfigWatcher.js.map +1 -1
- package/dist/config/configUtils.d.ts +9 -1
- package/dist/config/configUtils.js +254 -33
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/harperConfigEnvVars.d.ts +16 -0
- package/dist/config/harperConfigEnvVars.js +162 -25
- package/dist/config/harperConfigEnvVars.js.map +1 -1
- package/dist/dataLayer/blobBackup.d.ts +49 -20
- package/dist/dataLayer/blobBackup.js +139 -50
- package/dist/dataLayer/blobBackup.js.map +1 -1
- package/dist/dataLayer/delete.js +1 -1
- package/dist/dataLayer/delete.js.map +1 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.js +51 -10
- package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
- package/dist/dataLayer/hdbInfoController.d.ts +10 -0
- package/dist/dataLayer/hdbInfoController.js +30 -1
- package/dist/dataLayer/hdbInfoController.js.map +1 -1
- package/dist/dataLayer/insert.d.ts +9 -1
- package/dist/dataLayer/insert.js +30 -0
- package/dist/dataLayer/insert.js.map +1 -1
- package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
- package/dist/dataLayer/rocksdbBackup.js +45 -8
- package/dist/dataLayer/rocksdbBackup.js.map +1 -1
- package/dist/dataLayer/schema.js +8 -0
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +8 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +52 -0
- package/dist/resources/DatabaseTransaction.d.ts +55 -3
- package/dist/resources/DatabaseTransaction.js +460 -134
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.js +22 -4
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
- package/dist/resources/PrimaryRocksDatabase.js +30 -2
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
- package/dist/resources/RecordEncoder.d.ts +20 -0
- package/dist/resources/RecordEncoder.js +94 -9
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +2 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.js +20 -11
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +20 -1
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/RocksIndexStore.d.ts +6 -1
- package/dist/resources/RocksIndexStore.js +24 -10
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.d.ts +14 -1
- package/dist/resources/RocksTransactionLogStore.js +57 -17
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.d.ts +112 -8
- package/dist/resources/Table.js +1071 -197
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/auditStore.d.ts +11 -2
- package/dist/resources/auditStore.js +183 -18
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/blob.d.ts +129 -9
- package/dist/resources/blob.js +992 -114
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/branchDatabase.d.ts +48 -0
- package/dist/resources/branchDatabase.js +892 -0
- package/dist/resources/branchDatabase.js.map +1 -0
- package/dist/resources/crdt.js +50 -12
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/databases.d.ts +150 -9
- package/dist/resources/databases.js +1247 -525
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/defineTable.d.ts +10 -2
- package/dist/resources/defineTable.js +9 -1
- package/dist/resources/defineTable.js.map +1 -1
- package/dist/resources/graphql.d.ts +1 -1
- package/dist/resources/graphql.js +50 -15
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +1 -0
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +6 -2
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/longLivedTransactions.d.ts +71 -0
- package/dist/resources/longLivedTransactions.js +358 -0
- package/dist/resources/longLivedTransactions.js.map +1 -0
- package/dist/resources/models/backendRegistry.d.ts +26 -0
- package/dist/resources/models/backendRegistry.js +60 -2
- package/dist/resources/models/backendRegistry.js.map +1 -1
- package/dist/resources/models/bootstrap.d.ts +33 -1
- package/dist/resources/models/bootstrap.js +416 -31
- package/dist/resources/models/bootstrap.js.map +1 -1
- package/dist/resources/recordLock.d.ts +80 -0
- package/dist/resources/recordLock.js +184 -0
- package/dist/resources/recordLock.js.map +1 -0
- package/dist/resources/replayLogs.d.ts +14 -1
- package/dist/resources/replayLogs.js +152 -24
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/replayLogsGuards.d.ts +94 -7
- package/dist/resources/replayLogsGuards.js +111 -7
- package/dist/resources/replayLogsGuards.js.map +1 -1
- package/dist/resources/search.js +158 -32
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transactionBroadcast.d.ts +1 -1
- package/dist/resources/transactionBroadcast.js +2 -2
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/auth.js +61 -30
- package/dist/security/auth.js.map +1 -1
- package/dist/security/authn/oidc/claims.d.ts +22 -0
- package/dist/security/authn/oidc/claims.js +71 -0
- package/dist/security/authn/oidc/claims.js.map +1 -0
- package/dist/security/authn/oidc/identityToken.d.ts +27 -0
- package/dist/security/authn/oidc/identityToken.js +111 -0
- package/dist/security/authn/oidc/identityToken.js.map +1 -0
- package/dist/security/authn/oidc/jwks.d.ts +25 -0
- package/dist/security/authn/oidc/jwks.js +261 -0
- package/dist/security/authn/oidc/jwks.js.map +1 -0
- package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
- package/dist/security/authn/oidc/providers/generic.js +34 -0
- package/dist/security/authn/oidc/providers/generic.js.map +1 -0
- package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
- package/dist/security/authn/oidc/providers/githubActions.js +129 -0
- package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
- package/dist/security/authn/oidc/providers/index.d.ts +37 -0
- package/dist/security/authn/oidc/providers/index.js +24 -0
- package/dist/security/authn/oidc/providers/index.js.map +1 -0
- package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
- package/dist/security/authn/oidc/tokenExchange.js +306 -0
- package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
- package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
- package/dist/security/authn/oidc/types.d.ts +38 -0
- package/dist/security/authn/oidc/types.js +6 -0
- package/dist/security/authn/oidc/types.js.map +1 -0
- package/dist/security/certificateVerification/index.js +40 -11
- package/dist/security/certificateVerification/index.js.map +1 -1
- package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
- package/dist/security/certificateVerification/trustedIssuers.js +79 -0
- package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
- package/dist/security/certificateVerification/types.d.ts +1 -0
- package/dist/security/credentialProvenance.d.ts +35 -0
- package/dist/security/credentialProvenance.js +51 -0
- package/dist/security/credentialProvenance.js.map +1 -0
- package/dist/security/credentialRejection.d.ts +4 -0
- package/dist/security/credentialRejection.js +24 -0
- package/dist/security/credentialRejection.js.map +1 -0
- package/dist/security/deferredAuthentication.d.ts +36 -0
- package/dist/security/deferredAuthentication.js +70 -0
- package/dist/security/deferredAuthentication.js.map +1 -0
- package/dist/security/impersonation.d.ts +21 -0
- package/dist/security/impersonation.js +108 -9
- package/dist/security/impersonation.js.map +1 -1
- package/dist/security/jsLoader.d.ts +6 -0
- package/dist/security/jsLoader.js +75 -15
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +301 -71
- package/dist/security/keys.js.map +1 -1
- package/dist/security/operationScope.d.ts +21 -0
- package/dist/security/operationScope.js +36 -0
- package/dist/security/operationScope.js.map +1 -0
- package/dist/security/permissionsTranslator.js +21 -0
- package/dist/security/permissionsTranslator.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +19 -1
- package/dist/security/tokenAuthentication.js +191 -10
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/security/user.js +4 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
- package/dist/server/DurableSubscriptionsSession.js +67 -10
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +73 -0
- package/dist/server/REST.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -0
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +11 -0
- package/dist/server/http.js +88 -15
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +8 -1
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/jobs/jobProcess.js +6 -2
- package/dist/server/jobs/jobProcess.js.map +1 -1
- package/dist/server/jobs/jobs.js +4 -1
- package/dist/server/jobs/jobs.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +26 -4
- package/dist/server/liveSubscriptionAuth.js +105 -39
- package/dist/server/liveSubscriptionAuth.js.map +1 -1
- package/dist/server/loadRootComponents.js +49 -10
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/mqtt.d.ts +2 -0
- package/dist/server/mqtt.js +165 -30
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/nodeName.d.ts +2 -0
- package/dist/server/nodeName.js +107 -23
- package/dist/server/nodeName.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +25 -0
- package/dist/server/serverHelpers/Headers.js +139 -1
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.d.ts +9 -0
- package/dist/server/serverHelpers/contentTypes.js +32 -25
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
- package/dist/server/serverHelpers/deployValidationState.js +9 -19
- package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
- package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
- package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
- package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
- package/dist/server/serverHelpers/registeredOperations.js +74 -21
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
- package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
- package/dist/server/serverHelpers/requestSanitization.js +20 -0
- package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +6 -3
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
- package/dist/server/serverHelpers/serverUtilities.js +90 -20
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
- package/dist/server/serverHelpers/uwsServer.js +19 -1
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/static.js +24 -28
- package/dist/server/static.js.map +1 -1
- package/dist/server/storageReclamation.d.ts +5 -0
- package/dist/server/storageReclamation.js +16 -0
- package/dist/server/storageReclamation.js.map +1 -1
- package/dist/server/threads/itc.d.ts +7 -2
- package/dist/server/threads/itc.js +5 -1
- package/dist/server/threads/itc.js.map +1 -1
- package/dist/server/threads/manageThreads.d.ts +26 -4
- package/dist/server/threads/manageThreads.js +517 -72
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.js +89 -26
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadHeapMemory.d.ts +2 -0
- package/dist/server/threads/threadHeapMemory.js +31 -0
- package/dist/server/threads/threadHeapMemory.js.map +1 -0
- package/dist/server/threads/threadServer.js +46 -15
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/sqlEngine/config.d.ts +1 -3
- package/dist/sqlEngine/config.js +19 -16
- package/dist/sqlEngine/config.js.map +1 -1
- package/dist/sqlTranslator/index.d.ts +1 -1
- package/dist/sqlTranslator/index.js +30 -7
- package/dist/sqlTranslator/index.js.map +1 -1
- package/dist/upgrade/directives/5-3-0.d.ts +7 -0
- package/dist/upgrade/directives/5-3-0.js +148 -0
- package/dist/upgrade/directives/5-3-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/OperationFunctionCaller.js +2 -1
- package/dist/utility/OperationFunctionCaller.js.map +1 -1
- package/dist/utility/common_utils.d.ts +16 -0
- package/dist/utility/common_utils.js +32 -6
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/componentNames.d.ts +8 -0
- package/dist/utility/componentNames.js +12 -1
- package/dist/utility/componentNames.js.map +1 -1
- package/dist/utility/environment/environmentManager.js +3 -6
- package/dist/utility/environment/environmentManager.js.map +1 -1
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/environment/systemInformation.js +1 -0
- package/dist/utility/environment/systemInformation.js.map +1 -1
- package/dist/utility/errors/commonErrors.d.ts +2 -0
- package/dist/utility/errors/commonErrors.js +2 -0
- package/dist/utility/errors/commonErrors.js.map +1 -1
- package/dist/utility/errors/hdbError.d.ts +17 -0
- package/dist/utility/errors/hdbError.js +30 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/globalSchema.d.ts +18 -0
- package/dist/utility/hdbTerms.d.ts +16 -0
- package/dist/utility/hdbTerms.js +18 -2
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +2 -0
- package/dist/utility/logging/harper_logger.js +91 -14
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logRotator.js +76 -49
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/nodeIdentity.d.ts +9 -0
- package/dist/utility/nodeIdentity.js +58 -0
- package/dist/utility/nodeIdentity.js.map +1 -0
- package/dist/utility/npmUtilities.js +9 -7
- package/dist/utility/npmUtilities.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +3 -1
- package/dist/utility/operationPermissions.js +16 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +10 -7
- package/dist/utility/operation_authorization.js +212 -41
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/watchPath.d.ts +29 -0
- package/dist/utility/watchPath.js +68 -0
- package/dist/utility/watchPath.js.map +1 -0
- package/dist/utility/watcherFallback.d.ts +86 -0
- package/dist/utility/watcherFallback.js +278 -1
- package/dist/utility/watcherFallback.js.map +1 -1
- package/dist/validation/configValidator.d.ts +12 -0
- package/dist/validation/configValidator.js +199 -72
- package/dist/validation/configValidator.js.map +1 -1
- package/dist/validation/installValidator.js +12 -0
- package/dist/validation/installValidator.js.map +1 -1
- package/dist/validation/validationWrapper.d.ts +11 -0
- package/dist/validation/validationWrapper.js +16 -3
- package/dist/validation/validationWrapper.js.map +1 -1
- package/index.ts +2 -0
- package/json/systemSchema.json +52 -0
- package/npm-shrinkwrap.json +235 -233
- package/package.json +8 -7
- package/resources/DESIGN.md +114 -52
- package/resources/DatabaseTransaction.ts +522 -130
- package/resources/LMDBTransaction.ts +26 -4
- package/resources/PrimaryRocksDatabase.ts +29 -3
- package/resources/RecordEncoder.ts +103 -8
- package/resources/RequestTarget.ts +2 -0
- package/resources/Resource.ts +17 -9
- package/resources/ResourceInterface.ts +23 -0
- package/resources/RocksIndexStore.ts +30 -10
- package/resources/RocksTransactionLogStore.ts +78 -21
- package/resources/Table.ts +1107 -164
- package/resources/auditStore.ts +187 -21
- package/resources/blob.ts +1029 -110
- package/resources/branchDatabase.ts +941 -0
- package/resources/crdt.ts +76 -12
- package/resources/databases.ts +1354 -522
- package/resources/defineTable.ts +18 -2
- package/resources/graphql.ts +70 -16
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +6 -2
- package/resources/longLivedTransactions.ts +360 -0
- package/resources/models/backendRegistry.ts +84 -2
- package/resources/models/bootstrap.ts +473 -28
- package/resources/recordLock.ts +253 -0
- package/resources/replayLogs.ts +147 -26
- package/resources/replayLogsGuards.ts +171 -8
- package/resources/search.ts +154 -32
- package/resources/transactionBroadcast.ts +3 -3
- package/security/auth.ts +68 -29
- package/security/authn/oidc/claims.ts +72 -0
- package/security/authn/oidc/identityToken.ts +129 -0
- package/security/authn/oidc/jwks.ts +260 -0
- package/security/authn/oidc/providers/generic.ts +40 -0
- package/security/authn/oidc/providers/githubActions.ts +137 -0
- package/security/authn/oidc/providers/index.ts +52 -0
- package/security/authn/oidc/tokenExchange.ts +300 -0
- package/security/authn/oidc/trustPolicyOperations.ts +343 -0
- package/security/authn/oidc/types.ts +41 -0
- package/security/certificateVerification/index.ts +54 -13
- package/security/certificateVerification/trustedIssuers.ts +76 -0
- package/security/certificateVerification/types.ts +1 -0
- package/security/credentialProvenance.ts +47 -0
- package/security/credentialRejection.ts +22 -0
- package/security/deferredAuthentication.ts +71 -0
- package/security/impersonation.ts +117 -12
- package/security/jsLoader.ts +81 -18
- package/security/keys.ts +298 -72
- package/security/operationScope.ts +33 -0
- package/security/permissionsTranslator.js +23 -0
- package/security/tokenAuthentication.ts +233 -12
- package/security/user.ts +4 -3
- package/server/DESIGN.md +183 -16
- package/server/DurableSubscriptionsSession.ts +71 -11
- package/server/REST.ts +79 -1
- package/server/graphqlQuerying.ts +4 -0
- package/server/http.ts +99 -18
- package/server/itc/serverHandlers.js +8 -1
- package/server/jobs/jobProcess.ts +8 -2
- package/server/jobs/jobs.ts +4 -1
- package/server/liveSubscriptionAuth.ts +129 -46
- package/server/loadRootComponents.js +50 -8
- package/server/mqtt.ts +179 -38
- package/server/nodeName.ts +103 -21
- package/server/serverHelpers/Headers.ts +131 -0
- package/server/serverHelpers/contentTypes.ts +29 -21
- package/server/serverHelpers/deployValidationState.ts +24 -13
- package/server/serverHelpers/operationAuthorizationState.ts +34 -3
- package/server/serverHelpers/registeredOperations.ts +79 -22
- package/server/serverHelpers/requestSanitization.ts +15 -0
- package/server/serverHelpers/serverHandlers.js +6 -3
- package/server/serverHelpers/serverUtilities.ts +104 -31
- package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
- package/server/serverHelpers/uwsServer.ts +17 -2
- package/server/static.ts +23 -29
- package/server/storageReclamation.ts +13 -0
- package/server/threads/itc.js +11 -1
- package/server/threads/manageThreads.js +526 -63
- package/server/threads/socketRouter.ts +81 -26
- package/server/threads/threadHeapMemory.ts +26 -0
- package/server/threads/threadServer.js +43 -15
- package/sqlTranslator/index.ts +31 -8
- package/studio/web/assets/{Chat-Br06zdMA.js → Chat-BnCBegQz.js} +1 -1
- package/studio/web/assets/{FloatingChat-BWImX5fA.js → FloatingChat-CoDW1ySS.js} +3 -3
- package/studio/web/assets/{apiToken-DN0nmDsq.js → apiToken-Bwk5BLXW.js} +1 -1
- package/studio/web/assets/{applications-kSxVoyeU.js → applications-DHxGi7JH.js} +1 -1
- package/studio/web/assets/{cssMode-C1JeufH5.js → cssMode-s0cWI-_M.js} +1 -1
- package/studio/web/assets/{editor-qoo9CrGO.js → editor-DNcRHK54.js} +1 -1
- package/studio/web/assets/{html-Dt4IIy04.js → html-Bdssedlg.js} +1 -1
- package/studio/web/assets/{htmlMode-DXgKKr4C.js → htmlMode-CoDlJ3fw.js} +1 -1
- package/studio/web/assets/{index-6onkYFOG.js → index-D6sxmFLR.js} +5 -5
- package/studio/web/assets/{index.lazy-BrCFnpNJ.js → index.lazy-tmU5BS8s.js} +1 -1
- package/studio/web/assets/{javascript-DNCQGUBc.js → javascript-B8meVSTH.js} +1 -1
- package/studio/web/assets/{jsonMode-CR6HWruP.js → jsonMode-DpIPd35T.js} +1 -1
- package/studio/web/assets/{languageServices-BM4fI4rS.js → languageServices-C_5FMJzQ.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-DSa1ttcD.js → lspLanguageFeatures-BIzNBzPK.js} +1 -1
- package/studio/web/assets/{notifications-BHXLnh6x.js → notifications-CQf18QKb.js} +1 -1
- package/studio/web/assets/{notifications-CUmtIA6z.js → notifications-CvZivSbh.js} +1 -1
- package/studio/web/assets/{profile-8BeFSF3j.js → profile-DdOwtntb.js} +1 -1
- package/studio/web/assets/{regions-C8qR0HhD.js → regions-n69fwagr.js} +1 -1
- package/studio/web/assets/{register-B4n5i0SD.js → register-PfWTCXWB.js} +2 -2
- package/studio/web/assets/{setComponentFile-g0_B0lgX.js → setComponentFile-Bg6O7X0S.js} +1 -1
- package/studio/web/assets/{setup-B0CTj_Q5.js → setup-CUx_aUDl.js} +2 -2
- package/studio/web/assets/{status-C6Yib7-K.js → status-D7BVKqX9.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-Dgta7bVi.js → toggleHighContrast-DBSyXzMr.js} +1 -1
- package/studio/web/assets/{tsMode-CH_jHvU-.js → tsMode-BByKCjBS.js} +1 -1
- package/studio/web/assets/{typescript-Co9LCXd5.js → typescript-DDLnLpw9.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-wKC8NsC_.js → useEntityRestURL-31CHGaHk.js} +1 -1
- package/studio/web/assets/{workers-CWeLxCXA.js → workers-pR3jRY9D.js} +1 -1
- package/studio/web/assets/{xml-BadC-0Rk.js → xml-2iRnMhQO.js} +1 -1
- package/studio/web/assets/{yaml-BiUfxPbC.js → yaml-Bf92gJpd.js} +1 -1
- package/studio/web/index.html +1 -1
- package/upgrade/directives/5-3-0.ts +132 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/OperationFunctionCaller.ts +2 -1
- package/utility/common_utils.ts +30 -5
- package/utility/componentNames.ts +12 -0
- package/utility/environment/environmentManager.ts +3 -7
- package/utility/environment/systemInformation.ts +7 -0
- package/utility/errors/commonErrors.ts +4 -0
- package/utility/errors/hdbError.ts +29 -0
- package/utility/hdbTerms.ts +17 -0
- package/utility/logging/harper_logger.ts +87 -14
- package/utility/logging/logRotator.ts +72 -45
- package/utility/nodeIdentity.ts +45 -0
- package/utility/npmUtilities.ts +10 -8
- package/utility/operationPermissions.ts +18 -1
- package/utility/operation_authorization.ts +231 -42
- package/utility/watchPath.ts +63 -0
- package/utility/watcherFallback.ts +270 -0
- package/validation/configValidator.ts +209 -72
- package/validation/installValidator.ts +15 -0
- package/validation/validationWrapper.ts +18 -4
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { cleanupUnusedBlobs, collectRetainedFileIds } from './blob.ts';
|
|
2
2
|
import { Transaction as LMDBTransaction } from 'lmdb';
|
|
3
3
|
import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
|
|
4
|
-
import { ServerError } from '../utility/errors/hdbError.ts';
|
|
4
|
+
import { ServerError, TransactionCommitConflictTimeoutError } from '../utility/errors/hdbError.ts';
|
|
5
|
+
import { lockNotHeldError, type RecordLockHandle } from './recordLock.ts';
|
|
5
6
|
import * as harperLogger from '../utility/logging/harper_logger.ts';
|
|
6
7
|
import type { Context, Id } from './ResourceInterface.ts';
|
|
7
8
|
import * as envMngr from '../utility/environment/environmentManager.ts';
|
|
@@ -14,9 +15,17 @@ const RETRY_NOW_VALUE = constants.RETRY_NOW_VALUE;
|
|
|
14
15
|
import type { RootDatabaseKind } from './databases.ts';
|
|
15
16
|
import type { Entry } from './RecordEncoder.ts';
|
|
16
17
|
import { toBufferKey } from 'ordered-binary';
|
|
18
|
+
import {
|
|
19
|
+
createLongLivedHolderReportBudget,
|
|
20
|
+
describeHolderCandidates,
|
|
21
|
+
finishLongLivedHolderReports,
|
|
22
|
+
getReportThresholdMs,
|
|
23
|
+
rebaseLongLivedTransactionReports,
|
|
24
|
+
reportLongLivedHolder,
|
|
25
|
+
type LongLivedHolderReportBudget,
|
|
26
|
+
} from './longLivedTransactions.ts';
|
|
17
27
|
|
|
18
28
|
const trackedTxns = new Set<DatabaseTransaction>();
|
|
19
|
-
const readTransactionOwners = new WeakMap<ReadTransaction, DatabaseTransaction>();
|
|
20
29
|
// Read options for a rotated generation's native transactions; shared because they never vary.
|
|
21
30
|
const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
|
|
22
31
|
// Logical transactions the monitor supervises for their WRITES, kept apart from trackedTxns because the
|
|
@@ -24,7 +33,7 @@ const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
|
|
|
24
33
|
// is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the
|
|
25
34
|
// chain root — so a chain child can never become its own timeout root (issue #2231).
|
|
26
35
|
const supervisedWriteRoots = new Set<DatabaseTransaction>();
|
|
27
|
-
|
|
36
|
+
let MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them
|
|
28
37
|
const DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS);
|
|
29
38
|
export const TRANSACTION_STATE = {
|
|
30
39
|
CLOSED: 0, // the transaction has been committed or aborted and can no longer be used for writes (if read txn is active, it can be used for reads)
|
|
@@ -36,6 +45,12 @@ export const TRANSACTION_STATE = {
|
|
|
36
45
|
LINGERING: 2,
|
|
37
46
|
};
|
|
38
47
|
const MAX_RETRIES = 40;
|
|
48
|
+
// Over-limit monitor ticks a transaction parked in its commit phase is spared before it is aborted
|
|
49
|
+
// like any other over-time transaction. Sparing re-arms `timeout`, so each of these costs two
|
|
50
|
+
// monitor intervals — roughly 10 minutes at the 30s default. Generous, so a legitimately large blob
|
|
51
|
+
// write finishes, but bounded, so a pre-commit source that stalls instead of finishing can't pin its
|
|
52
|
+
// read snapshot forever (issue #2062).
|
|
53
|
+
export const COMMIT_PHASE_GRACE = 10;
|
|
39
54
|
// Cap the per-retry backoff so replication-applied transactions, which retry conflicts without a
|
|
40
55
|
// cap (see the commit rejection handler), don't grow the delay unbounded.
|
|
41
56
|
const MAX_RETRY_DELAY_MS = 1000;
|
|
@@ -69,7 +84,33 @@ let outstandingCommitCount = 0;
|
|
|
69
84
|
// the whole thread, regardless of how many distinct commits individually cross the threshold — see
|
|
70
85
|
// the comment at the log site for why a per-commit-only dedup isn't enough under sustained overload.
|
|
71
86
|
const OVERLOAD_LOG_MIN_INTERVAL_MS = 1000;
|
|
72
|
-
|
|
87
|
+
// One cooldown per reporting site, not one shared: `shed` fires on every bystander write during a
|
|
88
|
+
// wedge and would starve `abandon`, which names a different transaction and is the only server-side
|
|
89
|
+
// record of why a request was failed. Mutates only when it grants, so a caller with its own
|
|
90
|
+
// suppression as well (checkOverloaded's per-node `logged`) must test that first.
|
|
91
|
+
const lastStuckCommitLogAt = { shed: -Infinity, abandon: -Infinity };
|
|
92
|
+
function allowStuckCommitLog(site: 'shed' | 'abandon', now: number): boolean {
|
|
93
|
+
if (now - lastStuckCommitLogAt[site] <= OVERLOAD_LOG_MIN_INTERVAL_MS) return false;
|
|
94
|
+
lastStuckCommitLogAt[site] = now;
|
|
95
|
+
return true;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Which database/table, which native transaction, and which request a stuck commit belongs to —
|
|
99
|
+
// without it a wedge gives no indication of what to investigate (harper#2001).
|
|
100
|
+
function describeCommitIdentity(
|
|
101
|
+
store: any,
|
|
102
|
+
startedFrom: { resourceName: string; method: string } | undefined,
|
|
103
|
+
nativeTransaction: any
|
|
104
|
+
): string {
|
|
105
|
+
const nativeTransactionId = nativeTransaction?.id;
|
|
106
|
+
return (
|
|
107
|
+
`from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
|
|
108
|
+
(nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
|
|
109
|
+
(startedFrom?.resourceName
|
|
110
|
+
? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
|
|
111
|
+
: '')
|
|
112
|
+
);
|
|
113
|
+
}
|
|
73
114
|
|
|
74
115
|
// Track a submitted commit until it settles. Every attempt is tracked unconditionally: a
|
|
75
116
|
// coordinated retry round and a chained second-store commit are both issued from inside the
|
|
@@ -146,6 +187,20 @@ export function getOutstandingCommits(): { count: number; oldestAgeMs: number |
|
|
|
146
187
|
// about the caller's pattern, not the individual commit.
|
|
147
188
|
let replayedWritesWarned = false;
|
|
148
189
|
|
|
190
|
+
/**
|
|
191
|
+
* Abort a detached native handle. RocksTransaction.abort() throws on one that was already
|
|
192
|
+
* committed or aborted, and every caller is a cleanup path whose own callers have no handler — a
|
|
193
|
+
* throw there would abandon the rest of the cleanup.
|
|
194
|
+
*/
|
|
195
|
+
function abortNativeTransaction(transaction: RocksTransaction | null | undefined, context: string): void {
|
|
196
|
+
if (transaction == null) return;
|
|
197
|
+
try {
|
|
198
|
+
transaction.abort();
|
|
199
|
+
} catch (error) {
|
|
200
|
+
harperLogger.debug?.(context, error);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
149
204
|
// The analytics module registers a recorder here at load (dependency inversion, mirroring
|
|
150
205
|
// `replicationConfirmation` below) so the storage layer doesn't statically import the analytics/server
|
|
151
206
|
// modules. Unset until analytics loads, and when analytics is disabled the recorder call is cheap.
|
|
@@ -237,76 +292,6 @@ export function transactionOpenTooLongError(): ServerError {
|
|
|
237
292
|
);
|
|
238
293
|
}
|
|
239
294
|
|
|
240
|
-
class ReadSnapshotExpiredError extends ServerError {
|
|
241
|
-
constructor() {
|
|
242
|
-
super('Read scan snapshot expired; retry the read without replaying previously committed writes', 503);
|
|
243
|
-
this.name = 'ReadSnapshotExpiredError';
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
export function trackReadRange(transaction: ReadTransaction, createRange: () => any): any {
|
|
248
|
-
const owner = readTransactionOwners.get(transaction);
|
|
249
|
-
if (!owner) return createRange();
|
|
250
|
-
function checkActive() {
|
|
251
|
-
if (owner.timedOut) throw transactionOpenTooLongError();
|
|
252
|
-
if (owner.transaction !== transaction) {
|
|
253
|
-
throw new ReadSnapshotExpiredError();
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
checkActive();
|
|
257
|
-
const range = createRange();
|
|
258
|
-
const iterate = range.iterate;
|
|
259
|
-
range.iterate = function (options) {
|
|
260
|
-
const iterator = iterate.call(this, options);
|
|
261
|
-
let done = false;
|
|
262
|
-
// Closing the underlying iterator is the one step here that can throw for a reason the caller
|
|
263
|
-
// must not see: `next()` only reaches it once the snapshot is already gone, which is the
|
|
264
|
-
// likeliest moment for the native layer to object, and an error from cleanup would replace the
|
|
265
|
-
// named 503 with exactly the raw iterator error this wrapper exists to stop surfacing. The
|
|
266
|
-
// closure reference rather than `this` so a destructured `next` still cleans up.
|
|
267
|
-
const wrapper = {
|
|
268
|
-
[Symbol.iterator]() {
|
|
269
|
-
return this;
|
|
270
|
-
},
|
|
271
|
-
next() {
|
|
272
|
-
if (done) return { done: true, value: undefined };
|
|
273
|
-
try {
|
|
274
|
-
checkActive();
|
|
275
|
-
owner.rangeReadActive = true;
|
|
276
|
-
const result = iterator.next();
|
|
277
|
-
done = result.done === true;
|
|
278
|
-
return result;
|
|
279
|
-
} catch (error) {
|
|
280
|
-
closeQuietly();
|
|
281
|
-
throw error;
|
|
282
|
-
}
|
|
283
|
-
},
|
|
284
|
-
return(value?: any) {
|
|
285
|
-
if (!done) {
|
|
286
|
-
done = true;
|
|
287
|
-
iterator.return?.(value);
|
|
288
|
-
}
|
|
289
|
-
return { done: true, value };
|
|
290
|
-
},
|
|
291
|
-
throw(error) {
|
|
292
|
-
// Not delegated to `iterator.throw`: it closes and rethrows the same error anyway, and
|
|
293
|
-
// delegating after the close below would run it against an iterator already closed.
|
|
294
|
-
closeQuietly();
|
|
295
|
-
throw error;
|
|
296
|
-
},
|
|
297
|
-
};
|
|
298
|
-
function closeQuietly() {
|
|
299
|
-
try {
|
|
300
|
-
wrapper.return();
|
|
301
|
-
} catch {
|
|
302
|
-
// the error being propagated is the actionable one
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
return wrapper;
|
|
306
|
-
};
|
|
307
|
-
return range;
|
|
308
|
-
}
|
|
309
|
-
|
|
310
295
|
type MaybePromise<T> = T | Promise<T>;
|
|
311
296
|
|
|
312
297
|
export type CommitOptions = {
|
|
@@ -336,6 +321,9 @@ export type TransactionWrite = {
|
|
|
336
321
|
// overload accounting, the replay marker and a no-op write's removal all belong to the committer.
|
|
337
322
|
validate?: (txnTime: number, committedBy: DatabaseTransaction) => void;
|
|
338
323
|
fullUpdate?: boolean;
|
|
324
|
+
// The origin record version carried by an applied or replayed write. Bound it by the origin's
|
|
325
|
+
// transaction-log key so malformed or historical overloaded values cannot move ordering past the write.
|
|
326
|
+
recordVersion?: number;
|
|
339
327
|
saved?: boolean;
|
|
340
328
|
deferSave?: boolean;
|
|
341
329
|
skipReplicationConfirmation?: boolean;
|
|
@@ -374,8 +362,31 @@ export type TransactionWrite = {
|
|
|
374
362
|
// this write appended an audit entry, which references its saved blobs — they then belong to the
|
|
375
363
|
// audit trail (audit pruning deletes them), so the superseded-write cleanup must leave them alone
|
|
376
364
|
blobsAuditReferenced?: boolean;
|
|
365
|
+
// Lock handle set by Table._writeUpdate when the write is staged through a held record lock;
|
|
366
|
+
// used in DatabaseTransaction.save() to assign and track that handle's write versions.
|
|
367
|
+
lockHandle?: RecordLockHandle;
|
|
368
|
+
// Per-operation holder version, set once on first save() and reused on retry so that
|
|
369
|
+
// ImmediateTransaction's sequential immediateCommit saves don't collide: each write carries its
|
|
370
|
+
// own stamp independently of this.timestamp, which may not reset to 0 between saves.
|
|
371
|
+
lockStamp?: number;
|
|
372
|
+
// Version staged by a write that actually changed the record this round. It advances any
|
|
373
|
+
// same-key held handle's floor only after the native transaction commits successfully.
|
|
374
|
+
appliedRecordVersion?: number;
|
|
375
|
+
// Present only while a transaction owns record locks, keeping bookkeeping off ordinary writes.
|
|
376
|
+
trackRecordVersion?: boolean;
|
|
377
|
+
// Set by a table commit handler only when this retry round staged a record change.
|
|
378
|
+
recordVersionApplied?: boolean;
|
|
379
|
+
// Set by DatabaseTransaction.save() on the immediateCommit path: the Promise returned by the
|
|
380
|
+
// inner this.commit({ transaction }) call. The ImmediateTransaction outer commit resolves before
|
|
381
|
+
// this settles (fire-and-forget from the if-branch), so Table.save()'s lock-writable path awaits
|
|
382
|
+
// it to ensure the write is durable before resolving to the caller.
|
|
383
|
+
innerCommit?: MaybePromise<CommitResolution>;
|
|
377
384
|
};
|
|
378
385
|
|
|
386
|
+
export function getAppliedWriteVersion(recordVersion: number | undefined, txnLogKey: number): number {
|
|
387
|
+
return recordVersion == null ? txnLogKey : Math.min(recordVersion, txnLogKey);
|
|
388
|
+
}
|
|
389
|
+
|
|
379
390
|
/**
|
|
380
391
|
* The state a preceding write in this transaction left for `operation`'s key, or undefined if this
|
|
381
392
|
* is the first write to it. Within a transaction the writes are ordered by program order, and
|
|
@@ -435,7 +446,11 @@ export class DatabaseTransaction implements Transaction {
|
|
|
435
446
|
// link with no writes that is being read in a loop would otherwise masquerade as write activity and
|
|
436
447
|
// keep a write-holding head immortal.
|
|
437
448
|
declare writeTimeout: number;
|
|
449
|
+
writeTick = -1;
|
|
438
450
|
timeoutBudget = 0;
|
|
451
|
+
// Initialized rather than `declare`d so the class shape stays monomorphic on the write path. One clock
|
|
452
|
+
// read per handle is what lets the monitor age a handle without calling into the registry every tick.
|
|
453
|
+
handleOpenedAt = 0;
|
|
439
454
|
// save() only stages here; ImmediateTransaction overrides it to commit, which addWrite must not defer
|
|
440
455
|
saveCommits = false;
|
|
441
456
|
// True where save() puts the write into this transaction's native handle, which is what lets a scope
|
|
@@ -449,6 +464,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
449
464
|
declare next: DatabaseTransaction;
|
|
450
465
|
// The head of this multi-store chain, set when the link is created; absent on the head itself.
|
|
451
466
|
declare root?: DatabaseTransaction;
|
|
467
|
+
// When this logical commit first reached the storage engine, held on the chain root so every
|
|
468
|
+
// retry round and every chained store measures ONE elapsed wait (issue #2450). Deliberately not
|
|
469
|
+
// the per-attempt clock trackOutstandingCommit() keeps: that one drives thread-wide load
|
|
470
|
+
// shedding and must stay per-attempt, or a long uncapped source-apply retry would 503 every
|
|
471
|
+
// unrelated request on the thread. Cleared when the logical commit settles, so a reused
|
|
472
|
+
// transaction's next batch starts on a fresh budget.
|
|
473
|
+
declare commitStartedAt?: number;
|
|
452
474
|
// Whether this link is why its chain root is write-supervised (see endWriteSupervision).
|
|
453
475
|
declare writeSupervised?: boolean;
|
|
454
476
|
declare stale: boolean;
|
|
@@ -489,10 +511,31 @@ export class DatabaseTransaction implements Transaction {
|
|
|
489
511
|
// asks to stop reading a pinned snapshot, so re-pinning one for the rest of the scope would take
|
|
490
512
|
// back what it asked for.
|
|
491
513
|
snapshotFree = false;
|
|
514
|
+
// Set while commit() is parked in its pre-commit await (`before`/`beforeIntermediate` completions —
|
|
515
|
+
// in practice a blob's durable file write). The write set is sealed and the caller is awaiting the
|
|
516
|
+
// commit, so this is core's own I/O rather than an application holding a transaction open, which is
|
|
517
|
+
// what the open-transaction limit polices: the monitor spares it for COMMIT_PHASE_GRACE ticks
|
|
518
|
+
// instead of poisoning it (issue #2062).
|
|
519
|
+
committing = false;
|
|
520
|
+
commitPhaseTicks = 0;
|
|
521
|
+
declare commitChainHead?: DatabaseTransaction;
|
|
522
|
+
// O(1) lookup in recordLockFor; only lock() handles are registered here (no gate handles).
|
|
523
|
+
declare recordLocks?: Map<any, Map<unknown, RecordLockHandle>>;
|
|
524
|
+
// Tracks in-flight acquireRecordKey calls so concurrent lock() calls for the same key in one
|
|
525
|
+
// link (e.g. Promise.all([T.lock(id), T.lock(id)])) can coalesce rather than self-block.
|
|
526
|
+
declare pendingLocks?: Map<any, Map<unknown, Promise<RecordLockHandle>>>;
|
|
492
527
|
|
|
493
|
-
|
|
528
|
+
setCommitPhase(committing: boolean): void {
|
|
529
|
+
// A commit phase covers the sealed write set across the whole multi-store chain.
|
|
530
|
+
for (let txn: DatabaseTransaction = this; txn; txn = txn.next) {
|
|
531
|
+
txn.committing = committing;
|
|
532
|
+
txn.commitChainHead = committing ? this : undefined;
|
|
533
|
+
if (committing) txn.commitPhaseTicks = 0;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
494
536
|
|
|
495
|
-
|
|
537
|
+
getReadTxn(disableSnapshot?: boolean): ReadTransaction {
|
|
538
|
+
this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
|
|
496
539
|
// The limit is an IDLE limit. Writes always re-arm it (see addWrite), but reads only do so
|
|
497
540
|
// while no uncommitted writes are held: staged writes hold write intents that other writers'
|
|
498
541
|
// coordinated-retry commits park on, so a handler that wrote once and then only reads — an
|
|
@@ -505,11 +548,6 @@ export class DatabaseTransaction implements Transaction {
|
|
|
505
548
|
if ((this.writes.length === 0 && !this.next) || this.open !== TRANSACTION_STATE.OPEN || !this.hasPendingWrites()) {
|
|
506
549
|
this.timeout = Math.max(txnExpiration, this.timeoutBudget);
|
|
507
550
|
}
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
getReadTxn(disableSnapshot?: boolean): ReadTransaction {
|
|
511
|
-
this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
|
|
512
|
-
this.renewReadTimeout();
|
|
513
551
|
if (this.transaction) {
|
|
514
552
|
if ((this.transaction as any).openTimer) (this.transaction as any).openTimer = 0;
|
|
515
553
|
return this.transaction;
|
|
@@ -544,10 +582,9 @@ export class DatabaseTransaction implements Transaction {
|
|
|
544
582
|
// Monitor state is not ownership state: it stays with `trackedTxns.add` in getReadTxn().
|
|
545
583
|
private attachOwnedTransaction(transaction: RocksTransactionWithRetry): void {
|
|
546
584
|
this.transaction = transaction;
|
|
547
|
-
readTransactionOwners.set(transaction, this);
|
|
548
|
-
this.rangeReadActive = false;
|
|
549
585
|
this.readTxnsUsed = 1;
|
|
550
586
|
this.baseReadRefConsumed = false;
|
|
587
|
+
this.handleOpenedAt = performance.now();
|
|
551
588
|
}
|
|
552
589
|
|
|
553
590
|
/**
|
|
@@ -586,9 +623,9 @@ export class DatabaseTransaction implements Transaction {
|
|
|
586
623
|
trackedTxns.delete(this);
|
|
587
624
|
this.endWriteSupervision();
|
|
588
625
|
this.transaction = null;
|
|
589
|
-
this.rangeReadActive = false;
|
|
590
626
|
this.readTxnsUsed = 0;
|
|
591
627
|
this.readTxnRefCount = 0;
|
|
628
|
+
this.handleOpenedAt = 0;
|
|
592
629
|
return transaction;
|
|
593
630
|
}
|
|
594
631
|
|
|
@@ -635,11 +672,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
635
672
|
*/
|
|
636
673
|
releaseReadTxn(): void {
|
|
637
674
|
const transaction = this.detachOwnedTransaction();
|
|
638
|
-
|
|
639
|
-
transaction?.abort();
|
|
640
|
-
} catch (error) {
|
|
641
|
-
harperLogger.debug?.('releasing timed-out read transaction', error);
|
|
642
|
-
}
|
|
675
|
+
abortNativeTransaction(transaction, 'releasing timed-out read transaction');
|
|
643
676
|
this.completeDeferredContextRelease();
|
|
644
677
|
}
|
|
645
678
|
|
|
@@ -712,6 +745,70 @@ export class DatabaseTransaction implements Transaction {
|
|
|
712
745
|
}
|
|
713
746
|
}
|
|
714
747
|
|
|
748
|
+
registerRecordLock(handle: RecordLockHandle): void {
|
|
749
|
+
if (!this.recordLocks) this.recordLocks = new Map();
|
|
750
|
+
let storeMap = this.recordLocks.get(handle.store);
|
|
751
|
+
if (!storeMap) this.recordLocks.set(handle.store, (storeMap = new Map()));
|
|
752
|
+
storeMap.set(handle.keyId, handle);
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
recordLockFor(store: any, keyId: unknown): RecordLockHandle | undefined {
|
|
756
|
+
const storeMap = this.recordLocks?.get(store);
|
|
757
|
+
if (!storeMap) return undefined;
|
|
758
|
+
const h = storeMap.get(keyId);
|
|
759
|
+
if (!h) return undefined;
|
|
760
|
+
if (!h.released) return h;
|
|
761
|
+
// Prune released handles so they don't accumulate; an expired handle checking re-entrancy
|
|
762
|
+
// would otherwise be seen as the holder and incorrectly granted access.
|
|
763
|
+
storeMap.delete(keyId);
|
|
764
|
+
return undefined;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
unregisterRecordLock(handle: RecordLockHandle): void {
|
|
768
|
+
const storeMap = this.recordLocks?.get(handle.store);
|
|
769
|
+
if (storeMap?.get(handle.keyId) === handle) storeMap.delete(handle.keyId);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
registerPendingLock(store: any, keyId: unknown, pending: Promise<RecordLockHandle>): void {
|
|
773
|
+
if (!this.pendingLocks) this.pendingLocks = new Map();
|
|
774
|
+
let storeMap = this.pendingLocks.get(store);
|
|
775
|
+
if (!storeMap) this.pendingLocks.set(store, (storeMap = new Map()));
|
|
776
|
+
storeMap.set(keyId, pending);
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
pendingLockFor(store: any, keyId: unknown): Promise<RecordLockHandle> | undefined {
|
|
780
|
+
return this.pendingLocks?.get(store)?.get(keyId);
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
unregisterPendingLock(store: any, keyId: unknown): void {
|
|
784
|
+
this.pendingLocks?.get(store)?.delete(keyId);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/** Release every transaction-scoped lock() handle this link owns. */
|
|
788
|
+
releaseRecordLocks(): void {
|
|
789
|
+
const recordLocks = this.recordLocks;
|
|
790
|
+
if (!recordLocks) return;
|
|
791
|
+
for (const [store, storeMap] of recordLocks) {
|
|
792
|
+
for (const [keyId, handle] of storeMap) {
|
|
793
|
+
if (handle.hold) continue; // hold handles outlive the transaction; released by unlock()
|
|
794
|
+
handle.release();
|
|
795
|
+
storeMap.delete(keyId);
|
|
796
|
+
}
|
|
797
|
+
if (storeMap.size === 0) recordLocks.delete(store);
|
|
798
|
+
}
|
|
799
|
+
if (recordLocks.size === 0) this.recordLocks = undefined;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
private noteCommittedLockVersions(): void {
|
|
803
|
+
const recordLocks = this.recordLocks;
|
|
804
|
+
if (!recordLocks) return;
|
|
805
|
+
for (const write of this.writes) {
|
|
806
|
+
if (write?.appliedRecordVersion == null) continue;
|
|
807
|
+
const handle = write.lockHandle ?? recordLocks.get(write.store)?.get(writeKeyId(write.key));
|
|
808
|
+
if (handle && !handle.released) handle.noteHolderVersion(write.appliedRecordVersion);
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
|
|
715
812
|
/**
|
|
716
813
|
* Discard the staged write set (committed or aborted); the per-key chain must go with it so a
|
|
717
814
|
* reused transaction never bases a write on a previous batch's staged state.
|
|
@@ -771,26 +868,28 @@ export class DatabaseTransaction implements Transaction {
|
|
|
771
868
|
// original per-request log was fixed to avoid, just shifted from per-request to per-commit.
|
|
772
869
|
// A commit skipped by the cooldown is NOT marked `logged`, so it still gets a log later if
|
|
773
870
|
// it's still the oldest once the cooldown clears, rather than going silent forever.
|
|
774
|
-
if (!oldestOutstandingCommit.logged && now
|
|
871
|
+
if (!oldestOutstandingCommit.logged && allowStuckCommitLog('shed', now)) {
|
|
775
872
|
// Log once per stuck commit (not once per rejected request, harper#2001): a wedged
|
|
776
873
|
// thread otherwise logs nothing at all server-side while rejecting every write with a
|
|
777
874
|
// 503, which was the single biggest obstacle to root-causing a recurrence. The flag lives
|
|
778
875
|
// on the node itself, so if THIS commit settles while still over the limit and a
|
|
779
876
|
// different one is now oldest, that one logs too instead of staying silent forever.
|
|
780
877
|
oldestOutstandingCommit.logged = true;
|
|
781
|
-
lastOverloadLogAt = now;
|
|
782
|
-
const nativeTransactionId = oldestOutstandingCommit.nativeTransaction?.id;
|
|
783
|
-
const store = oldestOutstandingCommit.store;
|
|
784
|
-
const startedFrom = oldestOutstandingCommit.startedFrom;
|
|
785
878
|
harperLogger.error(
|
|
786
879
|
`Rejecting writes on this thread: a commit has been outstanding for ` +
|
|
787
880
|
`${Math.round(now - oldestOutstandingCommit.start)}ms (exceeds the ` +
|
|
788
|
-
`${MAX_OUTSTANDING_TXN_DURATION}ms limit),
|
|
789
|
-
(
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
881
|
+
`${MAX_OUTSTANDING_TXN_DURATION}ms limit), ` +
|
|
882
|
+
describeCommitIdentity(
|
|
883
|
+
oldestOutstandingCommit.store,
|
|
884
|
+
oldestOutstandingCommit.startedFrom,
|
|
885
|
+
oldestOutstandingCommit.nativeTransaction
|
|
886
|
+
) +
|
|
887
|
+
`.` +
|
|
888
|
+
describeHolderCandidates(
|
|
889
|
+
oldestOutstandingCommit.store?.rootStore?.path,
|
|
890
|
+
oldestOutstandingCommit.nativeTransaction?.id
|
|
891
|
+
) +
|
|
892
|
+
` Further record updates and publishes from new application requests on this thread ` +
|
|
794
893
|
`will be rejected with 503 until the commit settles or the process is restarted (deletes, ` +
|
|
795
894
|
`and writes applied from a canonical source, e.g. replication or a caching source, bypass this check).`
|
|
796
895
|
);
|
|
@@ -854,6 +953,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
854
953
|
// Independent write-recency signal for chainStillActive (see the field comment) — reads never
|
|
855
954
|
// touch this, only writes do.
|
|
856
955
|
this.writeTimeout = this.timeout;
|
|
956
|
+
this.writeTick = monitorTick;
|
|
857
957
|
this.linkWrite(operation);
|
|
858
958
|
this.writes.push(operation);
|
|
859
959
|
operation.stagedIn = this;
|
|
@@ -884,7 +984,32 @@ export class DatabaseTransaction implements Transaction {
|
|
|
884
984
|
}
|
|
885
985
|
|
|
886
986
|
save(operation: TransactionWrite, transaction?: RocksTransaction, reloadEntry = false, options?: CommitOptions) {
|
|
887
|
-
|
|
987
|
+
const lockHandle = operation.lockHandle;
|
|
988
|
+
// Guard: a write staged through an expired or released lock handle must not land.
|
|
989
|
+
// The handle's lease timer already unlocked the native key; another holder may have taken it.
|
|
990
|
+
if (lockHandle && (lockHandle.expired || lockHandle.released)) {
|
|
991
|
+
// Remove the operation from the staged set so subsequent writes on this context do not
|
|
992
|
+
// re-throw 409 due to a stale null-saved entry sitting in this.writes.
|
|
993
|
+
const failedIdx = this.writes.indexOf(operation);
|
|
994
|
+
if (failedIdx > -1) this.writes[failedIdx] = null;
|
|
995
|
+
throw lockNotHeldError(lockHandle);
|
|
996
|
+
}
|
|
997
|
+
// Lock-write timestamp rules.
|
|
998
|
+
if (lockHandle) {
|
|
999
|
+
if (this.open === TRANSACTION_STATE.CLOSED || this.saveCommits) {
|
|
1000
|
+
// CLOSED path (second+ write per ImmediateTransaction cycle) OR the first write in
|
|
1001
|
+
// an ImmediateTransaction (open=OPEN until commit sets it CLOSED, but saveCommits
|
|
1002
|
+
// signals the per-write-commit semantics): stamp with nextHolderVersion() so that
|
|
1003
|
+
// each sequential save() gets its own monotonically increasing stamp — scoped or hold.
|
|
1004
|
+
// The stamp stays on the operation and is consumed below rather than assigned to
|
|
1005
|
+
// this.timestamp: pinning the link's clock would stamp every OTHER write staged on
|
|
1006
|
+
// the same context before the commit resets it — a concurrent write in the caller's
|
|
1007
|
+
// own Promise.all, or the next operation in a retry/replay save loop — with the
|
|
1008
|
+
// lock's version, which LWW then silently drops against a newer record version.
|
|
1009
|
+
if (!operation.lockStamp) operation.lockStamp = lockHandle.holderVersionCandidate();
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
let txnTime = operation.lockStamp ?? this.timestamp;
|
|
888
1013
|
// Only an OPEN transaction accepts new staged writes. After commit, this.transaction may still
|
|
889
1014
|
// be retained for outstanding read iterators; staging into it would silently discard the write
|
|
890
1015
|
// when doneReadTxn() aborts the handle, so such writes commit immediately on a fresh
|
|
@@ -929,13 +1054,19 @@ export class DatabaseTransaction implements Transaction {
|
|
|
929
1054
|
(transaction as RocksTransactionWithRetry).isRetry = true;
|
|
930
1055
|
}
|
|
931
1056
|
if (!txnTime) txnTime = this.timestamp = transaction.getTimestamp();
|
|
1057
|
+
// `txnTime` is this transaction's timestamp — the key its entries take in the per-origin log.
|
|
1058
|
+
// A write applied from elsewhere carries the origin's record version too, and that is what the
|
|
1059
|
+
// record is stored at; the two coincide for every locally-originated write. Gated on the apply
|
|
1060
|
+
// flags so an ordinary write never reads the property (harper#2412).
|
|
1061
|
+
const writeVersion =
|
|
1062
|
+
this.sourceApply || this.isReplay ? getAppliedWriteVersion(operation.recordVersion, txnTime) : txnTime;
|
|
932
1063
|
if (reloadEntry || operation.entry === undefined) {
|
|
933
1064
|
operation.entry = operation.store.getEntry(operation.key, { transaction });
|
|
934
1065
|
}
|
|
935
1066
|
if (!operation.saved) {
|
|
936
1067
|
operation.saved = true;
|
|
937
1068
|
// immediately execute in this transaction
|
|
938
|
-
if ((operation.validate?.(
|
|
1069
|
+
if ((operation.validate?.(writeVersion, this) as any) === false) {
|
|
939
1070
|
operation.commit = () => {}; // noop if we try again
|
|
940
1071
|
return;
|
|
941
1072
|
}
|
|
@@ -944,7 +1075,11 @@ export class DatabaseTransaction implements Transaction {
|
|
|
944
1075
|
result = operation.beforeIntermediate?.() as Promise<void>;
|
|
945
1076
|
if (result?.then) this.stageCompletion(result);
|
|
946
1077
|
}
|
|
947
|
-
|
|
1078
|
+
if (lockHandle || this.recordLocks) operation.trackRecordVersion = true;
|
|
1079
|
+
if (operation.trackRecordVersion) operation.recordVersionApplied = false;
|
|
1080
|
+
const completion = operation.commit(writeVersion, operation.entry, this.retries > 0, transaction) as Promise<void>;
|
|
1081
|
+
if (operation.trackRecordVersion)
|
|
1082
|
+
operation.appliedRecordVersion = operation.recordVersionApplied ? writeVersion : undefined;
|
|
948
1083
|
if (typeof completion?.then === 'function') this.stageCompletion(completion);
|
|
949
1084
|
// Sticky record that THIS write staged with its audit entry appended (log entries batch on the
|
|
950
1085
|
// native transaction and are durably written by its commit attempt — even a failed one — so
|
|
@@ -956,7 +1091,13 @@ export class DatabaseTransaction implements Transaction {
|
|
|
956
1091
|
operation.appendedAuditEntry = true;
|
|
957
1092
|
}
|
|
958
1093
|
if (immediateCommit) {
|
|
959
|
-
|
|
1094
|
+
// immediately commit if the harper transaction is closed
|
|
1095
|
+
const innerCommit = this.commit({ ...options, transaction });
|
|
1096
|
+
// Expose on the operation so the lock-writable Table.save() path can await the real
|
|
1097
|
+
// native commit — without this the ImmediateTransaction outer commit resolves before
|
|
1098
|
+
// the native transaction.commit() settles (fire-and-forget from the if-branch).
|
|
1099
|
+
operation.innerCommit = innerCommit;
|
|
1100
|
+
return innerCommit;
|
|
960
1101
|
}
|
|
961
1102
|
}
|
|
962
1103
|
|
|
@@ -968,17 +1109,35 @@ export class DatabaseTransaction implements Transaction {
|
|
|
968
1109
|
// reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain —
|
|
969
1110
|
// but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction
|
|
970
1111
|
let transaction = options.transaction ?? this.transaction;
|
|
971
|
-
|
|
972
|
-
let
|
|
973
|
-
|
|
974
|
-
|
|
1112
|
+
try {
|
|
1113
|
+
for (let i = 0; i < this.writes.length; i++) {
|
|
1114
|
+
let operation = this.writes[i];
|
|
1115
|
+
if (!operation || (this.retries === 0 && operation.saved)) continue;
|
|
1116
|
+
this.save(operation, transaction, i < this.validated, options);
|
|
1117
|
+
}
|
|
1118
|
+
} catch (error) {
|
|
1119
|
+
this.abort();
|
|
1120
|
+
throw error;
|
|
975
1121
|
}
|
|
976
1122
|
this.validated = this.writes.length;
|
|
977
1123
|
const completions = this.completions;
|
|
978
1124
|
if (completions.length > 0) this.completions = []; // reset
|
|
1125
|
+
const stagedWrites = this.writes.length;
|
|
1126
|
+
if (completions.length > 0) {
|
|
1127
|
+
this.setCommitPhase(true);
|
|
1128
|
+
}
|
|
979
1129
|
return when(
|
|
980
1130
|
completions.length > 0 ? Promise.all(completions) : null,
|
|
981
1131
|
() => {
|
|
1132
|
+
if (completions.length > 0) this.setCommitPhase(false);
|
|
1133
|
+
// The transaction can be aborted underneath us while we are parked in the await above — by the
|
|
1134
|
+
// monitor once the commit phase outlives its grace, or through the multi-store poison chain.
|
|
1135
|
+
// abort() cleared the write set and released the handle, so resuming would commit nothing and
|
|
1136
|
+
// resolve as SUCCESS: the caller is told its write landed when it was dropped, and a write
|
|
1137
|
+
// carrying a blob is left holding an instance whose file was unlinked (issue #2062).
|
|
1138
|
+
if (this.timedOut) throw transactionOpenTooLongError();
|
|
1139
|
+
if (stagedWrites > 0 && this.writes.length === 0 && this.open === TRANSACTION_STATE.CLOSED)
|
|
1140
|
+
throw new ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
|
|
982
1141
|
if (this.writes.length > this.validated) {
|
|
983
1142
|
// check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
|
|
984
1143
|
return this.commit(options);
|
|
@@ -1075,8 +1234,6 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1075
1234
|
if (transaction) {
|
|
1076
1235
|
this.writes = this.writes.filter((write) => write); // filter out removed entries
|
|
1077
1236
|
if (this.writes.length > 0) {
|
|
1078
|
-
// Commit retries can construct fresh ranges on this live handle after read ownership ends.
|
|
1079
|
-
readTransactionOwners.delete(transaction);
|
|
1080
1237
|
// The transaction was created with coordinatedRetry:true (see
|
|
1081
1238
|
// getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
|
|
1082
1239
|
// sentinel (a number) is why commitResolution is typed
|
|
@@ -1114,8 +1271,12 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1114
1271
|
// claimed this per-database transaction in txnForContext and so can name the wrong
|
|
1115
1272
|
// table when a transaction spans more than one table in the same database.
|
|
1116
1273
|
trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
|
|
1274
|
+
// Every retry round and every chained store re-enters here and must inherit the chain
|
|
1275
|
+
// root's clock rather than restart it, so only the first submission stamps.
|
|
1276
|
+
const chainRoot = this.root ?? this;
|
|
1277
|
+
if (chainRoot.commitStartedAt == null) chainRoot.commitStartedAt = performance.now();
|
|
1117
1278
|
const completions = [];
|
|
1118
|
-
|
|
1279
|
+
const commitOutcome = commitResolution.then(
|
|
1119
1280
|
(commitResult) => {
|
|
1120
1281
|
if (commitResult === RETRY_NOW_VALUE) {
|
|
1121
1282
|
this.retries++;
|
|
@@ -1123,6 +1284,8 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1123
1284
|
// Mark this specific native transaction as a retry so RocksTransactionLogStore
|
|
1124
1285
|
// skips re-writing its already-staged txn-log entries (#2).
|
|
1125
1286
|
(transaction as RocksTransactionWithRetry).isRetry = true;
|
|
1287
|
+
const pastBudget = this.elapsedPastCommitBudget();
|
|
1288
|
+
if (pastBudget) this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
1126
1289
|
// Mirror the ERR_BUSY cap/warn policy: non-sourceApply transactions abort
|
|
1127
1290
|
// at MAX_RETRIES; sourceApply transactions keep retrying with periodic warn.
|
|
1128
1291
|
if (this.retries > MAX_RETRIES) {
|
|
@@ -1182,9 +1345,11 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1182
1345
|
if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
|
|
1183
1346
|
cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
|
|
1184
1347
|
}
|
|
1348
|
+
if (this.recordLocks) this.noteCommittedLockVersions();
|
|
1185
1349
|
// now reset transactions tracking; this transaction be reused and committed again
|
|
1186
1350
|
this.retries = 0; // reset per-native-transaction retry counter so a reused DatabaseTransaction's next batch starts fresh
|
|
1187
1351
|
this.clearWrites();
|
|
1352
|
+
this.releaseRecordLocks();
|
|
1188
1353
|
if (options.doneWriting) this.endScopeOwnership();
|
|
1189
1354
|
this.releaseContext(!!options.doneWriting);
|
|
1190
1355
|
let txnTime = this.timestamp;
|
|
@@ -1239,6 +1404,10 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1239
1404
|
// premature publish, and no fresh-transaction replay that would drop the entry.
|
|
1240
1405
|
// Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
|
|
1241
1406
|
(transaction as RocksTransactionWithRetry).isRetry = true;
|
|
1407
|
+
// Before the backoff gate below: the budget can already be spent on the first
|
|
1408
|
+
// retry when an earlier store in this chain consumed it.
|
|
1409
|
+
const pastBudget = this.elapsedPastCommitBudget();
|
|
1410
|
+
if (pastBudget) this.abandonCommitAfterDeadline(transaction, pastBudget);
|
|
1242
1411
|
if (this.retries > 2) {
|
|
1243
1412
|
// Transactions applying data from a canonical source of truth (replication peer or
|
|
1244
1413
|
// external caching source) must never drop a write on a transient conflict: there is no
|
|
@@ -1287,24 +1456,45 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1287
1456
|
// finished and its durability is unknown, so ownership goes with it.
|
|
1288
1457
|
this.endScopeOwnership();
|
|
1289
1458
|
this.releaseContext(!!options.doneWriting);
|
|
1459
|
+
this.releaseRecordLocks();
|
|
1460
|
+
this.timestamp = 0;
|
|
1290
1461
|
throw error;
|
|
1291
1462
|
}
|
|
1292
1463
|
}
|
|
1293
1464
|
);
|
|
1465
|
+
// `commitOutcome` settles when the LOGICAL commit ends — its success branch awaits the
|
|
1466
|
+
// chained stores' own commits — so releasing here covers every terminal exit (success,
|
|
1467
|
+
// retry exhaustion, abandonment, terminal failure) in one place rather than five.
|
|
1468
|
+
// Released through the RETURNED promise rather than a second subscriber on
|
|
1469
|
+
// `commitOutcome`: a subscriber would mark a dropped commit rejection as handled and
|
|
1470
|
+
// silence the unhandled-rejection that surfaces it.
|
|
1471
|
+
return commitOutcome.then(
|
|
1472
|
+
(resolution) => {
|
|
1473
|
+
chainRoot.commitStartedAt = undefined;
|
|
1474
|
+
return resolution;
|
|
1475
|
+
},
|
|
1476
|
+
(error) => {
|
|
1477
|
+
chainRoot.commitStartedAt = undefined;
|
|
1478
|
+
throw error;
|
|
1479
|
+
}
|
|
1480
|
+
);
|
|
1294
1481
|
}
|
|
1295
1482
|
for (const write of this.writes) {
|
|
1296
1483
|
if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
|
|
1297
1484
|
cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
|
|
1298
1485
|
}
|
|
1486
|
+
if (this.recordLocks) this.noteCommittedLockVersions();
|
|
1299
1487
|
this.clearWrites();
|
|
1488
|
+
this.releaseRecordLocks();
|
|
1300
1489
|
if (options.doneWriting) this.endScopeOwnership();
|
|
1301
1490
|
this.releaseContext(!!options.doneWriting);
|
|
1302
1491
|
const txnResolution: CommitResolution = {
|
|
1303
1492
|
txnTime: this.timestamp,
|
|
1304
1493
|
};
|
|
1494
|
+
this.timestamp = 0; // reset like the async path (~1279) so stale lock stamps don't persist
|
|
1305
1495
|
if (this.next) {
|
|
1306
1496
|
// now run any other transactions
|
|
1307
|
-
options.timestamp =
|
|
1497
|
+
options.timestamp = txnResolution.txnTime;
|
|
1308
1498
|
// as above: the next store must not inherit this store's explicit native transaction
|
|
1309
1499
|
let nextResolution;
|
|
1310
1500
|
try {
|
|
@@ -1323,7 +1513,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1323
1513
|
// scope resumable after a partially failed mid-scope commit.
|
|
1324
1514
|
this.completeMidScopeCommit(options);
|
|
1325
1515
|
return {
|
|
1326
|
-
txnTime:
|
|
1516
|
+
txnTime: txnResolution.txnTime,
|
|
1327
1517
|
next: nextResolution,
|
|
1328
1518
|
};
|
|
1329
1519
|
},
|
|
@@ -1342,6 +1532,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1342
1532
|
return txnResolution;
|
|
1343
1533
|
},
|
|
1344
1534
|
(error) => {
|
|
1535
|
+
this.setCommitPhase(false);
|
|
1345
1536
|
this.abort();
|
|
1346
1537
|
throw error;
|
|
1347
1538
|
}
|
|
@@ -1390,6 +1581,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1390
1581
|
// Defensively release any native handle whose reference bookkeeping was already consumed.
|
|
1391
1582
|
if (this.transaction) this.releaseReadTxn();
|
|
1392
1583
|
this.open = TRANSACTION_STATE.CLOSED;
|
|
1584
|
+
this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
|
|
1393
1585
|
this.drainCompletions();
|
|
1394
1586
|
try {
|
|
1395
1587
|
for (const write of this.writes) {
|
|
@@ -1399,6 +1591,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1399
1591
|
} finally {
|
|
1400
1592
|
this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
|
|
1401
1593
|
this.clearWrites();
|
|
1594
|
+
this.releaseRecordLocks();
|
|
1402
1595
|
// A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
|
|
1403
1596
|
// Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
|
|
1404
1597
|
// starting a fresh one) so the rest of the logical operation fails atomically via the
|
|
@@ -1417,6 +1610,62 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1417
1610
|
}
|
|
1418
1611
|
}
|
|
1419
1612
|
}
|
|
1613
|
+
/** How long this logical commit may keep retrying: the thread-wide queue limit, or its own larger explicit budget. */
|
|
1614
|
+
private commitConflictBudget(): number {
|
|
1615
|
+
return Math.max(MAX_OUTSTANDING_TXN_DURATION, (this.root ?? this).timeoutBudget || 0);
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
/**
|
|
1619
|
+
* How long this logical commit has been retrying once it is past its budget, else 0. rocksdb-js
|
|
1620
|
+
* returns control from a parked commit every `ROCKSDB_JS_PARK_TIMEOUT_MS` even when the intent
|
|
1621
|
+
* holder never releases, so without this bound a request-path commit retries the attempt cap out
|
|
1622
|
+
* — minutes past the queue limit an operator configured (issue #2450).
|
|
1623
|
+
*
|
|
1624
|
+
* Source-applied writes are exempt for the same reason they are exempt from the attempt cap:
|
|
1625
|
+
* there is no resubscribe/sequence-resume path, so dropping one permanently diverges this node.
|
|
1626
|
+
*/
|
|
1627
|
+
private elapsedPastCommitBudget(): number {
|
|
1628
|
+
if (this.sourceApply) return 0;
|
|
1629
|
+
const startedAt = (this.root ?? this).commitStartedAt;
|
|
1630
|
+
if (startedAt == null) return 0;
|
|
1631
|
+
const elapsed = performance.now() - startedAt;
|
|
1632
|
+
return elapsed > this.commitConflictBudget() ? elapsed : 0;
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
/**
|
|
1636
|
+
* Abandon a logical commit that stayed in write-intent conflict past its budget. Cleanup is
|
|
1637
|
+
* retry exhaustion's, so no link leaks a native handle or read snapshot; the error is distinct
|
|
1638
|
+
* because the condition is distinct — every attempt reported transient contention, so a later
|
|
1639
|
+
* request can succeed once the holder releases, which the generic exhaustion 500 does not say.
|
|
1640
|
+
*
|
|
1641
|
+
* Only a chain root may report `retryable`: a link commits solely from its predecessor's success
|
|
1642
|
+
* handler, so anywhere else in the chain an earlier store has already landed durable audit
|
|
1643
|
+
* entries and hooks that a replayed request would run twice. A head whose scope already rotated
|
|
1644
|
+
* through a mid-scope commit is in the same position.
|
|
1645
|
+
*/
|
|
1646
|
+
private abandonCommitAfterDeadline(headTransaction: RocksTransaction, elapsedMs: number): never {
|
|
1647
|
+
const elapsed = Math.round(elapsedMs);
|
|
1648
|
+
const budget = this.commitConflictBudget();
|
|
1649
|
+
const retryable = !this.root && !this.snapshotFree;
|
|
1650
|
+
if (allowStuckCommitLog('abandon', performance.now())) {
|
|
1651
|
+
harperLogger.error(
|
|
1652
|
+
`Abandoning a write transaction: its commit has been in write-intent conflict for ${elapsed}ms ` +
|
|
1653
|
+
`(exceeds the ${budget}ms limit) across ${this.retries} retries, ` +
|
|
1654
|
+
describeCommitIdentity(this.writes[0]?.store, this.startedFrom, headTransaction) +
|
|
1655
|
+
`.` +
|
|
1656
|
+
describeHolderCandidates(this.writes[0]?.store?.rootStore?.path, headTransaction?.id) +
|
|
1657
|
+
` Another transaction holds a conflicting write intent and has not completed; the request is ` +
|
|
1658
|
+
`failed with a 503${retryable ? '' : ' (not retryable — an earlier store in this transaction already committed)'} ` +
|
|
1659
|
+
`rather than waiting further.`
|
|
1660
|
+
);
|
|
1661
|
+
}
|
|
1662
|
+
this.abortChainAfterRetries(headTransaction);
|
|
1663
|
+
throw new TransactionCommitConflictTimeoutError(
|
|
1664
|
+
`Commit was in conflict with ongoing writes for ${elapsed}ms, exceeding the ${budget}ms limit; transaction abandoned after ${this.retries} retries`,
|
|
1665
|
+
retryable
|
|
1666
|
+
);
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1420
1669
|
/**
|
|
1421
1670
|
* Give up on a chain of linked transactions after exhausting conflict retries: poison every link
|
|
1422
1671
|
* first, then abort each link's native transaction and release its DatabaseTransaction-level
|
|
@@ -1506,11 +1755,7 @@ export class DatabaseTransaction implements Transaction {
|
|
|
1506
1755
|
// throws. abort() rather than the native abort alone: it reclaims blobs a replayed write
|
|
1507
1756
|
// staged.
|
|
1508
1757
|
this.detachOwnedTransaction();
|
|
1509
|
-
|
|
1510
|
-
transaction?.abort();
|
|
1511
|
-
} catch (abortError) {
|
|
1512
|
-
harperLogger.debug?.('aborting a transaction whose synchronous commit failed', abortError);
|
|
1513
|
-
}
|
|
1758
|
+
abortNativeTransaction(transaction, 'aborting a transaction whose synchronous commit failed');
|
|
1514
1759
|
try {
|
|
1515
1760
|
this.abort();
|
|
1516
1761
|
} catch (abortError) {
|
|
@@ -1537,6 +1782,20 @@ export interface Transaction {
|
|
|
1537
1782
|
abort?(): any;
|
|
1538
1783
|
}
|
|
1539
1784
|
|
|
1785
|
+
export function shouldSpareCommitPhase(
|
|
1786
|
+
txn: DatabaseTransaction,
|
|
1787
|
+
checkedCommitPhaseChains: Set<DatabaseTransaction>
|
|
1788
|
+
): boolean {
|
|
1789
|
+
if (!txn.committing) return false;
|
|
1790
|
+
if (txn.sourceApply || txn.isReplay) return true;
|
|
1791
|
+
const commitChainHead = txn.commitChainHead ?? txn;
|
|
1792
|
+
if (!checkedCommitPhaseChains.has(commitChainHead)) {
|
|
1793
|
+
checkedCommitPhaseChains.add(commitChainHead);
|
|
1794
|
+
commitChainHead.commitPhaseTicks++;
|
|
1795
|
+
}
|
|
1796
|
+
return commitChainHead.commitPhaseTicks <= COMMIT_PHASE_GRACE;
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1540
1799
|
export class ImmediateTransaction extends DatabaseTransaction {
|
|
1541
1800
|
isCommitting = false;
|
|
1542
1801
|
saveCommits = true;
|
|
@@ -1545,16 +1804,47 @@ export class ImmediateTransaction extends DatabaseTransaction {
|
|
|
1545
1804
|
this.db = db;
|
|
1546
1805
|
}
|
|
1547
1806
|
save(...args: any[]): any {
|
|
1548
|
-
const
|
|
1807
|
+
const operation = args[0]; // the staged write, not a transaction — commit() re-enters here with it
|
|
1549
1808
|
if (this.isCommitting) {
|
|
1550
1809
|
// if we are in the commit, do the save and force a reload so we get a read within the transaction
|
|
1551
|
-
super.save(
|
|
1810
|
+
super.save(operation, null as any, true);
|
|
1552
1811
|
} else {
|
|
1553
1812
|
this.isCommitting = true;
|
|
1554
|
-
|
|
1813
|
+
// A synchronous throw from commit() (e.g. a 409 from an expired lock handle) would
|
|
1814
|
+
// otherwise leave isCommitting latched at true, causing every subsequent save() in this
|
|
1815
|
+
// context to take the fire-and-forget if-branch and silently drop writes.
|
|
1816
|
+
let commitResult: any;
|
|
1817
|
+
try {
|
|
1818
|
+
commitResult = this.commit();
|
|
1819
|
+
} catch (err) {
|
|
1555
1820
|
this.isCommitting = false;
|
|
1556
|
-
|
|
1821
|
+
throw err;
|
|
1822
|
+
}
|
|
1823
|
+
return when(
|
|
1824
|
+
commitResult,
|
|
1825
|
+
() => {
|
|
1826
|
+
this.isCommitting = false;
|
|
1827
|
+
},
|
|
1828
|
+
(err: any) => {
|
|
1829
|
+
// Async rejection (e.g. a rejected rocksdb commit promise) must also clear the
|
|
1830
|
+
// latch; without this, every subsequent save() silently fire-and-forgets.
|
|
1831
|
+
this.isCommitting = false;
|
|
1832
|
+
throw err;
|
|
1833
|
+
}
|
|
1834
|
+
);
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
// Without an explicit transaction() a live lock is released only by unlock() or its lease, never
|
|
1839
|
+
// by the per-write commit. Expired handles are pruned so a reused context does not retain every key.
|
|
1840
|
+
releaseRecordLocks(): void {
|
|
1841
|
+
const recordLocks = this.recordLocks;
|
|
1842
|
+
if (!recordLocks) return;
|
|
1843
|
+
for (const [store, storeMap] of recordLocks) {
|
|
1844
|
+
for (const [keyId, handle] of storeMap) if (handle.released) storeMap.delete(keyId);
|
|
1845
|
+
if (storeMap.size === 0) recordLocks.delete(store);
|
|
1557
1846
|
}
|
|
1847
|
+
if (recordLocks.size === 0) this.recordLocks = undefined;
|
|
1558
1848
|
}
|
|
1559
1849
|
|
|
1560
1850
|
declare _timestamp: number;
|
|
@@ -1635,6 +1925,7 @@ export function isJoinableScope(transaction: DatabaseTransaction | null | undefi
|
|
|
1635
1925
|
}
|
|
1636
1926
|
|
|
1637
1927
|
let timer;
|
|
1928
|
+
let monitorTick = 0;
|
|
1638
1929
|
|
|
1639
1930
|
/**
|
|
1640
1931
|
* True when a link other than `txn` in the same multi-store chain was written recently enough to
|
|
@@ -1654,20 +1945,96 @@ function chainStillActive(txn: DatabaseTransaction): boolean {
|
|
|
1654
1945
|
return false;
|
|
1655
1946
|
}
|
|
1656
1947
|
|
|
1948
|
+
/**
|
|
1949
|
+
* Name a chain link still holding its native handle past the reporting threshold, and why the
|
|
1950
|
+
* branches below are not reaping it (harper#2471). Attribution only: it reads the recency clocks
|
|
1951
|
+
* those branches maintain rather than calling `chainStillActive`, which would decay `writeTimeout`
|
|
1952
|
+
* as a side effect, and it runs before them so it cannot reorder them.
|
|
1953
|
+
*/
|
|
1954
|
+
function reportLongLivedLink(
|
|
1955
|
+
link: DatabaseTransaction,
|
|
1956
|
+
thresholdMs: number,
|
|
1957
|
+
now: number,
|
|
1958
|
+
reportBudget: LongLivedHolderReportBudget
|
|
1959
|
+
): void {
|
|
1960
|
+
if (!link.transaction || link.handleOpenedAt === 0) return;
|
|
1961
|
+
const ageMs = now - link.handleOpenedAt;
|
|
1962
|
+
if (ageMs < thresholdMs) return;
|
|
1963
|
+
const states: string[] = [];
|
|
1964
|
+
if (link.sourceApply) states.push('source-apply');
|
|
1965
|
+
if (link.isReplay) states.push('replay');
|
|
1966
|
+
if (link.committing) states.push('commit-phase');
|
|
1967
|
+
if (link.open === TRANSACTION_STATE.CLOSED) states.push('closed-with-open-iterators');
|
|
1968
|
+
if (monitorTick - link.writeTick <= 1) states.push('active');
|
|
1969
|
+
if (states.length === 0) states.push('over-limit');
|
|
1970
|
+
reportLongLivedHolder(
|
|
1971
|
+
{
|
|
1972
|
+
databasePath: (link.db as any)?.rootStore?.path,
|
|
1973
|
+
nativeId: link.transaction.id,
|
|
1974
|
+
openedAt: link.handleOpenedAt,
|
|
1975
|
+
ageMs,
|
|
1976
|
+
databaseName: (link.db as any)?.rootStore?.databaseName,
|
|
1977
|
+
tableName: (link.db as any)?.name,
|
|
1978
|
+
countPendingWrites: () => {
|
|
1979
|
+
let pendingWrites = 0;
|
|
1980
|
+
for (const write of link.writes) if (write) pendingWrites++;
|
|
1981
|
+
return pendingWrites;
|
|
1982
|
+
},
|
|
1983
|
+
states,
|
|
1984
|
+
timeoutBudget: link.timeoutBudget,
|
|
1985
|
+
startedFrom: link.startedFrom,
|
|
1986
|
+
},
|
|
1987
|
+
reportBudget
|
|
1988
|
+
);
|
|
1989
|
+
}
|
|
1990
|
+
|
|
1991
|
+
/**
|
|
1992
|
+
* Report every link of this logical transaction that holds its own native handle. A blind write to a
|
|
1993
|
+
* second database attaches a handle to a `.next` link while only the root is supervised, so reporting
|
|
1994
|
+
* the entry alone names an id the sweep's line can never be joined to when the child is the holder.
|
|
1995
|
+
* Links that read are in `trackedTxns` and get their own visit from the tick.
|
|
1996
|
+
*/
|
|
1997
|
+
function reportIfLongLived(
|
|
1998
|
+
txn: DatabaseTransaction,
|
|
1999
|
+
thresholdMs: number,
|
|
2000
|
+
now: number,
|
|
2001
|
+
reportBudget: LongLivedHolderReportBudget
|
|
2002
|
+
): void {
|
|
2003
|
+
if (thresholdMs === 0) return;
|
|
2004
|
+
for (let link: DatabaseTransaction = txn; link; link = link.next) {
|
|
2005
|
+
if (link !== txn && trackedTxns.has(link)) break;
|
|
2006
|
+
reportLongLivedLink(link, thresholdMs, now, reportBudget);
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
|
|
1657
2010
|
function startMonitoringTxns() {
|
|
1658
2011
|
timer = setInterval(function () {
|
|
2012
|
+
monitorTick++;
|
|
2013
|
+
const checkedCommitPhaseChains = new Set<DatabaseTransaction>();
|
|
2014
|
+
const reportThresholdMs = getReportThresholdMs();
|
|
2015
|
+
rebaseLongLivedTransactionReports(reportThresholdMs);
|
|
2016
|
+
const reportBudget = createLongLivedHolderReportBudget(reportThresholdMs);
|
|
2017
|
+
const reportNow = performance.now();
|
|
1659
2018
|
// Both registries, in sequence rather than as a union: a root can be in each (it read, and a
|
|
1660
2019
|
// later link blind-wrote), and the membership check is cheaper than allocating per tick.
|
|
1661
|
-
for (const txn of trackedTxns)
|
|
1662
|
-
|
|
2020
|
+
for (const txn of trackedTxns)
|
|
2021
|
+
monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
|
|
2022
|
+
for (const txn of supervisedWriteRoots)
|
|
2023
|
+
if (!trackedTxns.has(txn))
|
|
2024
|
+
monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
|
|
2025
|
+
finishLongLivedHolderReports(reportBudget);
|
|
1663
2026
|
}, txnExpiration).unref();
|
|
1664
2027
|
|
|
1665
|
-
function monitorTransaction(
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
2028
|
+
function monitorTransaction(
|
|
2029
|
+
txn: DatabaseTransaction,
|
|
2030
|
+
checkedCommitPhaseChains: Set<DatabaseTransaction>,
|
|
2031
|
+
reportThresholdMs: number,
|
|
2032
|
+
reportNow: number,
|
|
2033
|
+
reportBudget: LongLivedHolderReportBudget
|
|
2034
|
+
) {
|
|
2035
|
+
reportIfLongLived(txn, reportThresholdMs, reportNow, reportBudget);
|
|
1670
2036
|
{
|
|
2037
|
+
const commitChainHead = txn.commitChainHead ?? txn;
|
|
1671
2038
|
// Decay write recency once per tick for every tracked link, independent of the `timeout`
|
|
1672
2039
|
// branches below — a tracked link that keeps its own idle limit alive by reading must not
|
|
1673
2040
|
// thereby keep chainStillActive believing it was written recently too.
|
|
@@ -1695,6 +2062,25 @@ function startMonitoringTxns() {
|
|
|
1695
2062
|
}`
|
|
1696
2063
|
);
|
|
1697
2064
|
txn.releaseReadTxn();
|
|
2065
|
+
} else if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
|
|
2066
|
+
// Parked in commit()'s pre-commit await — a `before`/`beforeIntermediate` hook, in practice a
|
|
2067
|
+
// blob's durable file write, which for a multi-tens-of-MB payload legitimately outruns the
|
|
2068
|
+
// limit. The write set is sealed and the caller is awaiting this commit, so the limit's
|
|
2069
|
+
// premise (the application is holding a transaction open) does not hold, and poisoning here
|
|
2070
|
+
// would unlink the blobs the write still references (issue #2062). The grace is bounded
|
|
2071
|
+
// because the transaction still pins a read snapshot: a source that stalls rather than
|
|
2072
|
+
// finishing falls through to the abort below once it runs out. A canonical-source apply or
|
|
2073
|
+
// replay is spared for as long as it takes: neither aborting it (harper-pro#348) nor the
|
|
2074
|
+
// force-commit below — which would durably commit a record whose blob file is still being
|
|
2075
|
+
// written — is acceptable, and their blob sources are bounded by the receive-side idle
|
|
2076
|
+
// watchdog instead.
|
|
2077
|
+
harperLogger.warn?.(
|
|
2078
|
+
`Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work (e.g. a large blob write); letting it complete, from table: ${
|
|
2079
|
+
(txn.db as any)?.name + (url ? ' path: ' + url : '')
|
|
2080
|
+
}`,
|
|
2081
|
+
...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : [])
|
|
2082
|
+
);
|
|
2083
|
+
txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
|
|
1698
2084
|
} else if (txn.hasPendingWrites() && chainStillActive(txn)) {
|
|
1699
2085
|
// A later link in the chain was written recently (writes re-arm only the link that
|
|
1700
2086
|
// receives them, and a multi-store transaction can be writing database B while this
|
|
@@ -1718,7 +2104,7 @@ function startMonitoringTxns() {
|
|
|
1718
2104
|
...(DEBUG_LONG_TXNS ? ['starting stack trace', txn.stackTraces] : [])
|
|
1719
2105
|
);
|
|
1720
2106
|
try {
|
|
1721
|
-
|
|
2107
|
+
commitChainHead.abortDueToTimeout();
|
|
1722
2108
|
} catch (error) {
|
|
1723
2109
|
harperLogger.debug?.(`Error aborting timed out transaction: ${error.message}`);
|
|
1724
2110
|
}
|
|
@@ -1756,6 +2142,12 @@ export function resetReplayedWritesWarning() {
|
|
|
1756
2142
|
replayedWritesWarned = false;
|
|
1757
2143
|
}
|
|
1758
2144
|
|
|
2145
|
+
export function setMaxOutstandingTxnDuration(ms: number): number {
|
|
2146
|
+
const previous = MAX_OUTSTANDING_TXN_DURATION;
|
|
2147
|
+
MAX_OUTSTANDING_TXN_DURATION = ms;
|
|
2148
|
+
return previous;
|
|
2149
|
+
}
|
|
2150
|
+
|
|
1759
2151
|
/** Test seam: whether the monitor supervises this logical transaction for its writes. */
|
|
1760
2152
|
export function isWriteSupervised(txn: DatabaseTransaction): boolean {
|
|
1761
2153
|
return supervisedWriteRoots.has(txn);
|