@harperfast/harper 5.2.9 → 5.3.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/mcpTools.ts +1 -1
- package/agent/session.ts +25 -14
- package/bin/cliOperations.ts +46 -9
- package/bin/copyDb.ts +282 -59
- package/bin/deploySetup.ts +16 -5
- package/bin/harper.ts +1 -1
- package/bin/help.ts +4 -1
- package/bin/lite.ts +4 -1
- package/bin/restart.ts +59 -3
- package/bin/run.ts +6 -11
- package/bin/upgrade.js +7 -3
- package/bin/workloadIdentity.ts +119 -0
- package/components/Application.ts +2414 -242
- package/components/ApplicationScope.ts +8 -0
- package/components/EntryHandler.ts +59 -39
- package/components/OptionsWatcher.ts +150 -74
- package/components/RuntimeModuleTracker.ts +38 -7
- package/components/Scope.ts +37 -15
- package/components/awaitRestart.ts +84 -0
- package/components/componentLoader.ts +340 -30
- package/components/componentPreparationLock.ts +16 -5
- package/components/mcp/adapters/harperHttp.ts +4 -0
- package/components/mcp/listChanged.ts +4 -0
- package/components/mcp/toolRegistry.ts +2 -0
- package/components/mcp/tools/operations.ts +9 -0
- package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
- package/components/operations.js +270 -78
- package/components/operationsValidation.js +49 -1
- package/components/status/ComponentStatusRegistry.ts +59 -0
- package/config/RootConfigWatcher.ts +80 -34
- package/config/configUtils.ts +291 -34
- package/config/harperConfigEnvVars.ts +170 -27
- package/config-root.schema.json +29 -0
- package/dataLayer/blobBackup.ts +160 -50
- package/dataLayer/delete.ts +6 -1
- package/dataLayer/harperBridge/ResourceBridge.ts +52 -8
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dataLayer/hdbInfoController.ts +34 -1
- package/dataLayer/insert.ts +44 -1
- package/dataLayer/rocksdbBackup.ts +53 -10
- package/dataLayer/schema.ts +11 -1
- package/dataLayer/schemaDescribe.ts +8 -1
- package/dist/agent/mcpTools.js +1 -1
- package/dist/agent/mcpTools.js.map +1 -1
- package/dist/agent/session.d.ts +22 -0
- package/dist/agent/session.js +26 -15
- package/dist/agent/session.js.map +1 -1
- package/dist/bin/cliOperations.js +48 -9
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/copyDb.d.ts +12 -1
- package/dist/bin/copyDb.js +248 -60
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/bin/deploySetup.d.ts +2 -0
- package/dist/bin/deploySetup.js +11 -3
- package/dist/bin/deploySetup.js.map +1 -1
- package/dist/bin/harper.js +1 -1
- package/dist/bin/harper.js.map +1 -1
- package/dist/bin/help.js +4 -1
- package/dist/bin/help.js.map +1 -1
- package/dist/bin/lite.js +4 -1
- package/dist/bin/lite.js.map +1 -1
- package/dist/bin/restart.js +54 -5
- package/dist/bin/restart.js.map +1 -1
- package/dist/bin/run.js +4 -10
- package/dist/bin/run.js.map +1 -1
- package/dist/bin/upgrade.js +4 -3
- package/dist/bin/upgrade.js.map +1 -1
- package/dist/bin/workloadIdentity.d.ts +18 -0
- package/dist/bin/workloadIdentity.js +100 -0
- package/dist/bin/workloadIdentity.js.map +1 -0
- package/dist/components/Application.d.ts +139 -16
- package/dist/components/Application.js +2215 -267
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +8 -0
- package/dist/components/ApplicationScope.js +7 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/EntryHandler.js +26 -10
- package/dist/components/EntryHandler.js.map +1 -1
- package/dist/components/OptionsWatcher.d.ts +1 -0
- package/dist/components/OptionsWatcher.js +141 -74
- package/dist/components/OptionsWatcher.js.map +1 -1
- package/dist/components/RuntimeModuleTracker.js +40 -6
- package/dist/components/RuntimeModuleTracker.js.map +1 -1
- package/dist/components/Scope.js +38 -13
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/awaitRestart.d.ts +33 -0
- package/dist/components/awaitRestart.js +61 -0
- package/dist/components/awaitRestart.js.map +1 -0
- package/dist/components/componentLoader.d.ts +38 -1
- package/dist/components/componentLoader.js +279 -22
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentPreparationLock.d.ts +5 -0
- package/dist/components/componentPreparationLock.js +14 -6
- package/dist/components/componentPreparationLock.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.js +4 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/listChanged.js +5 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +1 -0
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +5 -0
- package/dist/components/mcp/tools/operations.js +9 -0
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/operations.js +231 -77
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.js +49 -1
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
- package/dist/components/status/ComponentStatusRegistry.js +63 -0
- package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
- package/dist/config/RootConfigWatcher.d.ts +7 -1
- package/dist/config/RootConfigWatcher.js +64 -27
- package/dist/config/RootConfigWatcher.js.map +1 -1
- package/dist/config/configUtils.d.ts +9 -1
- package/dist/config/configUtils.js +254 -33
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/harperConfigEnvVars.d.ts +16 -0
- package/dist/config/harperConfigEnvVars.js +162 -25
- package/dist/config/harperConfigEnvVars.js.map +1 -1
- package/dist/dataLayer/blobBackup.d.ts +49 -20
- package/dist/dataLayer/blobBackup.js +139 -50
- package/dist/dataLayer/blobBackup.js.map +1 -1
- package/dist/dataLayer/delete.js +1 -1
- package/dist/dataLayer/delete.js.map +1 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.js +51 -10
- package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
- package/dist/dataLayer/hdbInfoController.d.ts +10 -0
- package/dist/dataLayer/hdbInfoController.js +30 -1
- package/dist/dataLayer/hdbInfoController.js.map +1 -1
- package/dist/dataLayer/insert.d.ts +9 -1
- package/dist/dataLayer/insert.js +30 -0
- package/dist/dataLayer/insert.js.map +1 -1
- package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
- package/dist/dataLayer/rocksdbBackup.js +45 -8
- package/dist/dataLayer/rocksdbBackup.js.map +1 -1
- package/dist/dataLayer/schema.js +8 -0
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +8 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +52 -0
- package/dist/resources/DatabaseTransaction.d.ts +55 -0
- package/dist/resources/DatabaseTransaction.js +459 -45
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.js +22 -4
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
- package/dist/resources/PrimaryRocksDatabase.js +29 -0
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
- package/dist/resources/RecordEncoder.d.ts +20 -0
- package/dist/resources/RecordEncoder.js +94 -9
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +2 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.js +20 -11
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +20 -1
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/RocksIndexStore.d.ts +6 -1
- package/dist/resources/RocksIndexStore.js +24 -9
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.d.ts +14 -1
- package/dist/resources/RocksTransactionLogStore.js +57 -17
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.d.ts +112 -8
- package/dist/resources/Table.js +1073 -191
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/auditStore.d.ts +11 -2
- package/dist/resources/auditStore.js +325 -71
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/blob.d.ts +129 -9
- package/dist/resources/blob.js +992 -114
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/branchDatabase.d.ts +48 -0
- package/dist/resources/branchDatabase.js +892 -0
- package/dist/resources/branchDatabase.js.map +1 -0
- package/dist/resources/crdt.js +50 -12
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/databases.d.ts +150 -0
- package/dist/resources/databases.js +1200 -239
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/defineTable.d.ts +10 -2
- package/dist/resources/defineTable.js +9 -1
- package/dist/resources/defineTable.js.map +1 -1
- package/dist/resources/graphql.d.ts +1 -1
- package/dist/resources/graphql.js +50 -15
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +1 -0
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +6 -2
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/longLivedTransactions.d.ts +71 -0
- package/dist/resources/longLivedTransactions.js +358 -0
- package/dist/resources/longLivedTransactions.js.map +1 -0
- package/dist/resources/models/backendRegistry.d.ts +26 -0
- package/dist/resources/models/backendRegistry.js +60 -2
- package/dist/resources/models/backendRegistry.js.map +1 -1
- package/dist/resources/models/bootstrap.d.ts +33 -1
- package/dist/resources/models/bootstrap.js +416 -31
- package/dist/resources/models/bootstrap.js.map +1 -1
- package/dist/resources/recordLock.d.ts +80 -0
- package/dist/resources/recordLock.js +184 -0
- package/dist/resources/recordLock.js.map +1 -0
- package/dist/resources/replayLogs.d.ts +14 -1
- package/dist/resources/replayLogs.js +152 -24
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/replayLogsGuards.d.ts +94 -7
- package/dist/resources/replayLogsGuards.js +111 -7
- package/dist/resources/replayLogsGuards.js.map +1 -1
- package/dist/resources/search.js +158 -6
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transactionBroadcast.d.ts +1 -1
- package/dist/resources/transactionBroadcast.js +2 -2
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/auth.js +61 -30
- package/dist/security/auth.js.map +1 -1
- package/dist/security/authn/oidc/claims.d.ts +22 -0
- package/dist/security/authn/oidc/claims.js +71 -0
- package/dist/security/authn/oidc/claims.js.map +1 -0
- package/dist/security/authn/oidc/identityToken.d.ts +27 -0
- package/dist/security/authn/oidc/identityToken.js +111 -0
- package/dist/security/authn/oidc/identityToken.js.map +1 -0
- package/dist/security/authn/oidc/jwks.d.ts +25 -0
- package/dist/security/authn/oidc/jwks.js +261 -0
- package/dist/security/authn/oidc/jwks.js.map +1 -0
- package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
- package/dist/security/authn/oidc/providers/generic.js +34 -0
- package/dist/security/authn/oidc/providers/generic.js.map +1 -0
- package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
- package/dist/security/authn/oidc/providers/githubActions.js +129 -0
- package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
- package/dist/security/authn/oidc/providers/index.d.ts +37 -0
- package/dist/security/authn/oidc/providers/index.js +24 -0
- package/dist/security/authn/oidc/providers/index.js.map +1 -0
- package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
- package/dist/security/authn/oidc/tokenExchange.js +306 -0
- package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
- package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
- package/dist/security/authn/oidc/types.d.ts +38 -0
- package/dist/security/authn/oidc/types.js +6 -0
- package/dist/security/authn/oidc/types.js.map +1 -0
- package/dist/security/certificateVerification/index.js +40 -11
- package/dist/security/certificateVerification/index.js.map +1 -1
- package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
- package/dist/security/certificateVerification/trustedIssuers.js +79 -0
- package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
- package/dist/security/certificateVerification/types.d.ts +1 -0
- package/dist/security/credentialProvenance.d.ts +35 -0
- package/dist/security/credentialProvenance.js +51 -0
- package/dist/security/credentialProvenance.js.map +1 -0
- package/dist/security/credentialRejection.d.ts +4 -0
- package/dist/security/credentialRejection.js +24 -0
- package/dist/security/credentialRejection.js.map +1 -0
- package/dist/security/deferredAuthentication.d.ts +36 -0
- package/dist/security/deferredAuthentication.js +70 -0
- package/dist/security/deferredAuthentication.js.map +1 -0
- package/dist/security/impersonation.d.ts +21 -0
- package/dist/security/impersonation.js +108 -9
- package/dist/security/impersonation.js.map +1 -1
- package/dist/security/jsLoader.d.ts +6 -0
- package/dist/security/jsLoader.js +75 -15
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +301 -71
- package/dist/security/keys.js.map +1 -1
- package/dist/security/operationScope.d.ts +21 -0
- package/dist/security/operationScope.js +36 -0
- package/dist/security/operationScope.js.map +1 -0
- package/dist/security/permissionsTranslator.js +21 -0
- package/dist/security/permissionsTranslator.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +19 -1
- package/dist/security/tokenAuthentication.js +191 -10
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/security/user.js +4 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
- package/dist/server/DurableSubscriptionsSession.js +67 -10
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +73 -0
- package/dist/server/REST.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -0
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +11 -0
- package/dist/server/http.js +88 -15
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +8 -1
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/jobs/jobProcess.js +6 -2
- package/dist/server/jobs/jobProcess.js.map +1 -1
- package/dist/server/jobs/jobs.js +4 -1
- package/dist/server/jobs/jobs.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +26 -4
- package/dist/server/liveSubscriptionAuth.js +105 -39
- package/dist/server/liveSubscriptionAuth.js.map +1 -1
- package/dist/server/loadRootComponents.js +49 -10
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/mqtt.d.ts +2 -0
- package/dist/server/mqtt.js +165 -30
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/nodeName.d.ts +2 -0
- package/dist/server/nodeName.js +107 -23
- package/dist/server/nodeName.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +25 -0
- package/dist/server/serverHelpers/Headers.js +139 -1
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.d.ts +9 -0
- package/dist/server/serverHelpers/contentTypes.js +32 -25
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
- package/dist/server/serverHelpers/deployValidationState.js +9 -19
- package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
- package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
- package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
- package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
- package/dist/server/serverHelpers/registeredOperations.js +74 -21
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
- package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
- package/dist/server/serverHelpers/requestSanitization.js +20 -0
- package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +6 -3
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
- package/dist/server/serverHelpers/serverUtilities.js +90 -20
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
- package/dist/server/serverHelpers/uwsServer.js +19 -1
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/static.js +24 -28
- package/dist/server/static.js.map +1 -1
- package/dist/server/storageReclamation.d.ts +12 -0
- package/dist/server/storageReclamation.js +26 -0
- package/dist/server/storageReclamation.js.map +1 -1
- package/dist/server/threads/itc.d.ts +7 -2
- package/dist/server/threads/itc.js +5 -1
- package/dist/server/threads/itc.js.map +1 -1
- package/dist/server/threads/manageThreads.d.ts +26 -3
- package/dist/server/threads/manageThreads.js +517 -65
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.js +89 -26
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadHeapMemory.d.ts +2 -0
- package/dist/server/threads/threadHeapMemory.js +31 -0
- package/dist/server/threads/threadHeapMemory.js.map +1 -0
- package/dist/server/threads/threadServer.js +46 -15
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/sqlEngine/config.d.ts +1 -3
- package/dist/sqlEngine/config.js +19 -16
- package/dist/sqlEngine/config.js.map +1 -1
- package/dist/sqlTranslator/index.d.ts +1 -1
- package/dist/sqlTranslator/index.js +30 -7
- package/dist/sqlTranslator/index.js.map +1 -1
- package/dist/upgrade/directives/5-3-0.d.ts +7 -0
- package/dist/upgrade/directives/5-3-0.js +148 -0
- package/dist/upgrade/directives/5-3-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/OperationFunctionCaller.js +2 -1
- package/dist/utility/OperationFunctionCaller.js.map +1 -1
- package/dist/utility/common_utils.d.ts +16 -0
- package/dist/utility/common_utils.js +32 -6
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/componentNames.d.ts +8 -0
- package/dist/utility/componentNames.js +12 -1
- package/dist/utility/componentNames.js.map +1 -1
- package/dist/utility/environment/environmentManager.js +3 -6
- package/dist/utility/environment/environmentManager.js.map +1 -1
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/environment/systemInformation.js +1 -0
- package/dist/utility/environment/systemInformation.js.map +1 -1
- package/dist/utility/errors/commonErrors.d.ts +2 -0
- package/dist/utility/errors/commonErrors.js +2 -0
- package/dist/utility/errors/commonErrors.js.map +1 -1
- package/dist/utility/errors/hdbError.d.ts +17 -0
- package/dist/utility/errors/hdbError.js +30 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/globalSchema.d.ts +18 -0
- package/dist/utility/hdbTerms.d.ts +16 -0
- package/dist/utility/hdbTerms.js +18 -2
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +2 -0
- package/dist/utility/logging/harper_logger.js +91 -14
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logRotator.js +76 -49
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/nodeIdentity.d.ts +9 -0
- package/dist/utility/nodeIdentity.js +58 -0
- package/dist/utility/nodeIdentity.js.map +1 -0
- package/dist/utility/npmUtilities.js +9 -7
- package/dist/utility/npmUtilities.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +3 -1
- package/dist/utility/operationPermissions.js +16 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +10 -7
- package/dist/utility/operation_authorization.js +212 -41
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/watchPath.d.ts +29 -0
- package/dist/utility/watchPath.js +68 -0
- package/dist/utility/watchPath.js.map +1 -0
- package/dist/utility/watcherFallback.d.ts +86 -0
- package/dist/utility/watcherFallback.js +278 -1
- package/dist/utility/watcherFallback.js.map +1 -1
- package/dist/validation/configValidator.d.ts +12 -0
- package/dist/validation/configValidator.js +199 -72
- package/dist/validation/configValidator.js.map +1 -1
- package/dist/validation/installValidator.js +12 -0
- package/dist/validation/installValidator.js.map +1 -1
- package/dist/validation/validationWrapper.d.ts +11 -0
- package/dist/validation/validationWrapper.js +16 -3
- package/dist/validation/validationWrapper.js.map +1 -1
- package/index.ts +2 -0
- package/json/systemSchema.json +52 -0
- package/npm-shrinkwrap.json +196 -194
- package/package.json +7 -6
- package/resources/DESIGN.md +114 -52
- package/resources/DatabaseTransaction.ts +521 -43
- package/resources/LMDBTransaction.ts +26 -4
- package/resources/PrimaryRocksDatabase.ts +28 -1
- package/resources/RecordEncoder.ts +103 -8
- package/resources/RequestTarget.ts +2 -0
- package/resources/Resource.ts +17 -9
- package/resources/ResourceInterface.ts +23 -0
- package/resources/RocksIndexStore.ts +30 -9
- package/resources/RocksTransactionLogStore.ts +78 -21
- package/resources/Table.ts +1110 -159
- package/resources/auditStore.ts +331 -73
- package/resources/blob.ts +1029 -110
- package/resources/branchDatabase.ts +941 -0
- package/resources/crdt.ts +76 -12
- package/resources/databases.ts +1310 -252
- package/resources/defineTable.ts +18 -2
- package/resources/graphql.ts +70 -16
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +6 -2
- package/resources/longLivedTransactions.ts +360 -0
- package/resources/models/backendRegistry.ts +84 -2
- package/resources/models/bootstrap.ts +473 -28
- package/resources/recordLock.ts +253 -0
- package/resources/replayLogs.ts +147 -26
- package/resources/replayLogsGuards.ts +171 -8
- package/resources/search.ts +154 -7
- package/resources/transactionBroadcast.ts +3 -3
- package/security/auth.ts +68 -29
- package/security/authn/oidc/claims.ts +72 -0
- package/security/authn/oidc/identityToken.ts +129 -0
- package/security/authn/oidc/jwks.ts +260 -0
- package/security/authn/oidc/providers/generic.ts +40 -0
- package/security/authn/oidc/providers/githubActions.ts +137 -0
- package/security/authn/oidc/providers/index.ts +52 -0
- package/security/authn/oidc/tokenExchange.ts +300 -0
- package/security/authn/oidc/trustPolicyOperations.ts +343 -0
- package/security/authn/oidc/types.ts +41 -0
- package/security/certificateVerification/index.ts +54 -13
- package/security/certificateVerification/trustedIssuers.ts +76 -0
- package/security/certificateVerification/types.ts +1 -0
- package/security/credentialProvenance.ts +47 -0
- package/security/credentialRejection.ts +22 -0
- package/security/deferredAuthentication.ts +71 -0
- package/security/impersonation.ts +117 -12
- package/security/jsLoader.ts +81 -18
- package/security/keys.ts +298 -72
- package/security/operationScope.ts +33 -0
- package/security/permissionsTranslator.js +23 -0
- package/security/tokenAuthentication.ts +233 -12
- package/security/user.ts +4 -3
- package/server/DESIGN.md +183 -16
- package/server/DurableSubscriptionsSession.ts +71 -11
- package/server/REST.ts +79 -1
- package/server/graphqlQuerying.ts +4 -0
- package/server/http.ts +99 -18
- package/server/itc/serverHandlers.js +8 -1
- package/server/jobs/jobProcess.ts +8 -2
- package/server/jobs/jobs.ts +4 -1
- package/server/liveSubscriptionAuth.ts +129 -46
- package/server/loadRootComponents.js +50 -8
- package/server/mqtt.ts +179 -38
- package/server/nodeName.ts +103 -21
- package/server/serverHelpers/Headers.ts +131 -0
- package/server/serverHelpers/contentTypes.ts +29 -21
- package/server/serverHelpers/deployValidationState.ts +24 -13
- package/server/serverHelpers/operationAuthorizationState.ts +34 -3
- package/server/serverHelpers/registeredOperations.ts +79 -22
- package/server/serverHelpers/requestSanitization.ts +15 -0
- package/server/serverHelpers/serverHandlers.js +6 -3
- package/server/serverHelpers/serverUtilities.ts +104 -31
- package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
- package/server/serverHelpers/uwsServer.ts +17 -2
- package/server/static.ts +23 -29
- package/server/storageReclamation.ts +23 -0
- package/server/threads/itc.js +11 -1
- package/server/threads/manageThreads.js +526 -56
- package/server/threads/socketRouter.ts +81 -26
- package/server/threads/threadHeapMemory.ts +26 -0
- package/server/threads/threadServer.js +43 -15
- package/sqlTranslator/index.ts +31 -8
- package/studio/web/assets/Chat-BnCBegQz.js +2267 -0
- package/studio/web/assets/FloatingChat-CoDW1ySS.js +23 -0
- package/studio/web/assets/{abnfDiagram-VCTEODGH-C0_BAZyO.js → abnfDiagram-VCTEODGH-B0BebmD2.js} +1 -1
- package/studio/web/assets/{alertDialog-DIHt7Z0r.js → alertDialog-CQyAJJhl.js} +1 -1
- package/studio/web/assets/{apiToken-c3Rd-w6g.js → apiToken-Bwk5BLXW.js} +1 -1
- package/studio/web/assets/applications-DHxGi7JH.js +296 -0
- package/studio/web/assets/architecture-7GRP2DOG-LB-MLAAb.js +1 -0
- package/studio/web/assets/{architectureDiagram-5GKGNRK7-BWzrASgm.js → architectureDiagram-5GKGNRK7-7SW3GD-K.js} +1 -1
- package/studio/web/assets/authStore-C3Nfubqr.js +3 -0
- package/studio/web/assets/{blockDiagram-NRAW4CY4-BdJX9Khj.js → blockDiagram-I7D4REHJ-BqguiadH.js} +2 -2
- package/studio/web/assets/{button-DhiX-njv.js → button-BIsUKRZq.js} +2 -2
- package/studio/web/assets/{c4Diagram-UCG6FXSJ-CI6MzGmQ.js → c4Diagram-7LVT6UL2-LBNf8t_X.js} +1 -1
- package/studio/web/assets/channel-yictG-U-.js +1 -0
- package/studio/web/assets/{chevron-up-Df2c8uoU.js → chevron-up-DtKGqDn3.js} +1 -1
- package/studio/web/assets/{chunk-TEH6E4GO-P87k5mNi.js → chunk-4HAMMTFA-DWtTut21.js} +1 -1
- package/studio/web/assets/{chunk-75Z2AOVW-BT8tVmks.js → chunk-75Z2AOVW-QGQD6th2.js} +1 -1
- package/studio/web/assets/{chunk-DU6HZSFF-9kAOOmI6.js → chunk-DU6HZSFF-Chq20Ba5.js} +1 -1
- package/studio/web/assets/{chunk-F27PBJKO-BW7ao8AY.js → chunk-F27PBJKO-BVA5EPhV.js} +1 -1
- package/studio/web/assets/{chunk-GMAD6QVW-BNyXpoQO.js → chunk-GMAD6QVW-BeS7S07A.js} +1 -1
- package/studio/web/assets/{chunk-OBVCFTLP-D4wWOqDQ.js → chunk-GVQU2GXP-sbwVIQ8i.js} +1 -1
- package/studio/web/assets/{chunk-G27WJ6UU-COyLMcgK.js → chunk-IMKFNOWR-Bnh3tAVd.js} +1 -1
- package/studio/web/assets/{chunk-JQ64N6SF-Cyz1IeLf.js → chunk-L3NEJ4N5-COfUyKII.js} +1 -1
- package/studio/web/assets/chunk-OSK3NFVY-ByciRftO.js +10 -0
- package/studio/web/assets/{chunk-P2QGCYS3-DmIFY4d7.js → chunk-P2QGCYS3-CP1VhG_c.js} +1 -1
- package/studio/web/assets/{chunk-POPQ4Y6H-BPrvMyKz.js → chunk-POPQ4Y6H-ClWhhkwW.js} +1 -1
- package/studio/web/assets/{chunk-PWAF6VOD-2zB6IW9i.js → chunk-PWAF6VOD-1z1THyS5.js} +1 -1
- package/studio/web/assets/{chunk-RHFEMEQ7-2FgyI8YU.js → chunk-SHT3W25Y-LpQkMsah.js} +2 -2
- package/studio/web/assets/{chunk-SVP7TREG-FwtbH2QC.js → chunk-SVP7TREG-jtdAHw0S.js} +1 -1
- package/studio/web/assets/{chunk-LCL6LL3I-HOzK_ppE.js → chunk-TICWLB2K-VOwzetX-.js} +1 -1
- package/studio/web/assets/classDiagram-ZZMXUADV-VaEwSy_g.js +1 -0
- package/studio/web/assets/classDiagram-v2-VYDZK3BY-VaEwSy_g.js +1 -0
- package/studio/web/assets/{createLucideIcon-BKGPfjm2.js → createLucideIcon-CzW9508A.js} +1 -1
- package/studio/web/assets/{cssMode-CEN2mzSA.js → cssMode-s0cWI-_M.js} +1 -1
- package/studio/web/assets/{cynefin-OW5HDTMX-BRkpLFQV.js → cynefin-OW5HDTMX-BbdbCvub.js} +1 -1
- package/studio/web/assets/{cynefinDiagram-5FMLGOSQ-CHT1DaX6.js → cynefinDiagram-5FMLGOSQ-TP-aIqbt.js} +1 -1
- package/studio/web/assets/{dagre-3AP2YEHR-DpUXBh63.js → dagre-GXQ25YYZ-DShnGpGo.js} +1 -1
- package/studio/web/assets/{diagram-S7CK7UJ4-BuymVFZT.js → diagram-S7CK7UJ4-aoCVTtcy.js} +1 -1
- package/studio/web/assets/{diagram-UQ7AKVKN-CyP148RM.js → diagram-UQ7AKVKN-DglXtQ6x.js} +1 -1
- package/studio/web/assets/{diagram-VSXAHHWV-CoCAg3M9.js → diagram-VSXAHHWV-fhEdmkwM.js} +1 -1
- package/studio/web/assets/{diagram-VX7I27RA-BpOqCFca.js → diagram-VX7I27RA-DccVJet6.js} +1 -1
- package/studio/web/assets/{diagram-Z3DM3KII-Bfpw7Vbj.js → diagram-Z3DM3KII-D-RyJJb7.js} +1 -1
- package/studio/web/assets/{dialog-CBf0Mr1d.js → dialog-Cn2uWgD4.js} +1 -1
- package/studio/web/assets/{dist-lkA3O3eM.js → dist-DP8UjMB_.js} +1 -1
- package/studio/web/assets/{download-BtTOBem-.js → download-B5T5r7ss.js} +1 -1
- package/studio/web/assets/{ebnfDiagram-PWID7BFC-DS_6aWqL.js → ebnfDiagram-PWID7BFC-DJGpIpz_.js} +1 -1
- package/studio/web/assets/{editor-D8oDeCTL.js → editor-DNcRHK54.js} +1 -1
- package/studio/web/assets/{erDiagram-SSCWMZ5O-DJNk6Fgw.js → erDiagram-RLTQ6QDP-CIfNlgkC.js} +1 -1
- package/studio/web/assets/eventmodeling-NTZA5JFV-CLxnp2CR.js +1 -0
- package/studio/web/assets/flowDiagram-HODETNUW-BIbhmz9f.js +1 -0
- package/studio/web/assets/{ganttDiagram-EL5Y4UJY-2pOExxMY.js → ganttDiagram-EL5Y4UJY-BxToTzzD.js} +1 -1
- package/studio/web/assets/{getAnalytics-D4LKGeVy.js → getAnalytics-GHK8ORfM.js} +1 -1
- package/studio/web/assets/{gitGraph-4MIJSDKK-CH5ZxwzF.js → gitGraph-4MIJSDKK-D2s2w8lE.js} +1 -1
- package/studio/web/assets/{gitGraphDiagram-WWUBYQGX-DVIsIhbO.js → gitGraphDiagram-WWUBYQGX-Dwntd4-x.js} +1 -1
- package/studio/web/assets/{html-u3vOg7LJ.js → html-Bdssedlg.js} +1 -1
- package/studio/web/assets/{htmlMode-DyO31v-P.js → htmlMode-CoDlJ3fw.js} +1 -1
- package/studio/web/assets/{index-Cxj2_wsl.css → index-7RMEgVG1.css} +1 -1
- package/studio/web/assets/index-D6sxmFLR.js +824 -0
- package/studio/web/assets/index.lazy-tmU5BS8s.js +2 -0
- package/studio/web/assets/{info-A6RAGUB7-CPQfTnaG.js → info-A6RAGUB7-DYjkvb0C.js} +1 -1
- package/studio/web/assets/{infoDiagram-RXCK75RN-DlwLYlwm.js → infoDiagram-27XIBGKW-Bnp1FJE5.js} +1 -1
- package/studio/web/assets/{ishikawaDiagram-5VMMS53U-BRXRp29U.js → ishikawaDiagram-5VMMS53U-D9Xh2r6X.js} +1 -1
- package/studio/web/assets/{javascript-CUvxOyTC.js → javascript-B8meVSTH.js} +1 -1
- package/studio/web/assets/{journeyDiagram-EYS64GPL-B0ou8k0n.js → journeyDiagram-3NMN7TZE-CokIi6ll.js} +2 -2
- package/studio/web/assets/{jsonMode-f_IwbF3D.js → jsonMode-DpIPd35T.js} +1 -1
- package/studio/web/assets/{kanban-definition-3QL26DDD-uYg7iYzp.js → kanban-definition-UXKFOSKX-CukSFJfX.js} +1 -1
- package/studio/web/assets/{languageServices-DXtZ6rEF.js → languageServices-C_5FMJzQ.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-B4pCF1zO.js → lspLanguageFeatures-BIzNBzPK.js} +1 -1
- package/studio/web/assets/{mermaid-parser.core-Ck-fC8b7.js → mermaid-parser.core-BlEsOWNO.js} +3 -3
- package/studio/web/assets/{mermaid.core-CP8aNNYm.js → mermaid.core-BlkGaMIH.js} +5 -5
- package/studio/web/assets/{mindmap-definition-FBJOCRG2-CgTZ-rit.js → mindmap-definition-YA3MSWOX-IprMc_0j.js} +1 -1
- package/studio/web/assets/{notifications-D3tIQ4sg.js → notifications-CQf18QKb.js} +1 -1
- package/studio/web/assets/notifications-CvZivSbh.js +1 -0
- package/studio/web/assets/{packet-AYTQ26CC-DEyoPtPb.js → packet-AYTQ26CC-Bi3V04Zi.js} +1 -1
- package/studio/web/assets/{pegDiagram-XKGWAZYB-DrD-7sD9.js → pegDiagram-XKGWAZYB-BNuPDLZY.js} +1 -1
- package/studio/web/assets/{pie-WAS4IAKB-wjj-EI1d.js → pie-WAS4IAKB-_6DoDbng.js} +1 -1
- package/studio/web/assets/{pieDiagram-E7YTZNPT-GntqDCzv.js → pieDiagram-E7YTZNPT-DqNb6Ht2.js} +1 -1
- package/studio/web/assets/{profile-DZWU7MgT.js → profile-DdOwtntb.js} +1 -1
- package/studio/web/assets/{quadrantDiagram-AXDQQJYC-0UeqXQGd.js → quadrantDiagram-AXDQQJYC-BGH9E2YR.js} +1 -1
- package/studio/web/assets/{radar-RG4KPBEZ-DFSA5h7k.js → radar-RG4KPBEZ-DDdVczcL.js} +1 -1
- package/studio/web/assets/{railroad-74A4TZTK-CaOUG9wR.js → railroad-74A4TZTK-BJUP4Jds.js} +1 -1
- package/studio/web/assets/railroad-abnf-HS5TGJTU-Bm3L1L0h.js +1 -0
- package/studio/web/assets/railroad-ebnf-LZEXJU2U-CKzLGlkw.js +1 -0
- package/studio/web/assets/railroad-peg-WCYAUIDC-S8xLjslx.js +1 -0
- package/studio/web/assets/{railroadDiagram-O6MQD6OU-yHUELZaV.js → railroadDiagram-O6MQD6OU-DGTPh2KZ.js} +1 -1
- package/studio/web/assets/{regions-CkyurXzE.js → regions-n69fwagr.js} +1 -1
- package/studio/web/assets/{register-BUyhWjBO.js → register-PfWTCXWB.js} +3 -3
- package/studio/web/assets/{requirementDiagram-EFPCY7ZU-DNEGFjuW.js → requirementDiagram-BXWQKSXE-BJnO6uLz.js} +1 -1
- package/studio/web/assets/{sankeyDiagram-P5KCCOFB-DpyAmSVR.js → sankeyDiagram-P5KCCOFB-0vSOdymH.js} +1 -1
- package/studio/web/assets/{sequenceDiagram-WJ2MYXX4-TyaT7xNk.js → sequenceDiagram-WJ2MYXX4-hETizDWE.js} +1 -1
- package/studio/web/assets/{setComponentFile-CeyKSZAa.js → setComponentFile-Bg6O7X0S.js} +1 -1
- package/studio/web/assets/{setup-D2kn7cAA.js → setup-CUx_aUDl.js} +2 -2
- package/studio/web/assets/{stateDiagram-HBIQ2CUA-CeEdTArZ.js → stateDiagram-D77RDMKH-CdYQ_KtC.js} +1 -1
- package/studio/web/assets/stateDiagram-v2-MP3YSRHH-CdKuzQMT.js +1 -0
- package/studio/web/assets/status-D7BVKqX9.js +61 -0
- package/studio/web/assets/{swimlanes-XN3QIQJK-B54FmF46.js → swimlanes-42K2YHIH-B8cHIpU4.js} +1 -1
- package/studio/web/assets/swimlanesDiagram-VR7AAH4N-DmOSJwaH.js +8 -0
- package/studio/web/assets/{tabs-B_G5zscN.js → tabs-BrHu7gJi.js} +1 -1
- package/studio/web/assets/{timeline-definition-24CTP7MA-D-a9ujbo.js → timeline-definition-24CTP7MA-BJWYSXqF.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-C0UW6rI2.js → toggleHighContrast-DBSyXzMr.js} +1 -1
- package/studio/web/assets/{treeView-Q6P3EWNA-CrW_6JnS.js → treeView-Q6P3EWNA-qxe_v6CQ.js} +1 -1
- package/studio/web/assets/{treemap-WGGIJYW6-BxyYLdP_.js → treemap-WGGIJYW6-dDo97XXF.js} +1 -1
- package/studio/web/assets/{tsMode-DBC0zmDx.js → tsMode-BByKCjBS.js} +1 -1
- package/studio/web/assets/{typescript-DApRQir3.js → typescript-DDLnLpw9.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-DB6JStU1.js → useEntityRestURL-31CHGaHk.js} +1 -1
- package/studio/web/assets/{useLocalStorage-Dtj1QS8_.js → useLocalStorage-BqMR3D8_.js} +1 -1
- package/studio/web/assets/vendor-core-c2JRRJpV.js +58 -0
- package/studio/web/assets/vendor-datadog-CLUcJXOo.js +6 -0
- package/studio/web/assets/{vendor-react-Dyj4O3HE.js → vendor-react-CJV_K1u4.js} +1 -1
- package/studio/web/assets/vendor-tanstack-DxzraizX.js +1 -0
- package/studio/web/assets/{vendor-ui-vhu-UHhF.js → vendor-ui-BUjK0h8a.js} +2 -2
- package/studio/web/assets/{vennDiagram-4TSXK5OY-Cy7s7Mpy.js → vennDiagram-4TSXK5OY-A3i-lCdl.js} +1 -1
- package/studio/web/assets/{wardley-WFR3VGLG-BeBL35g2.js → wardley-WFR3VGLG-B0ik-_6g.js} +1 -1
- package/studio/web/assets/{wardleyDiagram-VM6X3IG4-BylmIGSg.js → wardleyDiagram-VM6X3IG4-CjrkKWUR.js} +1 -1
- package/studio/web/assets/{workers-C0bFIedw.js → workers-pR3jRY9D.js} +1 -1
- package/studio/web/assets/x-DIzaLEdK.js +1 -0
- package/studio/web/assets/{xml-HWd01lU-.js → xml-2iRnMhQO.js} +1 -1
- package/studio/web/assets/{xychartDiagram-S5SC5T6Z-CoKALMXr.js → xychartDiagram-S5SC5T6Z-Biok4GYV.js} +1 -1
- package/studio/web/assets/{yaml-CIH0Nt-h.js → yaml-Bf92gJpd.js} +1 -1
- package/studio/web/index.html +14 -14
- package/upgrade/directives/5-3-0.ts +132 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/OperationFunctionCaller.ts +2 -1
- package/utility/common_utils.ts +30 -5
- package/utility/componentNames.ts +12 -0
- package/utility/environment/environmentManager.ts +3 -7
- package/utility/environment/systemInformation.ts +7 -0
- package/utility/errors/commonErrors.ts +4 -0
- package/utility/errors/hdbError.ts +29 -0
- package/utility/hdbTerms.ts +17 -0
- package/utility/logging/harper_logger.ts +87 -14
- package/utility/logging/logRotator.ts +72 -45
- package/utility/nodeIdentity.ts +45 -0
- package/utility/npmUtilities.ts +10 -8
- package/utility/operationPermissions.ts +18 -1
- package/utility/operation_authorization.ts +231 -42
- package/utility/watchPath.ts +63 -0
- package/utility/watcherFallback.ts +270 -0
- package/validation/configValidator.ts +209 -72
- package/validation/installValidator.ts +15 -0
- package/validation/validationWrapper.ts +18 -4
- package/studio/web/assets/Chat-JpO8EtUu.js +0 -2067
- package/studio/web/assets/FloatingChat-Bcj3xSZu.js +0 -23
- package/studio/web/assets/applications-ByqLRKyZ.js +0 -296
- package/studio/web/assets/architecture-7GRP2DOG-DNdx5tEU.js +0 -1
- package/studio/web/assets/authStore-qKmCZcaf.js +0 -3
- package/studio/web/assets/channel-DtCV8PTL.js +0 -1
- package/studio/web/assets/chunk-R7TYR2AO-Irip67yr.js +0 -10
- package/studio/web/assets/classDiagram-DTDB5LWJ-DbO_dCNE.js +0 -1
- package/studio/web/assets/classDiagram-v2-JRS7N3AN-DbO_dCNE.js +0 -1
- package/studio/web/assets/eventmodeling-NTZA5JFV-5jbe4A5P.js +0 -1
- package/studio/web/assets/flowDiagram-A5DVABFB-Dp9Ezlow.js +0 -1
- package/studio/web/assets/index-aSt5tY-L.js +0 -824
- package/studio/web/assets/index.lazy-B9jiPwT8.js +0 -2
- package/studio/web/assets/notifications-CUoYgU98.js +0 -1
- package/studio/web/assets/railroad-abnf-HS5TGJTU-Bc0Qi0WH.js +0 -1
- package/studio/web/assets/railroad-ebnf-LZEXJU2U-G8rVVZ2C.js +0 -1
- package/studio/web/assets/railroad-peg-WCYAUIDC-CrehKBhC.js +0 -1
- package/studio/web/assets/stateDiagram-v2-4QOOHH4V-D4tuw9Su.js +0 -1
- package/studio/web/assets/status-D7Xn5ePA.js +0 -61
- package/studio/web/assets/swimlanesDiagram-VK2B7HYN-XOhmNEvq.js +0 -8
- package/studio/web/assets/vendor-core-RCcadM3e.js +0 -73
- package/studio/web/assets/vendor-datadog-BRv-mOv1.js +0 -6
- package/studio/web/assets/vendor-tanstack-BiFWSB3W.js +0 -1
- package/studio/web/assets/x-B9o9hsep.js +0 -1
- /package/studio/web/assets/{sizeCapture-X5ZJPWSS-B0uUizjq.js → sizeCapture-INFHLROL-B0uUizjq.js} +0 -0
package/resources/Table.ts
CHANGED
|
@@ -14,7 +14,8 @@ import {
|
|
|
14
14
|
import { type Database } from 'lmdb';
|
|
15
15
|
import { Script } from 'node:vm';
|
|
16
16
|
import { randomUUID } from 'node:crypto';
|
|
17
|
-
import {
|
|
17
|
+
import { performance } from 'node:perf_hooks';
|
|
18
|
+
import { getIndexedValues, getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
|
|
18
19
|
import { getThisNodeId, exportIdMapping } from './nodeIdMapping.ts';
|
|
19
20
|
import lodash from 'lodash';
|
|
20
21
|
import { ExtendedIterable, SKIP } from '@harperfast/extended-iterable';
|
|
@@ -41,6 +42,14 @@ import {
|
|
|
41
42
|
TRANSACTION_STATE,
|
|
42
43
|
writeKeyId,
|
|
43
44
|
} from './DatabaseTransaction.ts';
|
|
45
|
+
import {
|
|
46
|
+
acquireRecordKey,
|
|
47
|
+
lockAttemptKey,
|
|
48
|
+
lockNotHeldError,
|
|
49
|
+
resolveLockOptions,
|
|
50
|
+
type RecordLockHandle,
|
|
51
|
+
type RecordLockOptions,
|
|
52
|
+
} from './recordLock.ts';
|
|
44
53
|
import * as envMngr from '../utility/environment/environmentManager.ts';
|
|
45
54
|
import { addSubscription } from './transactionBroadcast.ts';
|
|
46
55
|
import {
|
|
@@ -49,6 +58,7 @@ import {
|
|
|
49
58
|
ServerError,
|
|
50
59
|
AccessViolation,
|
|
51
60
|
ValidationError,
|
|
61
|
+
UpdateAttributesLockTimeoutError,
|
|
52
62
|
type ValidationIssue,
|
|
53
63
|
} from '../utility/errors/hdbError.ts';
|
|
54
64
|
import * as signalling from '../utility/signalling.ts';
|
|
@@ -58,6 +68,7 @@ import {
|
|
|
58
68
|
searchByIndex,
|
|
59
69
|
findAttribute,
|
|
60
70
|
estimateCondition,
|
|
71
|
+
estimatedEntryCount,
|
|
61
72
|
flattenKey,
|
|
62
73
|
COERCIBLE_OPERATORS,
|
|
63
74
|
executeConditions,
|
|
@@ -78,6 +89,7 @@ import {
|
|
|
78
89
|
PENDING_LOCAL_TIME,
|
|
79
90
|
RecordObject,
|
|
80
91
|
type Entry,
|
|
92
|
+
type StructureCounts,
|
|
81
93
|
entryMap,
|
|
82
94
|
storedFieldsOnly,
|
|
83
95
|
} from './RecordEncoder.ts';
|
|
@@ -86,7 +98,12 @@ import { rebuildUpdateBefore } from './crdt.ts';
|
|
|
86
98
|
import { appendHeader } from '../server/serverHelpers/Headers.ts';
|
|
87
99
|
import fs from 'node:fs';
|
|
88
100
|
import { Blob, deleteBlobsInObject, findBlobsInObject, startPreCommitBlobsForRecord } from './blob.ts';
|
|
89
|
-
import {
|
|
101
|
+
import {
|
|
102
|
+
onStorageReclamation,
|
|
103
|
+
removeStorageReclamation,
|
|
104
|
+
removeStorageReclamationHandler,
|
|
105
|
+
getStorageSpaceStats,
|
|
106
|
+
} from '../server/storageReclamation.ts';
|
|
90
107
|
import { RequestTarget } from './RequestTarget.ts';
|
|
91
108
|
import harperLogger from '../utility/logging/harper_logger.ts';
|
|
92
109
|
import { throttle } from '../server/throttle.ts';
|
|
@@ -130,7 +147,10 @@ type MaybePromise<T> = T | Promise<T>;
|
|
|
130
147
|
const NULL_WITH_TIMESTAMP = new Uint8Array(9);
|
|
131
148
|
NULL_WITH_TIMESTAMP[8] = 0xc0; // null
|
|
132
149
|
const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification
|
|
150
|
+
const MAX_DATE_TIMESTAMP = 8.64e15;
|
|
133
151
|
const RECORD_PRUNING_INTERVAL = 60000; // one minute
|
|
152
|
+
const MAX_CONCURRENT_HISTORY_REMOVALS = 10;
|
|
153
|
+
const MAX_CONCURRENT_LMDB_HISTORY_REMOVALS = 1000;
|
|
134
154
|
// RocksDB-only: number of eviction/tombstone removals coalesced into a single transaction commit.
|
|
135
155
|
// Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already
|
|
136
156
|
// coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path.
|
|
@@ -139,9 +159,93 @@ const EVICTION_BATCH_SIZE = 100;
|
|
|
139
159
|
// letting an unbounded number of open transactions (and their snapshots) accumulate.
|
|
140
160
|
const MAX_INFLIGHT_EVICTION_BATCHES = 4;
|
|
141
161
|
const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
|
|
162
|
+
// Guardrails for `Prefer: count=exact`: once the requested page has been collected, counting the rest
|
|
163
|
+
// of the match set is bounded by BOTH a row cap and a wall-clock budget, so a paginated read can't turn
|
|
164
|
+
// into an unbounded scan. Exceeding either reports an unknown total (Content-Range `.../*`) rather than
|
|
165
|
+
// truncating the page. These bound the count tail, not the page itself; a genuinely expensive query
|
|
166
|
+
// (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure.
|
|
167
|
+
const MAX_EXACT_COUNT_SCAN = 1_000_000;
|
|
168
|
+
const MAX_EXACT_COUNT_MS = 1_000;
|
|
169
|
+
// Largest page a `Prefer: count=` request will materialize. A request whose limit exceeds this (or is
|
|
170
|
+
// not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal
|
|
171
|
+
// streaming path with no count, so a count request can't be coerced into buffering an unbounded page.
|
|
172
|
+
const MAX_COUNT_PAGE = 10_000;
|
|
173
|
+
// How often the exact-count drain yields to the macrotask queue (must be a power of two for the bit-mask
|
|
174
|
+
// check). Keeps a large scan from monopolizing the event loop without adding a yield per row.
|
|
175
|
+
const COUNT_YIELD_INTERVAL = 2_048;
|
|
142
176
|
envMngr.initSync();
|
|
143
177
|
const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
|
|
144
178
|
const LOCK_TIMEOUT = 10000;
|
|
179
|
+
// This bounds schema-lock acquisition; LOCK_TIMEOUT bounds in-flight record writes during a drop.
|
|
180
|
+
export const UPDATE_ATTRIBUTES_LOCK_TIMEOUT = 10000;
|
|
181
|
+
const UPDATE_ATTRIBUTES_LOCK = 'update-attributes';
|
|
182
|
+
// Contention is otherwise only visible once it becomes a timeout (harper#2251).
|
|
183
|
+
export const UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT = 1000;
|
|
184
|
+
// raw ASCII bytes are ordered-binary's encoding of the string, so this addresses the same native
|
|
185
|
+
// lock as string-keyed tryLock/unlock calls
|
|
186
|
+
const updateAttributesLockKey = Buffer.from(UPDATE_ATTRIBUTES_LOCK);
|
|
187
|
+
const lockWait = new Int32Array(new SharedArrayBuffer(4));
|
|
188
|
+
|
|
189
|
+
/** The wait blocks the event loop, so the locked section must stay synchronous. */
|
|
190
|
+
export function acquireUpdateAttributesLock(
|
|
191
|
+
rootStore: RocksDatabase,
|
|
192
|
+
scopeDescription: string,
|
|
193
|
+
timeout = UPDATE_ATTRIBUTES_LOCK_TIMEOUT
|
|
194
|
+
) {
|
|
195
|
+
if (rootStore.tryLock(updateAttributesLockKey)) return;
|
|
196
|
+
const startTime = performance.now();
|
|
197
|
+
let waitTime = 1;
|
|
198
|
+
while (!rootStore.tryLock(updateAttributesLockKey)) {
|
|
199
|
+
const elapsed = performance.now() - startTime;
|
|
200
|
+
if (elapsed >= timeout) {
|
|
201
|
+
throw new UpdateAttributesLockTimeoutError(
|
|
202
|
+
`Timed out after ${Math.round(elapsed)}ms waiting for the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription}; the lock holder did not release it before the deadline, so this schema/attribute update cannot proceed`
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
if (elapsed >= 2) {
|
|
206
|
+
Atomics.wait(lockWait, 0, 0, Math.min(waitTime, timeout - elapsed));
|
|
207
|
+
if (waitTime < 16) waitTime *= 2;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const waited = performance.now() - startTime;
|
|
211
|
+
// The caller cannot register its release until we return, so a throw here would leak the lock
|
|
212
|
+
// with no `finally` able to reach it.
|
|
213
|
+
if (waited >= UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT)
|
|
214
|
+
try {
|
|
215
|
+
logger.warn?.(
|
|
216
|
+
`Acquired the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription} after waiting ${Math.round(waited)}ms; this worker's event loop was blocked for that wait, and a holder that runs past ${UPDATE_ATTRIBUTES_LOCK_TIMEOUT}ms fails the update outright`
|
|
217
|
+
);
|
|
218
|
+
} catch {}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export function releaseUpdateAttributesLock(rootStore: RocksDatabase) {
|
|
222
|
+
rootStore.unlock(updateAttributesLockKey);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
export function withUpdateAttributesLock<Callback extends () => unknown>(
|
|
226
|
+
rootStore: RocksDatabase,
|
|
227
|
+
scopeDescription: string,
|
|
228
|
+
callback: Callback & (ReturnType<Callback> extends PromiseLike<unknown> ? never : unknown)
|
|
229
|
+
): ReturnType<Callback> {
|
|
230
|
+
acquireUpdateAttributesLock(rootStore, scopeDescription);
|
|
231
|
+
try {
|
|
232
|
+
const result = callback();
|
|
233
|
+
if (typeof (result as any)?.then === 'function') {
|
|
234
|
+
Promise.resolve(result).catch((error) =>
|
|
235
|
+
logger.error?.(
|
|
236
|
+
`Async update-attributes callback rejected after its lock was released (${scopeDescription})`,
|
|
237
|
+
error
|
|
238
|
+
)
|
|
239
|
+
);
|
|
240
|
+
throw new TypeError(
|
|
241
|
+
`withUpdateAttributesLock callback must be synchronous (${scopeDescription}); asynchronous work may continue after the lock is released`
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
return result as ReturnType<Callback>;
|
|
245
|
+
} finally {
|
|
246
|
+
releaseUpdateAttributesLock(rootStore);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
145
249
|
// Tolerate a redundant column family drop. Drops are broadcast to every worker
|
|
146
250
|
// thread and each holds its own handle to the same underlying family, so a
|
|
147
251
|
// concurrent worker may already have dropped it; the storage engine reports
|
|
@@ -380,6 +484,24 @@ function chainKeyForId(id: any): string {
|
|
|
380
484
|
return typeof id === 'string' ? 's' + id : 'k' + writeKeyId(id);
|
|
381
485
|
}
|
|
382
486
|
|
|
487
|
+
/** Normalizes a passed `context` argument as `transactional()` does; undefined means fall back to ambient. */
|
|
488
|
+
function contextArgument(context: unknown): any {
|
|
489
|
+
if (!context || isReleasedTransaction(context)) return undefined;
|
|
490
|
+
const resolved = (context as any).getContext?.() || context;
|
|
491
|
+
return resolved instanceof DatabaseTransaction ? { transaction: resolved } : resolved;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/** Distinguishes bare lock options from a record target (id, URL, {id:...}). */
|
|
495
|
+
function isPlainOptions(value: unknown): boolean {
|
|
496
|
+
return (
|
|
497
|
+
typeof value === 'object' &&
|
|
498
|
+
value !== null &&
|
|
499
|
+
!Array.isArray(value) &&
|
|
500
|
+
!(value instanceof URLSearchParams) &&
|
|
501
|
+
(value as any).id === undefined
|
|
502
|
+
);
|
|
503
|
+
}
|
|
504
|
+
|
|
383
505
|
export function makeTable(options) {
|
|
384
506
|
const {
|
|
385
507
|
primaryKey,
|
|
@@ -398,6 +520,7 @@ export function makeTable(options) {
|
|
|
398
520
|
description,
|
|
399
521
|
hidden,
|
|
400
522
|
cacheControl,
|
|
523
|
+
isBranch,
|
|
401
524
|
} = options;
|
|
402
525
|
let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
|
|
403
526
|
evictionMs ??= 0;
|
|
@@ -410,6 +533,7 @@ export function makeTable(options) {
|
|
|
410
533
|
if (!properties) properties = projectAttributesToProperties(attributes);
|
|
411
534
|
const updateRecord = recordUpdater(primaryStore, tableId, auditStore);
|
|
412
535
|
let warnedNullSourcePut = false; // latched: one warn per table per worker (see _writeUpdate)
|
|
536
|
+
let warnedFutureSourceVersion = false; // likewise (see getFromSource)
|
|
413
537
|
let sourceLoad: any; // if a source has a load function (replicator), record it here
|
|
414
538
|
let hasSourceGet: any;
|
|
415
539
|
let primaryKeyAttribute: Attribute | undefined;
|
|
@@ -441,6 +565,10 @@ export function makeTable(options) {
|
|
|
441
565
|
let cleanupPriority = 0;
|
|
442
566
|
let lastCleanupInterval: number;
|
|
443
567
|
let cleanupTimer: NodeJS.Timeout;
|
|
568
|
+
let recordExpirationInterval: NodeJS.Timeout;
|
|
569
|
+
// a reclamation pass awaits a scheduled cleanup, which only settles from its timer
|
|
570
|
+
const pendingCleanupResolvers = new Set<() => void>();
|
|
571
|
+
let disposed = false;
|
|
444
572
|
// true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup
|
|
445
573
|
let expirationScanScheduled = false;
|
|
446
574
|
// set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table
|
|
@@ -489,9 +617,10 @@ export function makeTable(options) {
|
|
|
489
617
|
const MAX_PREFETCH_SEQUENCE = 10;
|
|
490
618
|
const MAX_PREFETCH_BUNDLE = 6;
|
|
491
619
|
if (audit) addDeleteRemoval();
|
|
492
|
-
|
|
620
|
+
const reclamationHandler = (priority: number) => {
|
|
493
621
|
if (hasSourceGet) return scheduleCleanup(priority);
|
|
494
|
-
}
|
|
622
|
+
};
|
|
623
|
+
onStorageReclamation(primaryStore.path, reclamationHandler);
|
|
495
624
|
|
|
496
625
|
class Updatable extends GenericTrackedObject implements RecordObject {
|
|
497
626
|
declare set: (property: string, value: any) => void;
|
|
@@ -582,13 +711,67 @@ export function makeTable(options) {
|
|
|
582
711
|
},
|
|
583
712
|
});
|
|
584
713
|
}
|
|
714
|
+
function resolveAuditHead(
|
|
715
|
+
id: Id,
|
|
716
|
+
version: number | undefined,
|
|
717
|
+
nodeId: number | undefined,
|
|
718
|
+
refs?: Array<{ version: number; nodeId: number }>
|
|
719
|
+
) {
|
|
720
|
+
if (!refs?.length) return { txnLogKey: version, nodeId };
|
|
721
|
+
const visited = new Set<string>();
|
|
722
|
+
function findHead(candidateRefs?: Array<{ version: number; nodeId: number }>) {
|
|
723
|
+
if (!candidateRefs) return;
|
|
724
|
+
const pending: Array<{ version: number; nodeId: number }> = candidateRefs.slice().reverse();
|
|
725
|
+
while (pending.length > 0) {
|
|
726
|
+
const ref = pending.pop()!;
|
|
727
|
+
const identity = `${ref.nodeId ?? 0}:${ref.version}`;
|
|
728
|
+
if (visited.has(identity)) continue;
|
|
729
|
+
visited.add(identity);
|
|
730
|
+
const entry = auditStore.getSync(ref.version, tableId, id, ref.nodeId);
|
|
731
|
+
if (!entry) continue;
|
|
732
|
+
if (entry.version === version && (nodeId == null || (entry.nodeId ?? 0) === nodeId))
|
|
733
|
+
return { txnLogKey: ref.version, nodeId: ref.nodeId };
|
|
734
|
+
const previousRefs = entry.previousAdditionalAuditRefs;
|
|
735
|
+
if (previousRefs) {
|
|
736
|
+
for (let index = previousRefs.length - 1; index >= 0; index--) pending.push(previousRefs[index]);
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
const referencedHead = findHead(refs);
|
|
741
|
+
if (referencedHead) return referencedHead;
|
|
742
|
+
if (version != null) {
|
|
743
|
+
const directHead = auditStore.getSync(version, tableId, id, nodeId);
|
|
744
|
+
if (directHead?.version === version && (nodeId == null || (directHead.nodeId ?? 0) === nodeId))
|
|
745
|
+
return { txnLogKey: version, nodeId };
|
|
746
|
+
}
|
|
747
|
+
return { txnLogKey: version, nodeId };
|
|
748
|
+
}
|
|
585
749
|
class TableResource<Record extends object = any> extends Resource<Record> {
|
|
586
750
|
#record: any; // the stored/frozen record from the database and stored in the cache (should not be modified directly)
|
|
587
751
|
#changes: any; // the changes to the record that have been made (should not be modified directly)
|
|
588
752
|
#version?: number; // version of the record
|
|
589
753
|
#entry?: Entry; // the entry from the database
|
|
590
754
|
#savingOperation?: any; // operation for the record is currently being saved
|
|
755
|
+
#lockHandle?: RecordLockHandle; // the record lock acquired by lock() — scoped or hold
|
|
756
|
+
#lockWritable?: boolean; // set by #reloadLocked to let save() stage lock-writable updates
|
|
591
757
|
declare getProperty: (name: string) => any;
|
|
758
|
+
|
|
759
|
+
/**
|
|
760
|
+
* Shared guard: if this instance is lock-writable but the handle is gone (expired or
|
|
761
|
+
* released), throw 409 before staging any write. Covers update/invalidate/relocate/delete
|
|
762
|
+
* in addition to the save() path. Every lock-writable instance carries its own handle in
|
|
763
|
+
* #lockHandle (scoped and hold alike), so we never need to search the registry here.
|
|
764
|
+
*/
|
|
765
|
+
#assertLiveHandle(id: Id): void {
|
|
766
|
+
if (!this.#lockWritable) return;
|
|
767
|
+
const handle = this.#lockHandle!;
|
|
768
|
+
// Off-key writes through the same resource instance are ordinary; only guard the
|
|
769
|
+
// exact key the lock was acquired for.
|
|
770
|
+
if (handle.keyId !== writeKeyId(id)) return;
|
|
771
|
+
if (handle.expired || handle.released) {
|
|
772
|
+
throw lockNotHeldError(handle);
|
|
773
|
+
}
|
|
774
|
+
}
|
|
592
775
|
// #section: static-config
|
|
593
776
|
static name = tableName; // for display/debugging purposes
|
|
594
777
|
static primaryStore = primaryStore;
|
|
@@ -704,6 +887,9 @@ export function makeTable(options) {
|
|
|
704
887
|
ensureLoaded: false,
|
|
705
888
|
nodeId: event.nodeId,
|
|
706
889
|
viaNodeId: event.viaNodeId,
|
|
890
|
+
// the origin's record version, stored as-is so every replica holds the version the
|
|
891
|
+
// origin holds; the transaction's own timestamp stays the origin's log key
|
|
892
|
+
version: event.version,
|
|
707
893
|
// use per-event expiresAt: batched txn context only holds the first event's expiration
|
|
708
894
|
expiresAt: event.expiresAt,
|
|
709
895
|
// bulk base-copy snapshot frame: apply current-state directly, without an audit/transaction-log
|
|
@@ -940,7 +1126,9 @@ export function makeTable(options) {
|
|
|
940
1126
|
continue;
|
|
941
1127
|
}
|
|
942
1128
|
}
|
|
943
|
-
//
|
|
1129
|
+
// A source that reports no log position of its own (no `timestamp`) has only one clock,
|
|
1130
|
+
// so its record version doubles as the apply transaction's timestamp. A replication
|
|
1131
|
+
// receiver always sets `timestamp` from the origin's log key and never reaches this.
|
|
944
1132
|
if (!event.timestamp && event.version) event.timestamp = event.version;
|
|
945
1133
|
const commitResolution = transaction(event, () => {
|
|
946
1134
|
if (event.type === 'transaction') {
|
|
@@ -1414,7 +1602,26 @@ export function makeTable(options) {
|
|
|
1414
1602
|
return coerceType(id, primaryKeyAttribute);
|
|
1415
1603
|
}
|
|
1416
1604
|
|
|
1605
|
+
/**
|
|
1606
|
+
* A branch's Table classes deliberately carry the BASE's logical database name so an
|
|
1607
|
+
* application's schema and code resolve unchanged (harper#643). That makes every schema
|
|
1608
|
+
* mutation resolve against the global catalog — a `dropTable()` through a branch would delete
|
|
1609
|
+
* the live base table. Reads and writes are per-branch and unaffected; DDL is refused until a
|
|
1610
|
+
* branch owns a schema identity of its own.
|
|
1611
|
+
*/
|
|
1612
|
+
static assertSchemaMutable(operation: string) {
|
|
1613
|
+
if (!isBranch) return;
|
|
1614
|
+
const error: any = new Error(
|
|
1615
|
+
`Cannot ${operation} through a branched database: '${tableName}' resolves to the schema of base ` +
|
|
1616
|
+
`database '${databaseName}', so the change would apply to the base rather than the branch`
|
|
1617
|
+
);
|
|
1618
|
+
error.statusCode = 400;
|
|
1619
|
+
throw error;
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1417
1622
|
static async dropTable() {
|
|
1623
|
+
TableResource.assertSchemaMutable('drop a table');
|
|
1624
|
+
const rootStore = primaryStore.rootStore;
|
|
1418
1625
|
if (databaseName === databasePath) {
|
|
1419
1626
|
// Persist a drop tombstone on the primary catalog entry BEFORE any
|
|
1420
1627
|
// destructive work. If the process dies or a column family drop fails
|
|
@@ -1423,8 +1630,9 @@ export function makeTable(options) {
|
|
|
1423
1630
|
// completeInterruptedDrop in databases.ts instead of resurrecting
|
|
1424
1631
|
// the table.
|
|
1425
1632
|
const primaryCatalogKey = TableResource.tableName + '/';
|
|
1426
|
-
const
|
|
1427
|
-
|
|
1633
|
+
const writeTombstone = () => {
|
|
1634
|
+
const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey);
|
|
1635
|
+
if (!primaryMeta || primaryMeta.dropping) return;
|
|
1428
1636
|
primaryMeta.dropping = true;
|
|
1429
1637
|
// Stamps this drop's identity so the interrupted-drop retry budget in
|
|
1430
1638
|
// databases.ts can be scoped to THIS drop rather than the table name: a
|
|
@@ -1434,11 +1642,28 @@ export function makeTable(options) {
|
|
|
1434
1642
|
// the budget by generation instead makes the new drop's tombstone carry
|
|
1435
1643
|
// its own fresh key regardless of what any worker last observed.
|
|
1436
1644
|
primaryMeta.dropGeneration = randomUUID();
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1645
|
+
return (dbisDb as any).put(primaryCatalogKey, primaryMeta);
|
|
1646
|
+
};
|
|
1647
|
+
if (rootStore instanceof RocksDatabase) {
|
|
1648
|
+
// withUpdateAttributesLock's locked section cannot be held across an await, so a durable
|
|
1649
|
+
// tombstone depends on put being rebound to putSync for RocksDB primary stores (see
|
|
1650
|
+
// createOpenDBIObject). Check that BEFORE writing anything: a tombstone left behind by a
|
|
1651
|
+
// refused drop would delete the table on the next load.
|
|
1652
|
+
if ((dbisDb as any).put !== (dbisDb as any).putSync)
|
|
1653
|
+
throw new Error(
|
|
1654
|
+
`Cannot drop ${databaseName}.${TableResource.tableName}: the catalog store's put is asynchronous, so the drop tombstone cannot be made durable before the column families are dropped`
|
|
1655
|
+
);
|
|
1656
|
+
withUpdateAttributesLock(
|
|
1657
|
+
rootStore,
|
|
1658
|
+
`drop table '${databaseName}.${TableResource.tableName}'`,
|
|
1659
|
+
writeTombstone
|
|
1660
|
+
);
|
|
1661
|
+
} else {
|
|
1662
|
+
let tombstoneWrite;
|
|
1663
|
+
rootStore.transactionSync(() => {
|
|
1664
|
+
tombstoneWrite = writeTombstone();
|
|
1665
|
+
});
|
|
1666
|
+
if (typeof tombstoneWrite?.then === 'function') await tombstoneWrite;
|
|
1442
1667
|
}
|
|
1443
1668
|
}
|
|
1444
1669
|
// A get() against a sourcedFrom table resolves to its caller before the resolved
|
|
@@ -1518,19 +1743,16 @@ export function makeTable(options) {
|
|
|
1518
1743
|
dbisDb.remove(TableResource.tableName + '/');
|
|
1519
1744
|
return true;
|
|
1520
1745
|
};
|
|
1521
|
-
const rootStore = primaryStore.rootStore;
|
|
1522
1746
|
if (rootStore instanceof RocksDatabase) {
|
|
1523
1747
|
// Serialize the drops + catalog removal against a concurrent
|
|
1524
1748
|
// same-name create (and completeInterruptedDrop) under the database's
|
|
1525
1749
|
// 'update-attributes' exclusive lock - the same lock the create path
|
|
1526
|
-
// holds. It is a synchronous
|
|
1750
|
+
// holds. It is a synchronous lock wait that blocks the event loop, so
|
|
1527
1751
|
// the locked section MUST stay synchronous: drop with dropSync (as
|
|
1528
1752
|
// completeInterruptedDrop does), never an awaited drop(), or a
|
|
1529
|
-
// concurrent create's
|
|
1530
|
-
//
|
|
1531
|
-
|
|
1532
|
-
let removed = false;
|
|
1533
|
-
try {
|
|
1753
|
+
// concurrent create's wait would be stuck on a drop that the blocked
|
|
1754
|
+
// event loop can never resolve, burning its full deadline before failing.
|
|
1755
|
+
const removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
|
|
1534
1756
|
for (const attribute of attributes) {
|
|
1535
1757
|
const index = indices[attribute.name];
|
|
1536
1758
|
if (index)
|
|
@@ -1545,10 +1767,8 @@ export function makeTable(options) {
|
|
|
1545
1767
|
} catch (error) {
|
|
1546
1768
|
ignoreAlreadyDropped(error);
|
|
1547
1769
|
}
|
|
1548
|
-
|
|
1549
|
-
}
|
|
1550
|
-
rootStore.unlock('update-attributes');
|
|
1551
|
-
}
|
|
1770
|
+
return removeTombstonedCatalog();
|
|
1771
|
+
});
|
|
1552
1772
|
if (removed) await dbisDb.committed;
|
|
1553
1773
|
} else {
|
|
1554
1774
|
// LMDB: no shared column-family double-drop, and its engine lock is
|
|
@@ -1564,7 +1784,12 @@ export function makeTable(options) {
|
|
|
1564
1784
|
if (removeTombstonedCatalog()) await dbisDb.committed;
|
|
1565
1785
|
}
|
|
1566
1786
|
} else {
|
|
1567
|
-
// legacy table per database
|
|
1787
|
+
// legacy table per database. The store to retire is this table's own audit store: nothing
|
|
1788
|
+
// assigns `primaryStore.auditStore` — openAuditStore() assigns `rootStore.auditStore`, and
|
|
1789
|
+
// this is the reference makeTable() was handed. Awaited so a pass suspended mid-removal has
|
|
1790
|
+
// released the primary DBI before it is closed and unlinked.
|
|
1791
|
+
await auditStore?.stopAuditCleanup?.();
|
|
1792
|
+
removeStorageReclamation(primaryStore.path);
|
|
1568
1793
|
await primaryStore.close();
|
|
1569
1794
|
fs.unlinkSync(primaryStore.path);
|
|
1570
1795
|
}
|
|
@@ -1884,13 +2109,71 @@ export function makeTable(options) {
|
|
|
1884
2109
|
* Save any changes into this instance to the current transaction
|
|
1885
2110
|
*/
|
|
1886
2111
|
save() {
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
2112
|
+
this.#assertLiveHandle(this.getId()); // a write through a released or expired lock never lands
|
|
2113
|
+
const operation = this.#savingOperation;
|
|
2114
|
+
if ((!operation || operation.dropped) && this.#lockWritable && this.#lockHandle?.hold) {
|
|
2115
|
+
// A held lock's record stages its update here rather than at lock() time: it is often
|
|
2116
|
+
// written after the acquiring transaction has already completed, which would have
|
|
2117
|
+
// dropped an update staged then. Nothing set means nothing to stage — a held-but-untouched
|
|
2118
|
+
// id stays untouched. Scoped locks do not take this branch: #reloadLocked stages their
|
|
2119
|
+
// TransactionWrite at lock() time (exactly like update()), so #savingOperation is always
|
|
2120
|
+
// set for a live scoped lock and the ordinary path below applies.
|
|
2121
|
+
// Verify the hold is still alive: if #lockWritable is set but the handle expired or was
|
|
2122
|
+
// released between lock acquisition and this save(), throw 409 rather than silently
|
|
2123
|
+
// committing stale data. Every lock-writable instance carries its own handle.
|
|
2124
|
+
const saveHandle = this.#lockHandle!;
|
|
2125
|
+
if (saveHandle.expired || saveHandle.released) {
|
|
2126
|
+
throw lockNotHeldError(saveHandle);
|
|
2127
|
+
}
|
|
2128
|
+
const changes = this.#changes;
|
|
2129
|
+
if (changes && Object.keys(changes).length > 0) {
|
|
2130
|
+
this.#savingOperation = null;
|
|
2131
|
+
return when(this._writeUpdate(this.getId(), changes, false), () => {
|
|
2132
|
+
const op = this.#savingOperation;
|
|
2133
|
+
if (op?.dropped) {
|
|
2134
|
+
this.#changes = undefined;
|
|
2135
|
+
return;
|
|
2136
|
+
}
|
|
2137
|
+
// Clear #savingOperation so the next sequential save() enters the lock-writable
|
|
2138
|
+
// path and creates a fresh write (otherwise a non-null #savingOperation makes
|
|
2139
|
+
// save() take the #saveOperation branch with an already-committed write, which
|
|
2140
|
+
// is a no-op, silently dropping the new change).
|
|
2141
|
+
// op.innerCommit is the real native-transaction commit Promise set on the
|
|
2142
|
+
// immediateCommit path in DatabaseTransaction.save(); await it to ensure
|
|
2143
|
+
// durability before resolving to the caller.
|
|
2144
|
+
if (op?.saved) {
|
|
2145
|
+
this.#savingOperation = null;
|
|
2146
|
+
return op?.innerCommit;
|
|
2147
|
+
}
|
|
2148
|
+
// op.saved = false means addWrite deferred the save; #saveOperation commits it
|
|
2149
|
+
// synchronously but ImmediateTransaction.save() returns undefined while the
|
|
2150
|
+
// inner rocksdb commit is still pending — return innerCommit so the caller
|
|
2151
|
+
// actually waits for durability.
|
|
2152
|
+
return when(this.save(), () => op?.innerCommit);
|
|
2153
|
+
});
|
|
2154
|
+
}
|
|
2155
|
+
// No changes: nothing to stage. A dropped operation (detached at a scoped→hold
|
|
2156
|
+
// upgrade — see detachScopedUpgradeWrite) must not fall through to the ordinary
|
|
2157
|
+
// #saveOperation path below with its now-detached reference.
|
|
2158
|
+
if (!operation || operation.dropped) {
|
|
1891
2159
|
this.#savingOperation = null;
|
|
2160
|
+
return;
|
|
1892
2161
|
}
|
|
1893
2162
|
}
|
|
2163
|
+
if (this.#savingOperation) {
|
|
2164
|
+
const operation = this.#savingOperation;
|
|
2165
|
+
this.#savingOperation = null;
|
|
2166
|
+
// A write that lands via a nested immediateCommit (e.g. a second sequential save() on
|
|
2167
|
+
// the same ImmediateTransaction context, once the first has already closed it) sets
|
|
2168
|
+
// operation.innerCommit to the real native-commit promise, but the commit() sweep loop
|
|
2169
|
+
// that triggers it discards its own return value — #saveOperation()'s result can
|
|
2170
|
+
// resolve before that native commit actually settles. Chain on innerCommit (as the
|
|
2171
|
+
// lock-writable hold branch above already does) so callers awaiting save() see the
|
|
2172
|
+
// write durably land, not just the outer (possibly premature) resolution.
|
|
2173
|
+
const result = this.#saveOperation(operation);
|
|
2174
|
+
const innerCommit = operation.innerCommit;
|
|
2175
|
+
return innerCommit ? when(innerCommit, () => result) : result;
|
|
2176
|
+
}
|
|
1894
2177
|
}
|
|
1895
2178
|
#saveOperation(operation: any) {
|
|
1896
2179
|
const transaction = txnForContext(this.getContext());
|
|
@@ -1968,6 +2251,7 @@ export function makeTable(options) {
|
|
|
1968
2251
|
});
|
|
1969
2252
|
}
|
|
1970
2253
|
_writeInvalidate(id: Id, partialRecord?: any, options?: any) {
|
|
2254
|
+
this.#assertLiveHandle(id);
|
|
1971
2255
|
const context = this.getContext();
|
|
1972
2256
|
checkValidId(id);
|
|
1973
2257
|
const transaction = txnForContext(this.getContext());
|
|
@@ -1976,9 +2260,13 @@ export function makeTable(options) {
|
|
|
1976
2260
|
store: primaryStore,
|
|
1977
2261
|
invalidated: true,
|
|
1978
2262
|
entry: this.#entry,
|
|
2263
|
+
recordVersion: options?.version,
|
|
2264
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
|
|
1979
2265
|
commit: (txnTime, existingEntry, _retry, transaction: any) => {
|
|
2266
|
+
const txnLogKey =
|
|
2267
|
+
isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
1980
2268
|
write.skipped = false; // reset on each retry; cleanup happens after commit if still true
|
|
1981
|
-
if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId)
|
|
2269
|
+
if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
|
|
1982
2270
|
write.skipped = true;
|
|
1983
2271
|
return;
|
|
1984
2272
|
}
|
|
@@ -2005,9 +2293,15 @@ export function makeTable(options) {
|
|
|
2005
2293
|
viaNodeId: options?.viaNodeId,
|
|
2006
2294
|
transaction,
|
|
2007
2295
|
tableToTrack: tableName,
|
|
2296
|
+
recordVersion: txnTime,
|
|
2297
|
+
additionalAuditRefs:
|
|
2298
|
+
isRocksDB && audit && txnLogKey !== txnTime
|
|
2299
|
+
? [{ version: txnLogKey, nodeId: options?.nodeId }]
|
|
2300
|
+
: undefined,
|
|
2008
2301
|
},
|
|
2009
2302
|
'invalidate'
|
|
2010
2303
|
);
|
|
2304
|
+
if (write.trackRecordVersion) write.recordVersionApplied = true;
|
|
2011
2305
|
// TODO: recordDeletion?
|
|
2012
2306
|
},
|
|
2013
2307
|
};
|
|
@@ -2015,20 +2309,25 @@ export function makeTable(options) {
|
|
|
2015
2309
|
transaction.addWrite(write);
|
|
2016
2310
|
}
|
|
2017
2311
|
_writeRelocate(id: Id, options: any) {
|
|
2312
|
+
this.#assertLiveHandle(id);
|
|
2018
2313
|
const context = this.getContext();
|
|
2019
2314
|
checkValidId(id);
|
|
2020
2315
|
const transaction = txnForContext(this.getContext());
|
|
2021
|
-
|
|
2316
|
+
const write: any = {
|
|
2022
2317
|
key: id,
|
|
2023
2318
|
store: primaryStore,
|
|
2024
2319
|
invalidated: true,
|
|
2025
2320
|
entry: this.#entry,
|
|
2321
|
+
recordVersion: options?.version,
|
|
2322
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
|
|
2026
2323
|
before:
|
|
2027
2324
|
(this.constructor as any).source?.relocate && !(context as any)?.source
|
|
2028
2325
|
? (this.constructor as any).source.relocate.bind((this.constructor as any).source, id, undefined, context)
|
|
2029
2326
|
: undefined,
|
|
2030
2327
|
commit: (txnTime, existingEntry, _retry, transaction: any) => {
|
|
2031
|
-
|
|
2328
|
+
const txnLogKey =
|
|
2329
|
+
isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
2330
|
+
if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) return;
|
|
2032
2331
|
const residency = TableResource.getResidencyRecord(options.residencyId);
|
|
2033
2332
|
let metadata = 0;
|
|
2034
2333
|
let newRecord = null;
|
|
@@ -2060,13 +2359,20 @@ export function makeTable(options) {
|
|
|
2060
2359
|
viaNodeId: options?.viaNodeId,
|
|
2061
2360
|
expiresAt: options.expiresAt,
|
|
2062
2361
|
transaction,
|
|
2362
|
+
recordVersion: txnTime,
|
|
2363
|
+
additionalAuditRefs:
|
|
2364
|
+
isRocksDB && audit && txnLogKey !== txnTime
|
|
2365
|
+
? [{ version: txnLogKey, nodeId: options?.nodeId }]
|
|
2366
|
+
: undefined,
|
|
2063
2367
|
},
|
|
2064
2368
|
'relocate',
|
|
2065
2369
|
false,
|
|
2066
2370
|
null
|
|
2067
2371
|
);
|
|
2372
|
+
if (write.trackRecordVersion) write.recordVersionApplied = true;
|
|
2068
2373
|
},
|
|
2069
|
-
}
|
|
2374
|
+
};
|
|
2375
|
+
transaction.addWrite(write);
|
|
2070
2376
|
}
|
|
2071
2377
|
|
|
2072
2378
|
/**
|
|
@@ -2182,10 +2488,260 @@ export function makeTable(options) {
|
|
|
2182
2488
|
}
|
|
2183
2489
|
}
|
|
2184
2490
|
/**
|
|
2185
|
-
*
|
|
2491
|
+
* Static entry point: `Table.lock(id, options?, context?)` — creates an instance in the given,
|
|
2492
|
+
* ambient, or a fresh context and delegates to the instance lock(). This shadows Resource.static
|
|
2493
|
+
* lock so that both callers share the same transaction link (required for cross-instance upgrade
|
|
2494
|
+
* detection). lock() is an in-process API with no authorization hook of its own; it is not
|
|
2495
|
+
* protocol-dispatched, so no allowUpdate/allowCreate check runs on acquisition.
|
|
2496
|
+
*
|
|
2497
|
+
* Dropping the trailing `context` leaks the key: the bare `{}` fallback is an
|
|
2498
|
+
* ImmediateTransaction, which releases no record locks.
|
|
2186
2499
|
*/
|
|
2187
|
-
lock(
|
|
2188
|
-
|
|
2500
|
+
static async lock(
|
|
2501
|
+
target?: RequestTargetOrId | RecordLockOptions,
|
|
2502
|
+
options?: RecordLockOptions,
|
|
2503
|
+
context?: any
|
|
2504
|
+
): Promise<any> {
|
|
2505
|
+
if (!isRocksDB) throw new ClientError('Record locks are not supported on LMDB', 501);
|
|
2506
|
+
if (options === undefined && isPlainOptions(target)) {
|
|
2507
|
+
options = target as RecordLockOptions;
|
|
2508
|
+
target = undefined;
|
|
2509
|
+
}
|
|
2510
|
+
const id = target != null ? requestTargetToId(target as RequestTargetOrId) : null;
|
|
2511
|
+
const resolvedContext: any = contextArgument(context) ?? contextStorage.getStore() ?? {};
|
|
2512
|
+
const resource = new TableResource(id, resolvedContext);
|
|
2513
|
+
return resource.lock(target, options);
|
|
2514
|
+
}
|
|
2515
|
+
/**
|
|
2516
|
+
* Acquire an exclusive lock on this record (or on `target`'s) and return it ready for updates
|
|
2517
|
+
* (harper#483, Phase 0: exclusive across every worker thread of this node). The lock is held
|
|
2518
|
+
* in process memory only — no durable writes. Phase 0 contract: lock() is mutually exclusive
|
|
2519
|
+
* with other lock() calls on the same key; plain writes (put/patch/delete/create) are never
|
|
2520
|
+
* gated or blocked. The generation expires after `lease` if it is never released.
|
|
2521
|
+
*
|
|
2522
|
+
* Transaction-scoped (default): write through the returned record (or the table's static verbs
|
|
2523
|
+
* in the same transaction), and the commit or abort releases it. `{ hold: true }`: the lock
|
|
2524
|
+
* outlives the transaction; write through the returned record and release with `unlock()`, or
|
|
2525
|
+
* let the lease expire.
|
|
2526
|
+
*/
|
|
2527
|
+
// async so option/id validation rejects rather than throwing past a caller's `.catch()`; the
|
|
2528
|
+
// body still runs to completion synchronously, which is what keeps concurrent lock() calls
|
|
2529
|
+
// on one key coalescing instead of racing to tryLock.
|
|
2530
|
+
async lock(target?: RequestTargetOrId | RecordLockOptions, options?: RecordLockOptions): Promise<any> {
|
|
2531
|
+
if (!isRocksDB) throw new ClientError('Record locks are not supported on LMDB', 501);
|
|
2532
|
+
if (options === undefined && isPlainOptions(target)) {
|
|
2533
|
+
options = target as RecordLockOptions;
|
|
2534
|
+
target = undefined;
|
|
2535
|
+
}
|
|
2536
|
+
const id = target != null ? requestTargetToId(target as RequestTargetOrId) : this.getId();
|
|
2537
|
+
checkValidId(id);
|
|
2538
|
+
const resolved = resolveLockOptions(options);
|
|
2539
|
+
const context = this.getContext();
|
|
2540
|
+
const link = txnForContext(context);
|
|
2541
|
+
const keyId = writeKeyId(id);
|
|
2542
|
+
const held = this.#lockHandle;
|
|
2543
|
+
if (held && !held.released && !held.expired && held.keyId === keyId) {
|
|
2544
|
+
// Re-entrant: upgrade to hold if requested, then preserve staged changes.
|
|
2545
|
+
if (resolved.hold && !held.hold) {
|
|
2546
|
+
held.upgradeToHold(resolved.lease);
|
|
2547
|
+
// The scoped phase eagerly staged a TransactionWrite (see #reloadLocked); hold
|
|
2548
|
+
// staging is deferred and explicit-save-only, so an unsaved scoped write left
|
|
2549
|
+
// dangling here would otherwise auto-commit at the transaction sweep and clobber
|
|
2550
|
+
// whatever the hold write lands. detachScopedUpgradeWrite marks it .dropped so a
|
|
2551
|
+
// later save() on this instance falls through to the hold branch instead of the
|
|
2552
|
+
// dead #savingOperation reference.
|
|
2553
|
+
detachScopedUpgradeWrite(link, keyId, held);
|
|
2554
|
+
}
|
|
2555
|
+
return Promise.resolve(this.#reloadLocked(id, undefined, true));
|
|
2556
|
+
}
|
|
2557
|
+
const scoped = link.recordLockFor(primaryStore, keyId);
|
|
2558
|
+
if (scoped && !scoped.released && !scoped.expired) {
|
|
2559
|
+
if (resolved.hold && !scoped.hold) {
|
|
2560
|
+
// Upgrade scoped → hold: flip the existing handle object to hold mode so every
|
|
2561
|
+
// instance that already references this handle stays valid. Retiring and creating a
|
|
2562
|
+
// new handle would invalidate those other references (their save() would then throw
|
|
2563
|
+
// 409 against a released handle). The native key stays locked throughout.
|
|
2564
|
+
scoped.upgradeToHold(resolved.lease);
|
|
2565
|
+
detachScopedUpgradeWrite(link, keyId, scoped);
|
|
2566
|
+
return Promise.resolve(this.#reloadLocked(id, scoped, true));
|
|
2567
|
+
}
|
|
2568
|
+
// Already held with the same type: re-entrant return. Preserve any staged changes.
|
|
2569
|
+
return Promise.resolve(this.#reloadLocked(id, scoped, true));
|
|
2570
|
+
}
|
|
2571
|
+
const key = lockAttemptKey(tableId, id);
|
|
2572
|
+
// Coalesce concurrent lock() calls for the same key inside one link so they don't
|
|
2573
|
+
// self-block: Promise.all([T.lock(id), T.lock(id)]) would otherwise have both calls
|
|
2574
|
+
// reach tryLock before either registers, making the second park against the first.
|
|
2575
|
+
const pending = link.pendingLockFor(primaryStore, keyId);
|
|
2576
|
+
if (pending) {
|
|
2577
|
+
// Wait for the in-flight acquisition, then take the re-entrant path as if
|
|
2578
|
+
// recordLockFor had found it. If the first attempt timed out, re-enter so
|
|
2579
|
+
// the second caller gets its own timeout.
|
|
2580
|
+
// The follower waits on the leader's acquisition, but only for its own timeout.
|
|
2581
|
+
let followerTimer: ReturnType<typeof setTimeout> | undefined;
|
|
2582
|
+
const followerTimedOut = Symbol('follower timeout');
|
|
2583
|
+
const followerStart = Date.now();
|
|
2584
|
+
const followerDeadline = new Promise<never>((_, reject) => {
|
|
2585
|
+
followerTimer = setTimeout(() => reject(followerTimedOut), resolved.timeout).unref();
|
|
2586
|
+
});
|
|
2587
|
+
// Try again on this caller's own terms with the budget it has left.
|
|
2588
|
+
const retryOnRemainingBudget = () => {
|
|
2589
|
+
// The enclosing transaction ended while we were parked. A retry re-resolves the
|
|
2590
|
+
// context, which no longer points at this link, so the handle it acquired would be
|
|
2591
|
+
// registered on a fresh transaction that no commit or abort ever releases — the
|
|
2592
|
+
// same abandonment the leader's own post-acquisition guard below rejects.
|
|
2593
|
+
if (link.open === TRANSACTION_STATE.CLOSED && !link.saveCommits)
|
|
2594
|
+
throw new ServerError('Transaction was closed while waiting for a record lock', 500);
|
|
2595
|
+
const remaining = resolved.timeout - (Date.now() - followerStart);
|
|
2596
|
+
if (remaining <= 0) throw new ClientError(`Record is locked and was not released in time`, 423);
|
|
2597
|
+
return this.lock(target, { ...resolved, timeout: remaining }) as Promise<any>;
|
|
2598
|
+
};
|
|
2599
|
+
return Promise.race([pending, followerDeadline]).then(
|
|
2600
|
+
() => {
|
|
2601
|
+
clearTimeout(followerTimer);
|
|
2602
|
+
const acquired = link.recordLockFor(primaryStore, keyId);
|
|
2603
|
+
if (acquired && !acquired.released && !acquired.expired) {
|
|
2604
|
+
if (resolved.hold && !acquired.hold) {
|
|
2605
|
+
detachScopedUpgradeWrite(link, keyId, acquired);
|
|
2606
|
+
acquired.upgradeToHold(resolved.lease);
|
|
2607
|
+
}
|
|
2608
|
+
return this.#reloadLocked(id, acquired, true);
|
|
2609
|
+
}
|
|
2610
|
+
return retryOnRemainingBudget();
|
|
2611
|
+
},
|
|
2612
|
+
(error) => {
|
|
2613
|
+
clearTimeout(followerTimer);
|
|
2614
|
+
if (error === followerTimedOut) throw new ClientError(`Record is locked and was not released in time`, 423);
|
|
2615
|
+
return retryOnRemainingBudget();
|
|
2616
|
+
}
|
|
2617
|
+
);
|
|
2618
|
+
}
|
|
2619
|
+
const pendingPromise = acquireRecordKey(
|
|
2620
|
+
link,
|
|
2621
|
+
primaryStore,
|
|
2622
|
+
key,
|
|
2623
|
+
keyId,
|
|
2624
|
+
resolved.timeout,
|
|
2625
|
+
resolved.lease,
|
|
2626
|
+
resolved.hold
|
|
2627
|
+
);
|
|
2628
|
+
link.registerPendingLock(primaryStore, keyId, pendingPromise);
|
|
2629
|
+
return pendingPromise.then(
|
|
2630
|
+
(handle) => {
|
|
2631
|
+
link.unregisterPendingLock(primaryStore, keyId);
|
|
2632
|
+
if (link.open === TRANSACTION_STATE.CLOSED && !link.saveCommits) {
|
|
2633
|
+
// The transaction was aborted while this call waited; nothing would ever release the handle.
|
|
2634
|
+
handle.release();
|
|
2635
|
+
throw new ServerError('Transaction was closed while waiting for a record lock', 500);
|
|
2636
|
+
}
|
|
2637
|
+
link.registerRecordLock(handle);
|
|
2638
|
+
if (link.saveCommits && (context as any)?.timestamp) handle.noteCandidateFloor((context as any).timestamp);
|
|
2639
|
+
if (link.open === TRANSACTION_STATE.OPEN && !link.saveCommits) {
|
|
2640
|
+
// Explicit transaction() (not ImmediateTransaction): pin the clock to
|
|
2641
|
+
// acquiredAt when no writes have been staged yet. When writes already
|
|
2642
|
+
// exist, leave the clock alone (ordering is best-effort; write held records
|
|
2643
|
+
// in their own transaction for the guarantee). ImmediateTransaction is
|
|
2644
|
+
// excluded (saveCommits=true) — its clock is never pinned in lock();
|
|
2645
|
+
// each save() stamps from the handle's committed version floor instead.
|
|
2646
|
+
if (link.writes.length === 0 && !link.timestamp) {
|
|
2647
|
+
link.timestamp = handle.acquiredAt;
|
|
2648
|
+
}
|
|
2649
|
+
if (!resolved.hold && link.transaction) {
|
|
2650
|
+
// Scoped lock: the read snapshot may predate the lock; drop it so the
|
|
2651
|
+
// scope reads what it locked. Hold locks use acquiredAt directly and
|
|
2652
|
+
// do not update the read snapshot.
|
|
2653
|
+
// The timestamp guard matches DatabaseTransaction's own setTimestamp calls: a
|
|
2654
|
+
// deferred update() write leaves the clock at 0, which rocksdb-js rejects.
|
|
2655
|
+
if (link.writes.length === 0 && link.readTxnsUsed <= 1) {
|
|
2656
|
+
link.releaseReadTxn();
|
|
2657
|
+
link.snapshotFree = true;
|
|
2658
|
+
} else if (link.timestamp) link.transaction.setTimestamp(link.timestamp);
|
|
2659
|
+
}
|
|
2660
|
+
}
|
|
2661
|
+
// ImmediateTransaction: no clock pinning in lock(); save() stamps each write
|
|
2662
|
+
// from the committed handle floor for both scoped and hold handles.
|
|
2663
|
+
return this.#reloadLocked(id, handle);
|
|
2664
|
+
},
|
|
2665
|
+
(err) => {
|
|
2666
|
+
link.unregisterPendingLock(primaryStore, keyId);
|
|
2667
|
+
throw err;
|
|
2668
|
+
}
|
|
2669
|
+
);
|
|
2670
|
+
}
|
|
2671
|
+
#reloadLocked(id: Id, holdHandle?: RecordLockHandle | null, preserveChanges = false) {
|
|
2672
|
+
// For freshness, read the committed entry (snapshot-free) so a hold lock sees concurrent
|
|
2673
|
+
// committed writes rather than a stale snapshot. A write earlier in THIS explicit
|
|
2674
|
+
// transaction has not landed in that committed entry yet (harper#1968: Harper defers an
|
|
2675
|
+
// explicit transaction's writes until the writing call actually runs them), so pull the
|
|
2676
|
+
// current value the same way a chained write picks up its basis (priorStagedWrite): the
|
|
2677
|
+
// record comes from the prior staged write, the rest of the entry (version, audit chain,
|
|
2678
|
+
// blob metadata) stays the pre-transaction one.
|
|
2679
|
+
const link = txnForContext(this.getContext());
|
|
2680
|
+
let entryForReload: any = primaryStore.getEntry(id);
|
|
2681
|
+
if (link.open === TRANSACTION_STATE.OPEN) {
|
|
2682
|
+
const keyId = writeKeyId(id);
|
|
2683
|
+
const tailWrite = link.writesByKey?.get(primaryStore)?.get(keyId);
|
|
2684
|
+
const priorStaged =
|
|
2685
|
+
tailWrite && (tailWrite.stagedEntry !== undefined ? tailWrite : priorStagedWrite(tailWrite));
|
|
2686
|
+
if (priorStaged?.stagedEntry !== undefined) {
|
|
2687
|
+
entryForReload = entryForReload
|
|
2688
|
+
? { ...entryForReload, value: priorStaged.stagedEntry.value }
|
|
2689
|
+
: { value: priorStaged.stagedEntry.value };
|
|
2690
|
+
if (entryForReload.value && typeof entryForReload.value === 'object') {
|
|
2691
|
+
// Register the merged entry in entryMap so getUpdatedTime() works.
|
|
2692
|
+
entryMap.set(entryForReload.value, entryForReload);
|
|
2693
|
+
}
|
|
2694
|
+
}
|
|
2695
|
+
}
|
|
2696
|
+
if (writeKeyId(id) !== writeKeyId(this.getId())) {
|
|
2697
|
+
// lock(target) where target differs from this record: return a separate instance.
|
|
2698
|
+
const fresh = new (this.constructor as any)(id, this.getContext());
|
|
2699
|
+
TableResource._updateResource(fresh, entryForReload);
|
|
2700
|
+
if (holdHandle != null) {
|
|
2701
|
+
fresh.#lockHandle = holdHandle;
|
|
2702
|
+
// Do not clear this.#lockHandle: the original instance keeps its own lock on its
|
|
2703
|
+
// own id; the fresh instance owns the lock on the target id independently.
|
|
2704
|
+
}
|
|
2705
|
+
fresh.#lockWritable = true;
|
|
2706
|
+
// Scoped (not hold) stages exactly like update(): create the TransactionWrite now so
|
|
2707
|
+
// save() is the ordinary #savingOperation path. Hold keeps deferred staging (the
|
|
2708
|
+
// acquiring transaction may commit before the holder ever writes).
|
|
2709
|
+
if (!fresh.#lockHandle!.hold) fresh._writeUpdate(id, fresh.#changes, false);
|
|
2710
|
+
return fresh;
|
|
2711
|
+
}
|
|
2712
|
+
// Store the handle for both scoped and hold locks; undefined (re-entrant hold fast-path)
|
|
2713
|
+
// must not clear a handle already set.
|
|
2714
|
+
if (holdHandle != null) this.#lockHandle = holdHandle;
|
|
2715
|
+
TableResource._updateResource(this, entryForReload);
|
|
2716
|
+
// Preserve staged changes when upgrading the same instance from scoped to hold so that
|
|
2717
|
+
// set() calls made under the scoped lock survive the reload.
|
|
2718
|
+
if (!preserveChanges) this.#changes = undefined;
|
|
2719
|
+
this.#lockWritable = true;
|
|
2720
|
+
// Scoped (not hold): stage now, same as update() would. Skip if a write from an earlier
|
|
2721
|
+
// lock() cycle on this instance is still pending (re-entrant call before its save()).
|
|
2722
|
+
if (!this.#lockHandle!.hold && !this.#savingOperation) this._writeUpdate(id, this.#changes, false);
|
|
2723
|
+
return this;
|
|
2724
|
+
}
|
|
2725
|
+
/**
|
|
2726
|
+
* Release the lock this instance holds. Resolves true when this call cleared the native key lock.
|
|
2727
|
+
* Works for both held (`{ hold: true }`) and transaction-scoped locks. After unlock() the
|
|
2728
|
+
* instance is no longer lock-writable; writes through it require a fresh lock.
|
|
2729
|
+
*/
|
|
2730
|
+
unlock(): Promise<boolean> {
|
|
2731
|
+
// Always clear the local lock-writable state so subsequent writes on this instance are
|
|
2732
|
+
// ungated, regardless of whether the handle was already released.
|
|
2733
|
+
const handle = this.#lockHandle;
|
|
2734
|
+
this.#lockHandle = undefined;
|
|
2735
|
+
this.#lockWritable = false;
|
|
2736
|
+
if (!handle || handle.released) return Promise.resolve(false);
|
|
2737
|
+
const link = txnForContext(this.getContext());
|
|
2738
|
+
// A scoped lock staged its write at lock() time; released before commit, that write must not
|
|
2739
|
+
// run into the released-handle guard at the sweep.
|
|
2740
|
+
if (this.#savingOperation && !this.#savingOperation.saved && this.#savingOperation.lockHandle === handle)
|
|
2741
|
+
this.#savingOperation = null;
|
|
2742
|
+
detachScopedUpgradeWrite(link, writeKeyId(this.getId()), handle);
|
|
2743
|
+
link.unregisterRecordLock(handle);
|
|
2744
|
+
return Promise.resolve(handle.release());
|
|
2189
2745
|
}
|
|
2190
2746
|
static operation(operation, context) {
|
|
2191
2747
|
operation.table ||= tableName;
|
|
@@ -2311,8 +2867,10 @@ export function makeTable(options) {
|
|
|
2311
2867
|
// a notification that a write has already occurred in the canonical data source, we need to update our
|
|
2312
2868
|
// local copy
|
|
2313
2869
|
_writeUpdate(id: Id, recordUpdate: any, fullUpdate: boolean, options?: any) {
|
|
2870
|
+
this.#assertLiveHandle(id);
|
|
2314
2871
|
const context = this.getContext();
|
|
2315
2872
|
const transaction = txnForContext(context);
|
|
2873
|
+
const replaying = transaction.isReplay === true;
|
|
2316
2874
|
checkValidId(id);
|
|
2317
2875
|
if (fullUpdate && recordUpdate == null && options?.isNotification) {
|
|
2318
2876
|
// A source/replication-applied put must carry the record; these applies skip record
|
|
@@ -2356,6 +2914,13 @@ export function makeTable(options) {
|
|
|
2356
2914
|
nodeName: (context as any)?.nodeName,
|
|
2357
2915
|
fullUpdate,
|
|
2358
2916
|
deferSave: true,
|
|
2917
|
+
// the origin's record version on an applied write; absent for a locally-originated one
|
|
2918
|
+
recordVersion: options?.version,
|
|
2919
|
+
// Include the lock handle (if any) so the expired-handle guard in
|
|
2920
|
+
// DatabaseTransaction.save() can throw 409 when the lease has lapsed.
|
|
2921
|
+
// Only attach the hold handle when it covers exactly this key; off-key writes
|
|
2922
|
+
// are ordinary and must not carry an unrelated hold's handle.
|
|
2923
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
|
|
2359
2924
|
validate: (txnTime, committedBy = transaction) => {
|
|
2360
2925
|
if (!recordUpdate) recordUpdate = this.#changes;
|
|
2361
2926
|
if (fullUpdate || (recordUpdate && hasChanges(this.#changes === recordUpdate ? this : recordUpdate))) {
|
|
@@ -2478,6 +3043,8 @@ export function makeTable(options) {
|
|
|
2478
3043
|
this.#savingOperation = null;
|
|
2479
3044
|
write.stagedIn = undefined; // nothing may pin this write's transaction past its commit
|
|
2480
3045
|
let omitLocalRecord = false;
|
|
3046
|
+
const txnLogKey =
|
|
3047
|
+
isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
2481
3048
|
// we use optimistic locking to only commit if the existing record state still holds true.
|
|
2482
3049
|
// this is superior to using an async transaction since it doesn't require JS execution
|
|
2483
3050
|
// during the write transaction.
|
|
@@ -2540,10 +3107,10 @@ export function makeTable(options) {
|
|
|
2540
3107
|
if (
|
|
2541
3108
|
existingEntry.additionalAuditRefs?.some(
|
|
2542
3109
|
(ref) =>
|
|
2543
|
-
ref.version ===
|
|
3110
|
+
ref.version === txnLogKey &&
|
|
2544
3111
|
precedesExistingVersion(
|
|
2545
3112
|
txnTime,
|
|
2546
|
-
{ version: txnTime, localTime:
|
|
3113
|
+
{ version: txnTime, localTime: txnLogKey, key: id, nodeId: ref.nodeId },
|
|
2547
3114
|
options?.nodeId
|
|
2548
3115
|
) === 0
|
|
2549
3116
|
)
|
|
@@ -2574,10 +3141,10 @@ export function makeTable(options) {
|
|
|
2574
3141
|
if (!oldestRetainedAuditTimeResolved) {
|
|
2575
3142
|
oldestRetainedAuditTimeResolved = true;
|
|
2576
3143
|
// getRange yields ascending by audit-log key, so the first entry is the oldest retained.
|
|
2577
|
-
// Mirror replicationConnection's retention check and the cleanup key basis (
|
|
2578
|
-
//
|
|
3144
|
+
// Mirror replicationConnection's retention check and the cleanup key basis (`txnLogKey`).
|
|
3145
|
+
// Fall back to the nominal time-based purge floor when the log is empty/unavailable.
|
|
2579
3146
|
for (const entry of auditStore.getRange({ start: 1, log: options?.nodeId })) {
|
|
2580
|
-
oldestRetainedAuditTime = entry.
|
|
3147
|
+
oldestRetainedAuditTime = entry.txnLogKey;
|
|
2581
3148
|
break;
|
|
2582
3149
|
}
|
|
2583
3150
|
oldestRetainedAuditTime ??= Date.now() - auditRetention;
|
|
@@ -2590,23 +3157,25 @@ export function makeTable(options) {
|
|
|
2590
3157
|
// depth-cap block. This is the same keyed lookup that block performs, hoisted ahead of the walk.
|
|
2591
3158
|
// It is what catches transitive/proxied re-deliveries: they arrive buried below the record head
|
|
2592
3159
|
// (so replication's head-tie fast-skip can't see them) yet are exact duplicates. Keyed by nodeId,
|
|
2593
|
-
// so it is correct across multiple source nodes.
|
|
2594
|
-
//
|
|
2595
|
-
//
|
|
3160
|
+
// so it is correct across multiple source nodes. The lookup key is this write's LOG key, not its
|
|
3161
|
+
// record version — a replication apply commits under the origin's log key while storing the
|
|
3162
|
+
// origin's version, and only the log key addresses the entry (harper#2412).
|
|
3163
|
+
// RocksDB-only: LMDB audit entries are keyed by local audit time, so this lookup doesn't apply
|
|
3164
|
+
// there (LMDB keeps the exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
|
|
2596
3165
|
// simply falls through to the walk, so this never changes correctness; the additionalAuditRefs
|
|
2597
3166
|
// check above remains the read-your-writes guard. Never when this write staged in a prior
|
|
2598
3167
|
// failed attempt: that attempt already appended this write's own audit entry, so the lookup
|
|
2599
3168
|
// would find it and skip the write as "already applied" when the record was never committed.
|
|
2600
3169
|
// A recommit of the same transaction survived that skip only because the old write batch
|
|
2601
3170
|
// still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
|
|
2602
|
-
if (isRocksDB && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(
|
|
2603
|
-
const priorAudit = auditStore.get(
|
|
3171
|
+
if (isRocksDB && !replaying && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnLogKey)) {
|
|
3172
|
+
const priorAudit = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
|
|
2604
3173
|
if (
|
|
2605
3174
|
priorAudit &&
|
|
2606
|
-
priorAudit.
|
|
3175
|
+
priorAudit.txnLogKey === txnLogKey &&
|
|
2607
3176
|
precedesExistingVersion(
|
|
2608
3177
|
txnTime,
|
|
2609
|
-
{ version: txnTime, localTime:
|
|
3178
|
+
{ version: txnTime, localTime: txnLogKey, key: id, nodeId: priorAudit.nodeId },
|
|
2610
3179
|
options?.nodeId
|
|
2611
3180
|
) === 0
|
|
2612
3181
|
) {
|
|
@@ -2615,7 +3184,10 @@ export function makeTable(options) {
|
|
|
2615
3184
|
}
|
|
2616
3185
|
}
|
|
2617
3186
|
// incremental CRDT updates are only available with audit logging on
|
|
2618
|
-
|
|
3187
|
+
const initialAuditHead = isRocksDB
|
|
3188
|
+
? resolveAuditHead(id, existingEntry.version, existingEntry.nodeId, existingEntry.additionalAuditRefs)
|
|
3189
|
+
: { txnLogKey: existingEntry.localTime, nodeId: existingEntry.nodeId };
|
|
3190
|
+
let localTime = initialAuditHead.txnLogKey;
|
|
2619
3191
|
let auditedVersion = existingEntry.version;
|
|
2620
3192
|
logger.debug?.(
|
|
2621
3193
|
'Applying CRDT update to record with id: ',
|
|
@@ -2628,22 +3200,42 @@ export function makeTable(options) {
|
|
|
2628
3200
|
new Date(localTime)
|
|
2629
3201
|
);
|
|
2630
3202
|
|
|
2631
|
-
let nodeId =
|
|
3203
|
+
let nodeId = initialAuditHead.nodeId;
|
|
2632
3204
|
const succeedingUpdates = []; // record the "future" updates, as we need to apply the updates in reverse order
|
|
2633
3205
|
const auditRefsToVisit: Array<{ localTime: number; nodeId: number }> = existingEntry.additionalAuditRefs
|
|
2634
3206
|
? existingEntry.additionalAuditRefs.map((ref) => ({ localTime: ref.version, nodeId: ref.nodeId }))
|
|
2635
3207
|
: [];
|
|
2636
3208
|
|
|
2637
|
-
//
|
|
3209
|
+
// Out-of-order merges retain every existing branch head; per-origin log keys are not globally ordered.
|
|
2638
3210
|
if (existingEntry.additionalAuditRefs) {
|
|
2639
3211
|
for (const ref of existingEntry.additionalAuditRefs) {
|
|
2640
|
-
|
|
2641
|
-
additionalAuditRefs.push(ref);
|
|
2642
|
-
}
|
|
3212
|
+
additionalAuditRefs.push(ref);
|
|
2643
3213
|
}
|
|
2644
3214
|
}
|
|
2645
3215
|
let addedAuditRef = false;
|
|
2646
3216
|
let nextRef: { localTime: number; nodeId: number };
|
|
3217
|
+
const visitedAuditRefs = new Set<string>();
|
|
3218
|
+
const queuePreviousAuditRefs = (auditRecord) => {
|
|
3219
|
+
const previousRefs = auditRecord.previousAdditionalAuditRefs;
|
|
3220
|
+
if (previousRefs) {
|
|
3221
|
+
for (const ref of previousRefs) {
|
|
3222
|
+
auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
|
|
3223
|
+
logger.debug?.('Adding audit ref from audit record to visit queue', {
|
|
3224
|
+
version: ref.version,
|
|
3225
|
+
nodeId: ref.nodeId,
|
|
3226
|
+
});
|
|
3227
|
+
}
|
|
3228
|
+
}
|
|
3229
|
+
};
|
|
3230
|
+
const advanceToPreviousAudit = (auditRecord) => {
|
|
3231
|
+
const previousRefs = auditRecord.previousAdditionalAuditRefs;
|
|
3232
|
+
const previousHead =
|
|
3233
|
+
isRocksDB && previousRefs?.length
|
|
3234
|
+
? resolveAuditHead(id, auditRecord.previousVersion, auditRecord.previousNodeId, previousRefs)
|
|
3235
|
+
: { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
|
|
3236
|
+
localTime = previousHead.txnLogKey;
|
|
3237
|
+
nodeId = previousHead.nodeId;
|
|
3238
|
+
};
|
|
2647
3239
|
let walkSteps = 0;
|
|
2648
3240
|
let auditWalkCapped = false;
|
|
2649
3241
|
// Early-out residual: as we walk the chain newest-first, fold each succeeding patch into a
|
|
@@ -2662,21 +3254,24 @@ export function makeTable(options) {
|
|
|
2662
3254
|
// appended this write's own audit entry, so the lookup would match it while the record was
|
|
2663
3255
|
// never committed (see the up-front keyed dedup above).
|
|
2664
3256
|
const isReDeliveredDuplicate = () => {
|
|
2665
|
-
if (stagedOwnAuditEntry) return false;
|
|
2666
|
-
if (!dedupVersionCouldBeRetained(
|
|
2667
|
-
const duplicate = auditStore.get(
|
|
3257
|
+
if (replaying || stagedOwnAuditEntry) return false;
|
|
3258
|
+
if (!dedupVersionCouldBeRetained(txnLogKey)) return false; // pre-retention log key — skip the end-of-log scan (best-effort; see above)
|
|
3259
|
+
const duplicate = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
|
|
2668
3260
|
return (
|
|
2669
3261
|
duplicate &&
|
|
2670
|
-
duplicate.
|
|
3262
|
+
duplicate.txnLogKey === txnLogKey &&
|
|
2671
3263
|
precedesExistingVersion(
|
|
2672
3264
|
txnTime,
|
|
2673
|
-
{ version: txnTime, localTime:
|
|
3265
|
+
{ version: txnTime, localTime: txnLogKey, key: id, nodeId: duplicate.nodeId },
|
|
2674
3266
|
options?.nodeId
|
|
2675
3267
|
) === 0
|
|
2676
3268
|
);
|
|
2677
3269
|
};
|
|
2678
3270
|
do {
|
|
2679
3271
|
while (localTime > txnTime || (auditedVersion >= txnTime && localTime > 0)) {
|
|
3272
|
+
const auditIdentity = `${nodeId ?? 0}:${localTime}`;
|
|
3273
|
+
if (visitedAuditRefs.has(auditIdentity)) break;
|
|
3274
|
+
visitedAuditRefs.add(auditIdentity);
|
|
2680
3275
|
// Bound the walk only for RocksDB, where the OOM was observed (issue #1114): each step
|
|
2681
3276
|
// is a transaction-log range scan + msgpackr decode, and the per-node logs can be huge.
|
|
2682
3277
|
// LMDB audit entries are keyed by local audit time (not version), so the duplicate
|
|
@@ -2687,6 +3282,21 @@ export function makeTable(options) {
|
|
|
2687
3282
|
}
|
|
2688
3283
|
const auditRecord = auditStore.get(localTime, tableId, id, nodeId);
|
|
2689
3284
|
if (!auditRecord) break;
|
|
3285
|
+
queuePreviousAuditRefs(auditRecord);
|
|
3286
|
+
if (
|
|
3287
|
+
isRocksDB &&
|
|
3288
|
+
!replaying &&
|
|
3289
|
+
!stagedOwnAuditEntry &&
|
|
3290
|
+
localTime === txnLogKey &&
|
|
3291
|
+
precedesExistingVersion(
|
|
3292
|
+
txnTime,
|
|
3293
|
+
{ version: txnTime, localTime: txnLogKey, key: id, nodeId: auditRecord.nodeId },
|
|
3294
|
+
options?.nodeId
|
|
3295
|
+
) === 0
|
|
3296
|
+
) {
|
|
3297
|
+
write.skipped = true;
|
|
3298
|
+
return;
|
|
3299
|
+
}
|
|
2690
3300
|
auditedVersion = auditRecord.version;
|
|
2691
3301
|
if (auditedVersion >= txnTime) {
|
|
2692
3302
|
if (auditedVersion === txnTime) {
|
|
@@ -2696,17 +3306,26 @@ export function makeTable(options) {
|
|
|
2696
3306
|
options?.nodeId
|
|
2697
3307
|
);
|
|
2698
3308
|
if (precedesExisting === 0) {
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
3309
|
+
if (isRocksDB && localTime !== txnLogKey) {
|
|
3310
|
+
// Same origin and record version, but a distinct write. Its per-origin log key
|
|
3311
|
+
// orders the otherwise non-unique record clock without comparing keys across origins.
|
|
3312
|
+
precedesExisting = txnLogKey > localTime ? 1 : -1;
|
|
3313
|
+
} else if (replaying || stagedOwnAuditEntry) {
|
|
3314
|
+
// The log entry being replayed (or staged by this write's failed attempt) is
|
|
3315
|
+
// the write itself, not proof that its primary-store mutation committed.
|
|
3316
|
+
precedesExisting = 1;
|
|
3317
|
+
} else {
|
|
3318
|
+
logger.debug?.(
|
|
3319
|
+
'The transaction time and log key match the existing write, treating as duplicate',
|
|
3320
|
+
id
|
|
3321
|
+
);
|
|
3322
|
+
write.skipped = true;
|
|
3323
|
+
return;
|
|
3324
|
+
}
|
|
2705
3325
|
}
|
|
2706
3326
|
if (precedesExisting > 0) {
|
|
2707
3327
|
// if the existing version is older, we can skip this update
|
|
2708
|
-
|
|
2709
|
-
nodeId = auditRecord.previousNodeId;
|
|
3328
|
+
advanceToPreviousAudit(auditRecord);
|
|
2710
3329
|
continue;
|
|
2711
3330
|
}
|
|
2712
3331
|
}
|
|
@@ -2743,24 +3362,16 @@ export function makeTable(options) {
|
|
|
2743
3362
|
}
|
|
2744
3363
|
if (!addedAuditRef && isRocksDB) {
|
|
2745
3364
|
addedAuditRef = true;
|
|
2746
|
-
// Add a reference to this older audit record if we had out-of-order writes
|
|
2747
|
-
|
|
3365
|
+
// Add a reference to this older audit record if we had out-of-order writes. The stored
|
|
3366
|
+
// value is a LOG key, not a record version: every consumer follows it straight into
|
|
3367
|
+
// `auditStore.get` (see the `auditRefsToVisit` mapping above and below), and on an
|
|
3368
|
+
// applied write those two clocks differ.
|
|
3369
|
+
additionalAuditRefs.push({ version: txnLogKey, nodeId: options?.nodeId });
|
|
2748
3370
|
logger.debug?.('Adding additional audit ref for out-of-order write', {
|
|
2749
|
-
|
|
3371
|
+
txnLogKey,
|
|
2750
3372
|
nodeId: options?.nodeId,
|
|
2751
3373
|
});
|
|
2752
3374
|
}
|
|
2753
|
-
// Collect any additional audit refs from this audit record to traverse other branches
|
|
2754
|
-
if (auditRecord.previousAdditionalAuditRefs) {
|
|
2755
|
-
for (const ref of auditRecord.previousAdditionalAuditRefs) {
|
|
2756
|
-
auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
|
|
2757
|
-
logger.debug?.('Adding audit ref from audit record to visit queue', {
|
|
2758
|
-
version: ref.version,
|
|
2759
|
-
nodeId: ref.nodeId,
|
|
2760
|
-
});
|
|
2761
|
-
}
|
|
2762
|
-
}
|
|
2763
|
-
|
|
2764
3375
|
// Every field of this write is overwritten by newer writes, and there is no alternate
|
|
2765
3376
|
// audit branch left to scan, so it is fully superseded — the same outcome as walking to
|
|
2766
3377
|
// the end and taking the `writeCommit(false)` escape below, reached without paying the rest
|
|
@@ -2779,8 +3390,7 @@ export function makeTable(options) {
|
|
|
2779
3390
|
return writeCommit(false);
|
|
2780
3391
|
}
|
|
2781
3392
|
|
|
2782
|
-
|
|
2783
|
-
nodeId = auditRecord.previousNodeId;
|
|
3393
|
+
advanceToPreviousAudit(auditRecord);
|
|
2784
3394
|
}
|
|
2785
3395
|
// Check if we need to scan additional audit refs from this record
|
|
2786
3396
|
if (auditWalkCapped) break;
|
|
@@ -2967,7 +3577,16 @@ export function makeTable(options) {
|
|
|
2967
3577
|
);
|
|
2968
3578
|
updateIndices(id, existingRecord, recordToStore, transaction && { transaction });
|
|
2969
3579
|
|
|
3580
|
+
// Preserve an addressable audit head when the record and log clocks diverge.
|
|
3581
|
+
if (isRocksDB && audit && !isCopyApply && txnLogKey !== txnTime) {
|
|
3582
|
+
const headIndex = additionalAuditRefs.findIndex(
|
|
3583
|
+
(ref) => ref.version === txnLogKey && (ref.nodeId ?? 0) === (options?.nodeId ?? 0)
|
|
3584
|
+
);
|
|
3585
|
+
if (headIndex > 0) additionalAuditRefs.unshift(additionalAuditRefs.splice(headIndex, 1)[0]);
|
|
3586
|
+
else if (headIndex < 0) additionalAuditRefs.unshift({ version: txnLogKey, nodeId: options?.nodeId });
|
|
3587
|
+
}
|
|
2970
3588
|
writeCommit(true);
|
|
3589
|
+
if (write.trackRecordVersion) write.recordVersionApplied = true;
|
|
2971
3590
|
if (expiresAt >= 0) {
|
|
2972
3591
|
scheduleCleanup(); // arm for replicated writes too, not just local-context writes
|
|
2973
3592
|
// A runtime per-record expiresAt on a table with no table-level expiration/eviction, no expiresAt
|
|
@@ -3002,6 +3621,8 @@ export function makeTable(options) {
|
|
|
3002
3621
|
user: (context as any)?.user,
|
|
3003
3622
|
residencyId,
|
|
3004
3623
|
expiresAt,
|
|
3624
|
+
recordVersion: txnTime,
|
|
3625
|
+
recordNodeId: precedesExisting < 0 ? existingEntry?.nodeId : options?.nodeId,
|
|
3005
3626
|
nodeId: options?.nodeId,
|
|
3006
3627
|
viaNodeId: options?.viaNodeId,
|
|
3007
3628
|
originatingOperation: (context as any)?.originatingOperation,
|
|
@@ -3112,6 +3733,7 @@ export function makeTable(options) {
|
|
|
3112
3733
|
return Boolean(this.#record);
|
|
3113
3734
|
}
|
|
3114
3735
|
_writeDelete(id: Id, options?: any) {
|
|
3736
|
+
this.#assertLiveHandle(id);
|
|
3115
3737
|
const context = this.getContext();
|
|
3116
3738
|
const transaction = txnForContext(context);
|
|
3117
3739
|
checkValidId(id);
|
|
@@ -3123,6 +3745,8 @@ export function makeTable(options) {
|
|
|
3123
3745
|
entry,
|
|
3124
3746
|
chainsStagedState: true,
|
|
3125
3747
|
nodeName: (context as any)?.nodeName,
|
|
3748
|
+
recordVersion: options?.version,
|
|
3749
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
|
|
3126
3750
|
before:
|
|
3127
3751
|
(this.constructor as any).source?.delete && !(context as any)?.source
|
|
3128
3752
|
? (this.constructor as any).source.delete.bind((this.constructor as any).source, id, undefined, context)
|
|
@@ -3135,6 +3759,8 @@ export function makeTable(options) {
|
|
|
3135
3759
|
const priorStagedOp = priorStagedWrite(write);
|
|
3136
3760
|
const priorStaged = priorStagedOp?.stagedEntry;
|
|
3137
3761
|
const existingRecord = priorStaged ? priorStaged.value : existingEntry?.value;
|
|
3762
|
+
const txnLogKey =
|
|
3763
|
+
isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
3138
3764
|
if (retry) {
|
|
3139
3765
|
if (context && existingEntry?.version > (context.lastModified || 0))
|
|
3140
3766
|
context.lastModified = existingEntry.version;
|
|
@@ -3163,6 +3789,11 @@ export function makeTable(options) {
|
|
|
3163
3789
|
viaNodeId: options?.viaNodeId,
|
|
3164
3790
|
transaction,
|
|
3165
3791
|
tableToTrack: tableName,
|
|
3792
|
+
recordVersion: txnTime,
|
|
3793
|
+
additionalAuditRefs:
|
|
3794
|
+
isRocksDB && audit && txnLogKey !== txnTime
|
|
3795
|
+
? [{ version: txnLogKey, nodeId: options?.nodeId }]
|
|
3796
|
+
: undefined,
|
|
3166
3797
|
},
|
|
3167
3798
|
'delete'
|
|
3168
3799
|
);
|
|
@@ -3172,6 +3803,7 @@ export function makeTable(options) {
|
|
|
3172
3803
|
removeEntry(primaryStore, existingEntry, isRocksDB && transaction ? { transaction } : undefined);
|
|
3173
3804
|
}
|
|
3174
3805
|
write.stagedEntry = { value: undefined }; // the key holds no record for the rest of this transaction
|
|
3806
|
+
if (write.trackRecordVersion) write.recordVersionApplied = true;
|
|
3175
3807
|
// the removal supersedes the nearest record an earlier write in this transaction stored
|
|
3176
3808
|
// (older ones were already marked by their staged successors), so its saved blobs are
|
|
3177
3809
|
// cleaned up post-commit unless its audit entry references them
|
|
@@ -3509,6 +4141,10 @@ export function makeTable(options) {
|
|
|
3509
4141
|
}
|
|
3510
4142
|
}
|
|
3511
4143
|
const select = target.select;
|
|
4144
|
+
// Whether the caller supplied real filter conditions — read from the raw request, NOT the
|
|
4145
|
+
// planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and
|
|
4146
|
+
// injected full-scan condition). Used to pick the count-estimate source below.
|
|
4147
|
+
const hasUserConditions = Array.isArray(target.conditions) && target.conditions.length > 0;
|
|
3512
4148
|
if (conditions.length === 0) {
|
|
3513
4149
|
conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }];
|
|
3514
4150
|
}
|
|
@@ -3592,12 +4228,117 @@ export function makeTable(options) {
|
|
|
3592
4228
|
readTxn,
|
|
3593
4229
|
transformToRecord
|
|
3594
4230
|
);
|
|
4231
|
+
const offset = target.offset || 0;
|
|
4232
|
+
const end = target.limit !== undefined ? offset + (target.limit as number) : undefined;
|
|
4233
|
+
// `Prefer: count=` (REST pagination): materialize the requested page and attach a total record
|
|
4234
|
+
// count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once,
|
|
4235
|
+
// windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/
|
|
4236
|
+
// table estimate. Opt-in only — the default streaming path below is untouched.
|
|
4237
|
+
//
|
|
4238
|
+
// Requires a bounded page AND window. Counting is a pagination feature; both the limit and the
|
|
4239
|
+
// offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the
|
|
4240
|
+
// window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/
|
|
4241
|
+
// oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity),
|
|
4242
|
+
// limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the
|
|
4243
|
+
// normal streaming path with no count. This bounds the offset too: without it a huge offset would
|
|
4244
|
+
// postpone the exact guardrail (which only engages past the page) until that offset was scanned.
|
|
4245
|
+
const pageLimit = target.limit as number;
|
|
4246
|
+
if (
|
|
4247
|
+
target.count &&
|
|
4248
|
+
Number.isInteger(pageLimit) &&
|
|
4249
|
+
pageLimit >= 0 &&
|
|
4250
|
+
pageLimit <= MAX_COUNT_PAGE &&
|
|
4251
|
+
Number.isInteger(offset) &&
|
|
4252
|
+
offset >= 0 &&
|
|
4253
|
+
offset + pageLimit <= MAX_EXACT_COUNT_SCAN
|
|
4254
|
+
) {
|
|
4255
|
+
const wantExact = target.count === 'exact';
|
|
4256
|
+
const pageEnd = offset + pageLimit;
|
|
4257
|
+
const countStart = performance.now();
|
|
4258
|
+
// A custom-index (vector/HNSW) traversal returns a bounded, approximate candidate set whose size is
|
|
4259
|
+
// chosen from `minResults` (offset + limit), so `scanned` over it tracks the requested page size, not
|
|
4260
|
+
// the true match count — the same query at limit(5) vs limit(200) would otherwise advertise two
|
|
4261
|
+
// different `count=exact` totals. Any query whose execution touches a custom index is affected: a
|
|
4262
|
+
// custom-index sort (its aligned pseudo-condition lands in `conditions`), a custom-index threshold
|
|
4263
|
+
// filter (an HNSW `lt`/`le` is the same minResults-widened traversal as a sort), or an opaque vector
|
|
4264
|
+
// filter. Report the total as unavailable for those rather than advertising it as count=exact
|
|
4265
|
+
// (mirroring how the estimated branch below bails to null for an opaque row/vector filter). A vector
|
|
4266
|
+
// sort applied as in-memory post-ordering leaves no custom-index condition here and stays exact.
|
|
4267
|
+
const touchesCustomIndex = (conds: any[]): boolean =>
|
|
4268
|
+
conds.some((c: any) => {
|
|
4269
|
+
if (!c) return false;
|
|
4270
|
+
if (c.conditions) return touchesCustomIndex(c.conditions);
|
|
4271
|
+
const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]);
|
|
4272
|
+
return typeof attr === 'string' && Boolean(indices[attr]?.customIndex);
|
|
4273
|
+
});
|
|
4274
|
+
const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions);
|
|
4275
|
+
return (async () => {
|
|
4276
|
+
const page: any = [];
|
|
4277
|
+
let scanned = 0;
|
|
4278
|
+
let exact = true;
|
|
4279
|
+
try {
|
|
4280
|
+
for await (const record of results) {
|
|
4281
|
+
if (scanned >= offset && scanned < pageEnd) page.push(record);
|
|
4282
|
+
scanned++;
|
|
4283
|
+
// A store whose async iterator settles synchronously (the common indexed-scan case) would
|
|
4284
|
+
// otherwise let this drain spin as one uninterrupted microtask run, blocking the event loop
|
|
4285
|
+
// for the whole count. Yield to the macrotask queue periodically so concurrent requests and
|
|
4286
|
+
// I/O still make progress during a large exact scan.
|
|
4287
|
+
if ((scanned & (COUNT_YIELD_INTERVAL - 1)) === 0) await new Promise((resolve) => setImmediate(resolve));
|
|
4288
|
+
// The page window [offset, pageEnd) is always collected in full first — the guardrail
|
|
4289
|
+
// only ever abandons the running TOTAL, never truncates the page body.
|
|
4290
|
+
if (scanned >= pageEnd) {
|
|
4291
|
+
// `estimated` needs nothing past the page; an approximate (vector) exact total is going to
|
|
4292
|
+
// be reported unavailable anyway, so don't drain its tail for a number we won't publish.
|
|
4293
|
+
if (!wantExact || approximateResultSet) break;
|
|
4294
|
+
// `exact` keeps counting the tail, bounded by a row cap AND a time budget so a
|
|
4295
|
+
// large match set can't turn a bounded page fetch into an unbounded scan.
|
|
4296
|
+
if (scanned > MAX_EXACT_COUNT_SCAN || performance.now() - countStart > MAX_EXACT_COUNT_MS) {
|
|
4297
|
+
exact = false;
|
|
4298
|
+
break;
|
|
4299
|
+
}
|
|
4300
|
+
}
|
|
4301
|
+
}
|
|
4302
|
+
} finally {
|
|
4303
|
+
// We own the iteration here (no results.onDone consumer), so release the read
|
|
4304
|
+
// transaction unconditionally — including when the drain throws — or the snapshot leaks.
|
|
4305
|
+
txn.doneReadTxn();
|
|
4306
|
+
}
|
|
4307
|
+
let total: number | null;
|
|
4308
|
+
if (wantExact) {
|
|
4309
|
+
// `scanned` is only an authoritative total when the iteration was exhaustive and deterministic;
|
|
4310
|
+
// an approximate (vector/HNSW) result set is neither, so report the total as unavailable.
|
|
4311
|
+
total = exact && !approximateResultSet ? scanned : null;
|
|
4312
|
+
} else if (boundRowFilter || typeof target.vectorFilter === 'function') {
|
|
4313
|
+
// An opaque row/vector filter shapes the result but isn't reflected in the index/condition
|
|
4314
|
+
// estimate; guessing would both mislead and disclose cardinality the filter hides.
|
|
4315
|
+
total = null;
|
|
4316
|
+
} else if (!hasUserConditions) {
|
|
4317
|
+
total = estimatedEntryCount(primaryStore);
|
|
4318
|
+
} else {
|
|
4319
|
+
// Estimate from the real conditions only — drop the planner's synthetic `sort`
|
|
4320
|
+
// pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality.
|
|
4321
|
+
const est = estimateCondition(TableResource)({
|
|
4322
|
+
conditions: conditions.filter((c: any) => c.comparator !== 'sort'),
|
|
4323
|
+
operator: operator ? String(operator).toLowerCase() : 'and',
|
|
4324
|
+
});
|
|
4325
|
+
total = isFinite(est) ? Math.round(est) : null;
|
|
4326
|
+
}
|
|
4327
|
+
// For an estimate, never report a total below the last row actually returned — keeps the
|
|
4328
|
+
// Content-Range valid (start-end/total) when an estimate undershoots a non-empty page.
|
|
4329
|
+
// Exact totals are authoritative (and an empty page past the end must not be clamped up).
|
|
4330
|
+
if (!wantExact && total != null && page.length > 0 && total < offset + page.length) {
|
|
4331
|
+
total = offset + page.length;
|
|
4332
|
+
}
|
|
4333
|
+
page.recordCount = total;
|
|
4334
|
+
page.recordCountExact = wantExact && exact && !approximateResultSet;
|
|
4335
|
+
page.selectApplied = true;
|
|
4336
|
+
page.getColumns = getColumns;
|
|
4337
|
+
return page;
|
|
4338
|
+
})() as any;
|
|
4339
|
+
}
|
|
3595
4340
|
// apply any offset/limit after all the sorting and filtering
|
|
3596
|
-
if (target.offset || target.limit !== undefined)
|
|
3597
|
-
results = results.slice(
|
|
3598
|
-
target.offset,
|
|
3599
|
-
target.limit !== undefined ? (target.offset || 0) + target.limit : undefined
|
|
3600
|
-
);
|
|
4341
|
+
if (target.offset || target.limit !== undefined) results = results.slice(offset, end);
|
|
3601
4342
|
results.onDone = () => {
|
|
3602
4343
|
results.onDone = null; // ensure that it isn't called twice
|
|
3603
4344
|
txn.doneReadTxn();
|
|
@@ -4056,6 +4797,10 @@ export function makeTable(options) {
|
|
|
4056
4797
|
}
|
|
4057
4798
|
if (!auditStore) throw new Error('Can not subscribe to a table without an audit log');
|
|
4058
4799
|
if (!audit) {
|
|
4800
|
+
// Turning auditing on is a schema write, and a branch's Table classes carry the base's
|
|
4801
|
+
// logical name: without this a subscribe through a branched application would enable
|
|
4802
|
+
// auditing on the live base table for every other consumer, with no DDL call involved.
|
|
4803
|
+
TableResource.assertSchemaMutable('enable auditing for a subscription');
|
|
4059
4804
|
table({ table: tableName, database: databaseName, schemaDefined, attributes, audit: true });
|
|
4060
4805
|
}
|
|
4061
4806
|
const getFullRecord = !request.rawEvents;
|
|
@@ -4110,7 +4855,7 @@ export function makeTable(options) {
|
|
|
4110
4855
|
const subscription = addSubscription(
|
|
4111
4856
|
TableResource,
|
|
4112
4857
|
thisId,
|
|
4113
|
-
function (id: Id, auditRecord?: any,
|
|
4858
|
+
function (id: Id, auditRecord?: any, txnLogKey?: any, beginTxn?: any) {
|
|
4114
4859
|
if (dropDuringReplay) return;
|
|
4115
4860
|
try {
|
|
4116
4861
|
let type = auditRecord.type;
|
|
@@ -4133,8 +4878,7 @@ export function makeTable(options) {
|
|
|
4133
4878
|
// been written, so are fresh in memory.
|
|
4134
4879
|
const entry: Entry = primaryStore.getEntry(id);
|
|
4135
4880
|
if (entry) {
|
|
4136
|
-
|
|
4137
|
-
if (entry.version !== (auditRecord.recordVersion ?? auditRecord.version)) return; // out of order event, with old update, don't send anything
|
|
4881
|
+
if (entry.version !== auditRecord.version) return; // out of order event, with old update, don't send anything
|
|
4138
4882
|
value = entry.value;
|
|
4139
4883
|
type = entry.metadataFlags & INVALIDATED ? 'invalidate' : value ? 'put' : 'delete';
|
|
4140
4884
|
} else {
|
|
@@ -4143,7 +4887,7 @@ export function makeTable(options) {
|
|
|
4143
4887
|
}
|
|
4144
4888
|
const event = {
|
|
4145
4889
|
id,
|
|
4146
|
-
localTime,
|
|
4890
|
+
localTime: txnLogKey,
|
|
4147
4891
|
value,
|
|
4148
4892
|
version: auditRecord.version,
|
|
4149
4893
|
type,
|
|
@@ -4208,11 +4952,11 @@ export function makeTable(options) {
|
|
|
4208
4952
|
if (auditRecord.tableId !== tableId) continue;
|
|
4209
4953
|
const id = auditRecord.recordId;
|
|
4210
4954
|
if (thisId == null || isDescendantId(thisId, id)) {
|
|
4211
|
-
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.
|
|
4955
|
+
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
|
|
4212
4956
|
if (
|
|
4213
4957
|
!send({
|
|
4214
4958
|
id,
|
|
4215
|
-
localTime: auditRecord.
|
|
4959
|
+
localTime: auditRecord.txnLogKey,
|
|
4216
4960
|
value,
|
|
4217
4961
|
version: auditRecord.version,
|
|
4218
4962
|
type: auditRecord.type,
|
|
@@ -4225,7 +4969,7 @@ export function makeTable(options) {
|
|
|
4225
4969
|
if ((await subscription.waitForDrain()) === false) return;
|
|
4226
4970
|
}
|
|
4227
4971
|
}
|
|
4228
|
-
subscription!.startTime = auditRecord.
|
|
4972
|
+
subscription!.startTime = auditRecord.txnLogKey; // update so we don't double send
|
|
4229
4973
|
}
|
|
4230
4974
|
} finally {
|
|
4231
4975
|
// replay is done, we can start sending real-time messages again
|
|
@@ -4257,10 +5001,10 @@ export function makeTable(options) {
|
|
|
4257
5001
|
);
|
|
4258
5002
|
break;
|
|
4259
5003
|
}
|
|
4260
|
-
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.
|
|
5004
|
+
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
|
|
4261
5005
|
const historyEntry = {
|
|
4262
5006
|
id,
|
|
4263
|
-
localTime: auditRecord.
|
|
5007
|
+
localTime: auditRecord.txnLogKey,
|
|
4264
5008
|
value,
|
|
4265
5009
|
version: auditRecord.version,
|
|
4266
5010
|
type: auditRecord.type,
|
|
@@ -4274,7 +5018,7 @@ export function makeTable(options) {
|
|
|
4274
5018
|
if (--count <= 0) break;
|
|
4275
5019
|
}
|
|
4276
5020
|
} catch (error) {
|
|
4277
|
-
logger.error?.('Error getting history entry', auditRecord.
|
|
5021
|
+
logger.error?.('Error getting history entry', auditRecord.txnLogKey, error);
|
|
4278
5022
|
}
|
|
4279
5023
|
}
|
|
4280
5024
|
for (let i = history.length; i > 0;) {
|
|
@@ -4354,6 +5098,12 @@ export function makeTable(options) {
|
|
|
4354
5098
|
logger.trace?.('re-retrieved record', localTime, this.#entry?.localTime);
|
|
4355
5099
|
localTime = entry?.localTime;
|
|
4356
5100
|
}
|
|
5101
|
+
let nodeId = entry?.nodeId;
|
|
5102
|
+
if (isRocksDB && entry) {
|
|
5103
|
+
const head = resolveAuditHead(thisId, entry.version, nodeId, entry.additionalAuditRefs);
|
|
5104
|
+
localTime = head.txnLogKey;
|
|
5105
|
+
nodeId = head.nodeId;
|
|
5106
|
+
}
|
|
4357
5107
|
logger.trace?.('Subscription from', startTime, 'from', thisId, localTime);
|
|
4358
5108
|
if (startTime < localTime) {
|
|
4359
5109
|
// start time specified, get the audit history for this record. Set startTime up
|
|
@@ -4364,7 +5114,6 @@ export function makeTable(options) {
|
|
|
4364
5114
|
const history = [];
|
|
4365
5115
|
let inspected = 0;
|
|
4366
5116
|
let nextTime = localTime;
|
|
4367
|
-
let nodeId = entry?.nodeId;
|
|
4368
5117
|
do {
|
|
4369
5118
|
if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
|
|
4370
5119
|
recordsSinceYield = 0;
|
|
@@ -4389,8 +5138,16 @@ export function makeTable(options) {
|
|
|
4389
5138
|
if (count) count--;
|
|
4390
5139
|
} else if (!isActive()) return;
|
|
4391
5140
|
}
|
|
4392
|
-
|
|
4393
|
-
|
|
5141
|
+
const previousHead = isRocksDB
|
|
5142
|
+
? resolveAuditHead(
|
|
5143
|
+
thisId,
|
|
5144
|
+
auditRecord.previousVersion,
|
|
5145
|
+
auditRecord.previousNodeId,
|
|
5146
|
+
auditRecord.previousAdditionalAuditRefs
|
|
5147
|
+
)
|
|
5148
|
+
: { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
|
|
5149
|
+
nextTime = previousHead.txnLogKey;
|
|
5150
|
+
nodeId = previousHead.nodeId;
|
|
4394
5151
|
} else break;
|
|
4395
5152
|
} while (nextTime > startTime && count !== 0);
|
|
4396
5153
|
for (let i = history.length; i > 0;) {
|
|
@@ -4569,6 +5326,7 @@ export function makeTable(options) {
|
|
|
4569
5326
|
store: primaryStore,
|
|
4570
5327
|
entry: this.#entry,
|
|
4571
5328
|
nodeName: (context as any)?.nodeName,
|
|
5329
|
+
recordVersion: options?.version,
|
|
4572
5330
|
validate: () => {
|
|
4573
5331
|
if (!(context as any)?.source) {
|
|
4574
5332
|
transaction.checkOverloaded();
|
|
@@ -4830,6 +5588,7 @@ export function makeTable(options) {
|
|
|
4830
5588
|
return this.#version;
|
|
4831
5589
|
}
|
|
4832
5590
|
static async addAttributes(attributesToAdd: Attribute[]) {
|
|
5591
|
+
TableResource.assertSchemaMutable('add attributes');
|
|
4833
5592
|
const new_attributes = attributes.slice(0);
|
|
4834
5593
|
for (const attribute of attributesToAdd) {
|
|
4835
5594
|
if (!attribute.name) throw new ClientError('Attribute name is required');
|
|
@@ -4847,6 +5606,7 @@ export function makeTable(options) {
|
|
|
4847
5606
|
return (TableResource as any).indexingOperation;
|
|
4848
5607
|
}
|
|
4849
5608
|
static async removeAttributes(names: string[]) {
|
|
5609
|
+
TableResource.assertSchemaMutable('remove attributes');
|
|
4850
5610
|
const new_attributes = attributes.filter((attribute) => !names.includes(attribute.name));
|
|
4851
5611
|
table({
|
|
4852
5612
|
table: tableName,
|
|
@@ -4867,6 +5627,10 @@ export function makeTable(options) {
|
|
|
4867
5627
|
const stats = primaryStore.getStats();
|
|
4868
5628
|
return (stats.treeBranchPageCount + stats.treeLeafPageCount + stats.overflowPages) * stats.pageSize;
|
|
4869
5629
|
}
|
|
5630
|
+
/** Sizes of this table's durable record-structure dictionaries. */
|
|
5631
|
+
static getStructureCounts(): StructureCounts | undefined {
|
|
5632
|
+
return primaryStore.encoder?.getStructureCounts?.();
|
|
5633
|
+
}
|
|
4870
5634
|
static getAuditSize(): number {
|
|
4871
5635
|
const stats = auditStore?.getStats();
|
|
4872
5636
|
return (
|
|
@@ -5253,29 +6017,94 @@ export function makeTable(options) {
|
|
|
5253
6017
|
this.userSetEmbedders.add(attribute_name);
|
|
5254
6018
|
}
|
|
5255
6019
|
static async deleteHistory(endTime = 0, cleanupDeletedRecords = false): Promise<number> {
|
|
5256
|
-
|
|
6020
|
+
const maxConcurrentRemovals = isRocksDB ? MAX_CONCURRENT_HISTORY_REMOVALS : MAX_CONCURRENT_LMDB_HISTORY_REMOVALS;
|
|
6021
|
+
const inFlightRemovals = new Set<Promise<void>>();
|
|
6022
|
+
const removalSlotWaiters: Array<() => void> = [];
|
|
6023
|
+
let removalsAttempted = 0;
|
|
6024
|
+
let removalsSucceeded = 0;
|
|
6025
|
+
let firstRemovalError: unknown;
|
|
6026
|
+
function startRemoval(remove: () => MaybePromise<void>, errorMessage: string, onSuccess?: () => void): void {
|
|
6027
|
+
removalsAttempted++;
|
|
6028
|
+
const removal = new Promise<void>((resolve) => resolve(remove()))
|
|
6029
|
+
.then(
|
|
6030
|
+
() => {
|
|
6031
|
+
removalsSucceeded++;
|
|
6032
|
+
onSuccess?.();
|
|
6033
|
+
},
|
|
6034
|
+
(error) => {
|
|
6035
|
+
// capture before logging: a throwing logger must not cost us the error we may rethrow
|
|
6036
|
+
if (firstRemovalError === undefined) firstRemovalError = error;
|
|
6037
|
+
harperLogger.warn(errorMessage, error);
|
|
6038
|
+
}
|
|
6039
|
+
)
|
|
6040
|
+
.catch(() => undefined)
|
|
6041
|
+
.finally(() => {
|
|
6042
|
+
inFlightRemovals.delete(removal);
|
|
6043
|
+
removalSlotWaiters.shift()?.();
|
|
6044
|
+
});
|
|
6045
|
+
inFlightRemovals.add(removal);
|
|
6046
|
+
}
|
|
6047
|
+
function queueRemoval(
|
|
6048
|
+
remove: () => MaybePromise<void>,
|
|
6049
|
+
errorMessage: string,
|
|
6050
|
+
onSuccess?: () => void
|
|
6051
|
+
): Promise<void> | undefined {
|
|
6052
|
+
if (inFlightRemovals.size >= maxConcurrentRemovals) {
|
|
6053
|
+
return new Promise<void>((resolve) => {
|
|
6054
|
+
removalSlotWaiters.push(resolve);
|
|
6055
|
+
}).then(() => startRemoval(remove, errorMessage, onSuccess));
|
|
6056
|
+
}
|
|
6057
|
+
startRemoval(remove, errorMessage, onSuccess);
|
|
6058
|
+
}
|
|
6059
|
+
const drainRemovals = () => Promise.all(inFlightRemovals);
|
|
5257
6060
|
let entriesDeleted = 0;
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
6061
|
+
try {
|
|
6062
|
+
for (const auditRecord of auditStore.getRange({
|
|
6063
|
+
start: 1, // must not be zero; see getHistory below for why
|
|
6064
|
+
end: endTime,
|
|
6065
|
+
})) {
|
|
6066
|
+
await rest(); // yield to other async operations
|
|
6067
|
+
if (auditRecord.tableId !== tableId) continue;
|
|
6068
|
+
const backpressure = queueRemoval(
|
|
6069
|
+
() => removeAuditEntry(auditStore, auditRecord),
|
|
6070
|
+
'Error removing audit entry during deleteHistory',
|
|
6071
|
+
() => {
|
|
6072
|
+
entriesDeleted++;
|
|
6073
|
+
}
|
|
6074
|
+
);
|
|
6075
|
+
if (backpressure) await backpressure;
|
|
6076
|
+
}
|
|
6077
|
+
} finally {
|
|
6078
|
+
await drainRemovals();
|
|
5266
6079
|
}
|
|
5267
6080
|
if (cleanupDeletedRecords) {
|
|
5268
6081
|
// this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't
|
|
5269
6082
|
// ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records
|
|
5270
|
-
|
|
5271
|
-
const {
|
|
5272
|
-
|
|
5273
|
-
|
|
5274
|
-
|
|
6083
|
+
try {
|
|
6084
|
+
for (const entry of primaryStore.getRange({ start: 0, versions: true })) {
|
|
6085
|
+
const { key, value, localTime, version } = entry;
|
|
6086
|
+
await rest(); // yield to other async operations
|
|
6087
|
+
const auditTime =
|
|
6088
|
+
isRocksDB && version != null
|
|
6089
|
+
? resolveAuditHead(key, version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
|
|
6090
|
+
: localTime;
|
|
6091
|
+
if (value === null && version != null && auditTime < endTime) {
|
|
6092
|
+
const backpressure = queueRemoval(
|
|
6093
|
+
() => primaryStore.remove(key, version),
|
|
6094
|
+
'Error removing deleted record during deleteHistory'
|
|
6095
|
+
);
|
|
6096
|
+
if (backpressure) await backpressure;
|
|
6097
|
+
}
|
|
5275
6098
|
}
|
|
6099
|
+
} finally {
|
|
6100
|
+
await drainRemovals();
|
|
5276
6101
|
}
|
|
5277
6102
|
}
|
|
5278
|
-
|
|
6103
|
+
if (removalsAttempted > 0 && removalsSucceeded === 0) {
|
|
6104
|
+
// zero progress must not report the same success as "nothing was eligible" (see DESIGN.md);
|
|
6105
|
+
// partial failures stay best-effort, logged and excluded from the returned count
|
|
6106
|
+
throw firstRemovalError ?? new Error('Every removal attempted during deleteHistory failed');
|
|
6107
|
+
}
|
|
5279
6108
|
return entriesDeleted;
|
|
5280
6109
|
}
|
|
5281
6110
|
static async *getHistory(startTime = 0, endTime = Infinity) {
|
|
@@ -5287,10 +6116,11 @@ export function makeTable(options) {
|
|
|
5287
6116
|
if (auditRecord.tableId !== tableId) continue;
|
|
5288
6117
|
yield {
|
|
5289
6118
|
id: auditRecord.recordId,
|
|
5290
|
-
|
|
6119
|
+
// Compatibility-facing LMDB history has always reported/grouped by record version.
|
|
6120
|
+
localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
|
|
5291
6121
|
version: auditRecord.version,
|
|
5292
6122
|
type: auditRecord.type,
|
|
5293
|
-
value: auditRecord.getValue(primaryStore, true, auditRecord.
|
|
6123
|
+
value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
|
|
5294
6124
|
user: auditRecord.user,
|
|
5295
6125
|
operation: auditRecord.originatingOperation,
|
|
5296
6126
|
};
|
|
@@ -5301,7 +6131,9 @@ export function makeTable(options) {
|
|
|
5301
6131
|
if (id == undefined) throw new Error('An id is required');
|
|
5302
6132
|
const entry = primaryStore.getEntry(id);
|
|
5303
6133
|
if (!entry) return history;
|
|
5304
|
-
let nextVersion =
|
|
6134
|
+
let nextVersion = isRocksDB
|
|
6135
|
+
? resolveAuditHead(id, entry.version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
|
|
6136
|
+
: entry.localTime;
|
|
5305
6137
|
if (!nextVersion) throw new Error('The entry does not have a local audit time');
|
|
5306
6138
|
const count = 0;
|
|
5307
6139
|
const auditWindow = 100;
|
|
@@ -5314,17 +6146,25 @@ export function makeTable(options) {
|
|
|
5314
6146
|
if (auditRecord.tableId === tableId && compareKeys(auditRecord.recordId, id) === 0) {
|
|
5315
6147
|
history.splice(insertionPoint, 0, {
|
|
5316
6148
|
id: auditRecord.recordId,
|
|
5317
|
-
localTime: auditRecord.version,
|
|
6149
|
+
localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
|
|
5318
6150
|
version: auditRecord.version,
|
|
5319
6151
|
type: auditRecord.type,
|
|
5320
|
-
// reconstruct each entry's record image as of its own
|
|
6152
|
+
// reconstruct each entry's record image as of its own log position, not the audit
|
|
5321
6153
|
// window boundary (nextVersion), matching getHistory (issue #1330)
|
|
5322
|
-
value: auditRecord.getValue(primaryStore, true, auditRecord.
|
|
6154
|
+
value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
|
|
5323
6155
|
user: auditRecord.user,
|
|
5324
6156
|
operation: auditRecord.originatingOperation,
|
|
5325
6157
|
});
|
|
5326
|
-
|
|
5327
|
-
|
|
6158
|
+
const previousVersion = isRocksDB
|
|
6159
|
+
? resolveAuditHead(
|
|
6160
|
+
id,
|
|
6161
|
+
auditRecord.previousVersion,
|
|
6162
|
+
auditRecord.previousNodeId,
|
|
6163
|
+
auditRecord.previousAdditionalAuditRefs
|
|
6164
|
+
).txnLogKey
|
|
6165
|
+
: auditRecord.previousVersion;
|
|
6166
|
+
if (previousVersion > highestPreviousVersion && previousVersion < start) {
|
|
6167
|
+
highestPreviousVersion = previousVersion;
|
|
5328
6168
|
}
|
|
5329
6169
|
}
|
|
5330
6170
|
}
|
|
@@ -5343,8 +6183,14 @@ export function makeTable(options) {
|
|
|
5343
6183
|
}
|
|
5344
6184
|
return Promise.all(promises);
|
|
5345
6185
|
}
|
|
6186
|
+
/** Release everything makeTable() registered process-wide; the class must not be used afterwards. */
|
|
5346
6187
|
static cleanup() {
|
|
6188
|
+
disposed = true;
|
|
6189
|
+
clearTimeout(cleanupTimer);
|
|
6190
|
+
settlePendingCleanup();
|
|
6191
|
+
clearInterval(recordExpirationInterval);
|
|
5347
6192
|
deleteCallbackHandle?.remove();
|
|
6193
|
+
removeStorageReclamationHandler(primaryStore.path, reclamationHandler);
|
|
5348
6194
|
}
|
|
5349
6195
|
static _readTxnForContext(context) {
|
|
5350
6196
|
return txnForContext(context).getReadTxn();
|
|
@@ -5366,9 +6212,14 @@ export function makeTable(options) {
|
|
|
5366
6212
|
}
|
|
5367
6213
|
);
|
|
5368
6214
|
|
|
5369
|
-
|
|
5370
|
-
|
|
5371
|
-
|
|
6215
|
+
try {
|
|
6216
|
+
TableResource.updatedAttributes(); // on creation, update accessors as well
|
|
6217
|
+
if (expirationMs) TableResource.setTTLExpiration(expirationMs / 1000);
|
|
6218
|
+
if (expiresAtProperty) runRecordExpirationEviction();
|
|
6219
|
+
} catch (error) {
|
|
6220
|
+
TableResource.cleanup();
|
|
6221
|
+
throw error;
|
|
6222
|
+
}
|
|
5372
6223
|
return TableResource;
|
|
5373
6224
|
function updateIndices(id: any, existingRecord: any, record: any, options?: any) {
|
|
5374
6225
|
let hasChanges;
|
|
@@ -5775,6 +6626,22 @@ export function makeTable(options) {
|
|
|
5775
6626
|
return transaction;
|
|
5776
6627
|
}
|
|
5777
6628
|
}
|
|
6629
|
+
/**
|
|
6630
|
+
* Detach an unsaved TransactionWrite that a scoped lock() eagerly staged (see #reloadLocked)
|
|
6631
|
+
* once its handle upgrades to hold: hold staging is deferred and explicit-save-only, so a
|
|
6632
|
+
* dangling scoped write would otherwise auto-commit at the transaction sweep and clobber
|
|
6633
|
+
* whatever the hold write lands. Marking it .dropped lets a later save() on the instance that
|
|
6634
|
+
* owns it (checked via #savingOperation === this write) fall through to the hold branch
|
|
6635
|
+
* instead of resolving a detached, dead reference.
|
|
6636
|
+
*/
|
|
6637
|
+
function detachScopedUpgradeWrite(link: any, keyId: unknown, handle: RecordLockHandle): void {
|
|
6638
|
+
for (const write of link.writes) {
|
|
6639
|
+
if (write && !write.saved && write.lockHandle === handle && writeKeyId(write.key) === keyId) {
|
|
6640
|
+
write.dropped = true;
|
|
6641
|
+
link.detachWrite(write);
|
|
6642
|
+
}
|
|
6643
|
+
}
|
|
6644
|
+
}
|
|
5778
6645
|
function getAttributeValue(entry, attribute_name, context, sort?) {
|
|
5779
6646
|
if (!entry) {
|
|
5780
6647
|
return;
|
|
@@ -5892,6 +6759,11 @@ export function makeTable(options) {
|
|
|
5892
6759
|
const metadataFlags = existingEntry?.metadataFlags;
|
|
5893
6760
|
|
|
5894
6761
|
const existingVersion = existingEntry?.version;
|
|
6762
|
+
const existingRecord = existingEntry?.value;
|
|
6763
|
+
const inheritedTimestamp = context?.timestamp || context?.transaction?.timestamp;
|
|
6764
|
+
const sourceTimestamp =
|
|
6765
|
+
inheritedTimestamp ||
|
|
6766
|
+
(isRocksDB ? (primaryStore as RocksDatabase).getMonotonicTimestamp() : getNextMonotonicTime());
|
|
5895
6767
|
let whenResolved, timer;
|
|
5896
6768
|
// We start by locking the record so that there is only one resolution happening at once;
|
|
5897
6769
|
// if there is already a resolution in process, we want to use the results of that resolution
|
|
@@ -5930,10 +6802,8 @@ export function makeTable(options) {
|
|
|
5930
6802
|
// lock acquired — this request will actually load from source
|
|
5931
6803
|
setLoadedFromSource(target, true);
|
|
5932
6804
|
|
|
5933
|
-
const existingRecord = existingEntry?.value;
|
|
5934
6805
|
// it is important to remember that this is _NOT_ part of the current transaction; nothing is changing
|
|
5935
|
-
// with the canonical data, we are simply fulfilling our local copy of the canonical data
|
|
5936
|
-
// want a timestamp later than the current transaction
|
|
6806
|
+
// with the canonical data, we are simply fulfilling our local copy of the canonical data.
|
|
5937
6807
|
// we create a new context for the source, we want to determine the timestamp and don't want to
|
|
5938
6808
|
// attribute this to the current user
|
|
5939
6809
|
const sourceContext = {
|
|
@@ -5969,13 +6839,37 @@ export function makeTable(options) {
|
|
|
5969
6839
|
// before the drain's fail-closed timeout below.
|
|
5970
6840
|
const commitPromise = transaction(sourceContext, async (_txn) => {
|
|
5971
6841
|
const start = performance.now();
|
|
5972
|
-
let updatedRecord;
|
|
6842
|
+
let updatedRecord, assignCreatedTime, sourceVersion;
|
|
5973
6843
|
let hasChanges, invalidated;
|
|
5974
6844
|
try {
|
|
5975
6845
|
updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry);
|
|
5976
6846
|
invalidated = metadataFlags & INVALIDATED;
|
|
5977
|
-
|
|
5978
|
-
|
|
6847
|
+
const reportedVersion = sourceContext.lastModified;
|
|
6848
|
+
const validReportedVersion =
|
|
6849
|
+
typeof reportedVersion === 'number' &&
|
|
6850
|
+
Number.isFinite(reportedVersion) &&
|
|
6851
|
+
reportedVersion > 0 &&
|
|
6852
|
+
reportedVersion <= MAX_DATE_TIMESTAMP;
|
|
6853
|
+
if (validReportedVersion) {
|
|
6854
|
+
// A record version is also this node's ordering token (precedesExistingVersion), so a
|
|
6855
|
+
// source-reported version ahead of local time would make every subsequent local write look
|
|
6856
|
+
// out-of-order and be discarded until wall-clock caught up — freezing the row. Honor what
|
|
6857
|
+
// the source reports, but never beyond now.
|
|
6858
|
+
const versionCeiling = Math.max(sourceTimestamp, Date.now());
|
|
6859
|
+
sourceVersion = Math.min(reportedVersion, versionCeiling);
|
|
6860
|
+
if (sourceVersion !== reportedVersion) {
|
|
6861
|
+
logger.trace?.(
|
|
6862
|
+
`Capping future source version for ${tableName} id ${id}: ${reportedVersion} -> ${sourceVersion}`
|
|
6863
|
+
);
|
|
6864
|
+
if (!warnedFutureSourceVersion) {
|
|
6865
|
+
warnedFutureSourceVersion = true;
|
|
6866
|
+
logger.warn?.(
|
|
6867
|
+
`The source for ${tableName} reported a lastModified ahead of local time (${new Date(reportedVersion).toISOString()}) for id ${id}; capping cached record versions at local time`
|
|
6868
|
+
);
|
|
6869
|
+
}
|
|
6870
|
+
}
|
|
6871
|
+
} else sourceVersion = sourceTimestamp;
|
|
6872
|
+
hasChanges = invalidated || (validReportedVersion && reportedVersion > existingVersion) || !existingRecord;
|
|
5979
6873
|
const resolveDuration = performance.now() - start;
|
|
5980
6874
|
recordAction(resolveDuration, 'cache-resolution', tableName, null, 'success');
|
|
5981
6875
|
if (responseHeaders)
|
|
@@ -5989,7 +6883,7 @@ export function makeTable(options) {
|
|
|
5989
6883
|
if (status === 304) {
|
|
5990
6884
|
// revalidation of our current cached record
|
|
5991
6885
|
updatedRecord = existingRecord;
|
|
5992
|
-
|
|
6886
|
+
sourceVersion = existingVersion;
|
|
5993
6887
|
} else if (!CACHEABLE_STATUS_CODES.has(status)) {
|
|
5994
6888
|
// non-cacheable status - propagate to client without caching
|
|
5995
6889
|
throw new ServerError(updatedRecord.body || 'Error from source', status);
|
|
@@ -6059,10 +6953,15 @@ export function makeTable(options) {
|
|
|
6059
6953
|
updatedRecord = storedFieldsOnly(primaryStore.encoder, updatedRecord);
|
|
6060
6954
|
if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id;
|
|
6061
6955
|
}
|
|
6956
|
+
assignCreatedTime = createdTimeProperty && updatedRecord?.[createdTimeProperty.name] == null;
|
|
6062
6957
|
resolved = true;
|
|
6958
|
+
const resolvedVersion =
|
|
6959
|
+
isRocksDB && updatedRecord && existingVersion != null
|
|
6960
|
+
? Math.max(sourceVersion, existingVersion)
|
|
6961
|
+
: sourceVersion;
|
|
6063
6962
|
const resolvedEntry: Entry = {
|
|
6064
6963
|
key: id,
|
|
6065
|
-
version,
|
|
6964
|
+
version: resolvedVersion,
|
|
6066
6965
|
value: updatedRecord,
|
|
6067
6966
|
expiresAt: sourceContext.expiresAt,
|
|
6068
6967
|
metadataFlags: 0,
|
|
@@ -6119,16 +7018,33 @@ export function makeTable(options) {
|
|
|
6119
7018
|
const sourceWrite: any = {
|
|
6120
7019
|
key: id,
|
|
6121
7020
|
store: primaryStore,
|
|
6122
|
-
entry:
|
|
7021
|
+
entry: undefined,
|
|
6123
7022
|
nodeName: 'source',
|
|
6124
|
-
commit: (
|
|
7023
|
+
commit: (_txnTime, existingEntry, _retry, transaction: any) => {
|
|
6125
7024
|
sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true
|
|
6126
|
-
|
|
6127
|
-
|
|
7025
|
+
const racedVersion = existingEntry?.version;
|
|
7026
|
+
// A first fill may replace a record that raced it only when its candidate version strictly
|
|
7027
|
+
// orders after that record. The comparison has to be replica-independent, so a tie leaves the
|
|
7028
|
+
// raced record in place: precedesExistingVersion() would break the tie with *this* node's
|
|
7029
|
+
// name, and a fill from a shared source has no node identity of its own, so two replicas
|
|
7030
|
+
// resolving the same tie could keep different values at the same version.
|
|
7031
|
+
const replacesRacedRecord = racedVersion == null || sourceVersion > racedVersion;
|
|
7032
|
+
if (
|
|
7033
|
+
racedVersion !== existingVersion &&
|
|
7034
|
+
// Revalidations retain exact-CAS semantics; first fills use deterministic ordering.
|
|
7035
|
+
(existingVersion != null || !updatedRecord || !replacesRacedRecord)
|
|
7036
|
+
) {
|
|
7037
|
+
logger.trace?.(
|
|
7038
|
+
`Discarding resolved record from source with id: ${id}, source version: ${sourceVersion}, current version: ${racedVersion}`
|
|
7039
|
+
);
|
|
6128
7040
|
sourceWrite.skipped = true;
|
|
6129
7041
|
return;
|
|
6130
7042
|
}
|
|
6131
|
-
|
|
7043
|
+
const currentRecord = existingEntry?.value;
|
|
7044
|
+
const recordVersion =
|
|
7045
|
+
isRocksDB && racedVersion != null ? Math.max(sourceVersion, racedVersion) : sourceVersion;
|
|
7046
|
+
const txnLogKey = isRocksDB ? transaction?.getTimestamp?.() : recordVersion;
|
|
7047
|
+
updateIndices(id, currentRecord, updatedRecord, transaction && { transaction });
|
|
6132
7048
|
if (updatedRecord) {
|
|
6133
7049
|
if (existingEntry) {
|
|
6134
7050
|
context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
|
|
@@ -6139,22 +7055,22 @@ export function makeTable(options) {
|
|
|
6139
7055
|
if (updatedTimeProperty) {
|
|
6140
7056
|
updatedRecord[updatedTimeProperty.name] =
|
|
6141
7057
|
updatedTimeProperty.type === 'Date'
|
|
6142
|
-
? new Date(
|
|
7058
|
+
? new Date(recordVersion)
|
|
6143
7059
|
: updatedTimeProperty.type === 'String'
|
|
6144
|
-
? new Date(
|
|
6145
|
-
:
|
|
7060
|
+
? new Date(recordVersion).toISOString()
|
|
7061
|
+
: recordVersion;
|
|
6146
7062
|
}
|
|
6147
|
-
if (
|
|
6148
|
-
const existingCreatedTime =
|
|
7063
|
+
if (assignCreatedTime) {
|
|
7064
|
+
const existingCreatedTime = currentRecord?.[createdTimeProperty.name];
|
|
6149
7065
|
if (existingCreatedTime != null) {
|
|
6150
7066
|
updatedRecord[createdTimeProperty.name] = existingCreatedTime;
|
|
6151
7067
|
} else {
|
|
6152
7068
|
updatedRecord[createdTimeProperty.name] =
|
|
6153
7069
|
createdTimeProperty.type === 'Date'
|
|
6154
|
-
? new Date(
|
|
7070
|
+
? new Date(recordVersion)
|
|
6155
7071
|
: createdTimeProperty.type === 'String'
|
|
6156
|
-
? new Date(
|
|
6157
|
-
:
|
|
7072
|
+
? new Date(recordVersion).toISOString()
|
|
7073
|
+
: recordVersion;
|
|
6158
7074
|
}
|
|
6159
7075
|
}
|
|
6160
7076
|
const residency = residencyFromFunction(TableResource.getResidency(updatedRecord, context));
|
|
@@ -6186,22 +7102,25 @@ export function makeTable(options) {
|
|
|
6186
7102
|
residencyId = getResidencyId(residency);
|
|
6187
7103
|
}
|
|
6188
7104
|
logger.trace?.(
|
|
6189
|
-
`Writing resolved record from source with id: ${id}, timestamp: ${new Date(
|
|
7105
|
+
`Writing resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`
|
|
6190
7106
|
);
|
|
6191
7107
|
// TODO: We are doing a double check for ifVersion that should probably be cleaned out
|
|
7108
|
+
const writeAudit = (audit && (hasChanges || omitLocalRecord)) || null;
|
|
6192
7109
|
updateRecord(
|
|
6193
7110
|
id,
|
|
6194
7111
|
updatedRecord,
|
|
6195
7112
|
existingEntry,
|
|
6196
|
-
|
|
7113
|
+
recordVersion,
|
|
6197
7114
|
omitLocalRecord ? INVALIDATED : 0,
|
|
6198
|
-
|
|
7115
|
+
writeAudit,
|
|
6199
7116
|
{
|
|
6200
7117
|
user: (sourceContext as any)?.user,
|
|
6201
7118
|
expiresAt: sourceContext.expiresAt,
|
|
6202
7119
|
residencyId,
|
|
6203
7120
|
transaction,
|
|
6204
7121
|
tableToTrack: tableName,
|
|
7122
|
+
additionalAuditRefs:
|
|
7123
|
+
writeAudit && txnLogKey !== recordVersion ? [{ version: txnLogKey, nodeId: 0 }] : undefined,
|
|
6205
7124
|
},
|
|
6206
7125
|
'put',
|
|
6207
7126
|
Boolean(invalidated),
|
|
@@ -6211,17 +7130,26 @@ export function makeTable(options) {
|
|
|
6211
7130
|
if (sourceContext.expiresAt) scheduleCleanup();
|
|
6212
7131
|
} else if (existingEntry) {
|
|
6213
7132
|
logger.trace?.(
|
|
6214
|
-
`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(
|
|
7133
|
+
`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`
|
|
6215
7134
|
);
|
|
6216
7135
|
if (audit || trackDeletes) {
|
|
6217
7136
|
updateRecord(
|
|
6218
7137
|
id,
|
|
6219
7138
|
null,
|
|
6220
7139
|
existingEntry,
|
|
6221
|
-
|
|
7140
|
+
recordVersion,
|
|
6222
7141
|
0,
|
|
6223
7142
|
(audit && hasChanges) || null,
|
|
6224
|
-
{
|
|
7143
|
+
{
|
|
7144
|
+
user: (sourceContext as any)?.user,
|
|
7145
|
+
transaction,
|
|
7146
|
+
tableToTrack: tableName,
|
|
7147
|
+
recordVersion,
|
|
7148
|
+
additionalAuditRefs:
|
|
7149
|
+
audit && hasChanges && txnLogKey !== recordVersion
|
|
7150
|
+
? [{ version: txnLogKey, nodeId: 0 }]
|
|
7151
|
+
: undefined,
|
|
7152
|
+
},
|
|
6225
7153
|
'delete',
|
|
6226
7154
|
Boolean(invalidated)
|
|
6227
7155
|
);
|
|
@@ -6384,7 +7312,13 @@ export function makeTable(options) {
|
|
|
6384
7312
|
},
|
|
6385
7313
|
};
|
|
6386
7314
|
}
|
|
7315
|
+
function settlePendingCleanup() {
|
|
7316
|
+
for (const resolve of pendingCleanupResolvers) resolve();
|
|
7317
|
+
pendingCleanupResolvers.clear();
|
|
7318
|
+
}
|
|
6387
7319
|
function scheduleCleanup(priority?: number): Promise<void> | void {
|
|
7320
|
+
// a reclamation run may still hold this class's handler after cleanup(); a promise here would never settle
|
|
7321
|
+
if (disposed) return;
|
|
6388
7322
|
let runImmediately = false;
|
|
6389
7323
|
if (priority) {
|
|
6390
7324
|
// run immediately if there is a big increase in priority
|
|
@@ -6397,8 +7331,18 @@ export function makeTable(options) {
|
|
|
6397
7331
|
if (getWorkerIndex() === getWorkerCount() - 1) {
|
|
6398
7332
|
// run on the last thread so we aren't overloading lower-numbered threads
|
|
6399
7333
|
if (cleanupTimer) clearTimeout(cleanupTimer);
|
|
6400
|
-
if (!cleanupInterval)
|
|
6401
|
-
|
|
7334
|
+
if (!cleanupInterval) {
|
|
7335
|
+
// no replacement pass is being scheduled, so nothing is left to settle a superseded one
|
|
7336
|
+
settlePendingCleanup();
|
|
7337
|
+
return;
|
|
7338
|
+
}
|
|
7339
|
+
// This pass adopts the awaiters of the pass whose timer it just cleared: they settle when
|
|
7340
|
+
// this pass's scan completes, so a reclamation run is never told the storage was reclaimed
|
|
7341
|
+
// before any scan ran. It has to run now, though — that run blocks its whole path on the
|
|
7342
|
+
// promise, and the replacement's own slot can be a full interval out.
|
|
7343
|
+
if (pendingCleanupResolvers.size > 0) runImmediately = true;
|
|
7344
|
+
return new Promise<void>((resolve) => {
|
|
7345
|
+
pendingCleanupResolvers.add(resolve);
|
|
6402
7346
|
const startOfYear = new Date();
|
|
6403
7347
|
startOfYear.setMonth(0);
|
|
6404
7348
|
startOfYear.setDate(1);
|
|
@@ -6411,6 +7355,7 @@ export function makeTable(options) {
|
|
|
6411
7355
|
? Date.now()
|
|
6412
7356
|
: Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime();
|
|
6413
7357
|
const startNextTimer = (nextScheduled) => {
|
|
7358
|
+
if (disposed) return;
|
|
6414
7359
|
logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`);
|
|
6415
7360
|
// noinspection JSVoidFunctionReturnValueUsed
|
|
6416
7361
|
cleanupTimer = setTimeout(
|
|
@@ -6421,8 +7366,11 @@ export function makeTable(options) {
|
|
|
6421
7366
|
const rootStore = primaryStore.rootStore;
|
|
6422
7367
|
if (rootStore.status !== 'open') {
|
|
6423
7368
|
clearTimeout(cleanupTimer);
|
|
7369
|
+
settlePendingCleanup();
|
|
6424
7370
|
return;
|
|
6425
7371
|
}
|
|
7372
|
+
// snapshot: an awaiter that arrives during this scan belongs to the pass that supersedes it
|
|
7373
|
+
const settling = [...pendingCleanupResolvers];
|
|
6426
7374
|
const MAX_CLEANUP_CONCURRENCY = 50;
|
|
6427
7375
|
const outstandingCleanupOperations = new Array(MAX_CLEANUP_CONCURRENCY);
|
|
6428
7376
|
let cleanupIndex = 0;
|
|
@@ -6502,7 +7450,10 @@ export function makeTable(options) {
|
|
|
6502
7450
|
} catch (error) {
|
|
6503
7451
|
logger.warn?.(`Error in cleanup scan for ${tableName}:`, error);
|
|
6504
7452
|
}
|
|
6505
|
-
|
|
7453
|
+
for (const settle of settling) {
|
|
7454
|
+
pendingCleanupResolvers.delete(settle);
|
|
7455
|
+
settle();
|
|
7456
|
+
}
|
|
6506
7457
|
cleanupPriority = 0; // reset the priority
|
|
6507
7458
|
})),
|
|
6508
7459
|
Math.min(nextScheduled - Date.now(), MAX_SET_TIMEOUT_MS) // make sure it can fit in 32-bit signed number
|
|
@@ -6514,17 +7465,17 @@ export function makeTable(options) {
|
|
|
6514
7465
|
}
|
|
6515
7466
|
function addDeleteRemoval() {
|
|
6516
7467
|
deleteCallbackHandle = auditStore?.addDeleteRemovalCallback(tableId, primaryStore, (id: Id, version: number) => {
|
|
6517
|
-
primaryStore.remove(id, version);
|
|
7468
|
+
return primaryStore.remove(id, version);
|
|
6518
7469
|
});
|
|
6519
7470
|
}
|
|
6520
7471
|
function runRecordExpirationEviction() {
|
|
6521
7472
|
// Periodically evict expired records, searching for records who expiresAt timestamp is before now
|
|
6522
7473
|
if (getWorkerIndex() === 0) {
|
|
6523
7474
|
// we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting
|
|
6524
|
-
setInterval(async () => {
|
|
7475
|
+
recordExpirationInterval = setInterval(async () => {
|
|
6525
7476
|
// go through each database and table and then search for expired entries
|
|
6526
7477
|
// find any entries that are set to expire before now
|
|
6527
|
-
if (runningRecordExpiration) return;
|
|
7478
|
+
if (disposed || runningRecordExpiration) return;
|
|
6528
7479
|
runningRecordExpiration = true;
|
|
6529
7480
|
try {
|
|
6530
7481
|
const expiresAtName = expiresAtProperty.name;
|