@harperfast/harper 5.2.9 → 5.3.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/mcpTools.ts +1 -1
- package/agent/session.ts +25 -14
- package/bin/cliOperations.ts +46 -9
- package/bin/copyDb.ts +282 -59
- package/bin/deploySetup.ts +16 -5
- package/bin/harper.ts +1 -1
- package/bin/help.ts +4 -1
- package/bin/lite.ts +4 -1
- package/bin/restart.ts +59 -3
- package/bin/run.ts +6 -11
- package/bin/upgrade.js +7 -3
- package/bin/workloadIdentity.ts +119 -0
- package/components/Application.ts +2414 -242
- package/components/ApplicationScope.ts +8 -0
- package/components/EntryHandler.ts +59 -39
- package/components/OptionsWatcher.ts +150 -74
- package/components/RuntimeModuleTracker.ts +38 -7
- package/components/Scope.ts +37 -15
- package/components/awaitRestart.ts +84 -0
- package/components/componentLoader.ts +340 -30
- package/components/componentPreparationLock.ts +16 -5
- package/components/mcp/adapters/harperHttp.ts +4 -0
- package/components/mcp/listChanged.ts +4 -0
- package/components/mcp/toolRegistry.ts +2 -0
- package/components/mcp/tools/operations.ts +9 -0
- package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
- package/components/operations.js +270 -78
- package/components/operationsValidation.js +49 -1
- package/components/status/ComponentStatusRegistry.ts +59 -0
- package/config/RootConfigWatcher.ts +80 -34
- package/config/configUtils.ts +291 -34
- package/config/harperConfigEnvVars.ts +170 -27
- package/config-root.schema.json +29 -0
- package/dataLayer/blobBackup.ts +160 -50
- package/dataLayer/delete.ts +6 -1
- package/dataLayer/harperBridge/ResourceBridge.ts +52 -8
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dataLayer/hdbInfoController.ts +34 -1
- package/dataLayer/insert.ts +44 -1
- package/dataLayer/rocksdbBackup.ts +53 -10
- package/dataLayer/schema.ts +11 -1
- package/dataLayer/schemaDescribe.ts +8 -1
- package/dist/agent/mcpTools.js +1 -1
- package/dist/agent/mcpTools.js.map +1 -1
- package/dist/agent/session.d.ts +22 -0
- package/dist/agent/session.js +26 -15
- package/dist/agent/session.js.map +1 -1
- package/dist/bin/cliOperations.js +48 -9
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/copyDb.d.ts +12 -1
- package/dist/bin/copyDb.js +248 -60
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/bin/deploySetup.d.ts +2 -0
- package/dist/bin/deploySetup.js +11 -3
- package/dist/bin/deploySetup.js.map +1 -1
- package/dist/bin/harper.js +1 -1
- package/dist/bin/harper.js.map +1 -1
- package/dist/bin/help.js +4 -1
- package/dist/bin/help.js.map +1 -1
- package/dist/bin/lite.js +4 -1
- package/dist/bin/lite.js.map +1 -1
- package/dist/bin/restart.js +54 -5
- package/dist/bin/restart.js.map +1 -1
- package/dist/bin/run.js +4 -10
- package/dist/bin/run.js.map +1 -1
- package/dist/bin/upgrade.js +4 -3
- package/dist/bin/upgrade.js.map +1 -1
- package/dist/bin/workloadIdentity.d.ts +18 -0
- package/dist/bin/workloadIdentity.js +100 -0
- package/dist/bin/workloadIdentity.js.map +1 -0
- package/dist/components/Application.d.ts +139 -16
- package/dist/components/Application.js +2215 -267
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +8 -0
- package/dist/components/ApplicationScope.js +7 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/EntryHandler.js +26 -10
- package/dist/components/EntryHandler.js.map +1 -1
- package/dist/components/OptionsWatcher.d.ts +1 -0
- package/dist/components/OptionsWatcher.js +141 -74
- package/dist/components/OptionsWatcher.js.map +1 -1
- package/dist/components/RuntimeModuleTracker.js +40 -6
- package/dist/components/RuntimeModuleTracker.js.map +1 -1
- package/dist/components/Scope.js +38 -13
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/awaitRestart.d.ts +33 -0
- package/dist/components/awaitRestart.js +61 -0
- package/dist/components/awaitRestart.js.map +1 -0
- package/dist/components/componentLoader.d.ts +38 -1
- package/dist/components/componentLoader.js +279 -22
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentPreparationLock.d.ts +5 -0
- package/dist/components/componentPreparationLock.js +14 -6
- package/dist/components/componentPreparationLock.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.js +4 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/listChanged.js +5 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +1 -0
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +5 -0
- package/dist/components/mcp/tools/operations.js +9 -0
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/operations.js +231 -77
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.js +49 -1
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
- package/dist/components/status/ComponentStatusRegistry.js +63 -0
- package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
- package/dist/config/RootConfigWatcher.d.ts +7 -1
- package/dist/config/RootConfigWatcher.js +64 -27
- package/dist/config/RootConfigWatcher.js.map +1 -1
- package/dist/config/configUtils.d.ts +9 -1
- package/dist/config/configUtils.js +254 -33
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/harperConfigEnvVars.d.ts +16 -0
- package/dist/config/harperConfigEnvVars.js +162 -25
- package/dist/config/harperConfigEnvVars.js.map +1 -1
- package/dist/dataLayer/blobBackup.d.ts +49 -20
- package/dist/dataLayer/blobBackup.js +139 -50
- package/dist/dataLayer/blobBackup.js.map +1 -1
- package/dist/dataLayer/delete.js +1 -1
- package/dist/dataLayer/delete.js.map +1 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.js +51 -10
- package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
- package/dist/dataLayer/hdbInfoController.d.ts +10 -0
- package/dist/dataLayer/hdbInfoController.js +30 -1
- package/dist/dataLayer/hdbInfoController.js.map +1 -1
- package/dist/dataLayer/insert.d.ts +9 -1
- package/dist/dataLayer/insert.js +30 -0
- package/dist/dataLayer/insert.js.map +1 -1
- package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
- package/dist/dataLayer/rocksdbBackup.js +45 -8
- package/dist/dataLayer/rocksdbBackup.js.map +1 -1
- package/dist/dataLayer/schema.js +8 -0
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +8 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +52 -0
- package/dist/resources/DatabaseTransaction.d.ts +55 -0
- package/dist/resources/DatabaseTransaction.js +459 -45
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.js +22 -4
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
- package/dist/resources/PrimaryRocksDatabase.js +29 -0
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
- package/dist/resources/RecordEncoder.d.ts +20 -0
- package/dist/resources/RecordEncoder.js +94 -9
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +2 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.js +20 -11
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +20 -1
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/RocksIndexStore.d.ts +6 -1
- package/dist/resources/RocksIndexStore.js +24 -9
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.d.ts +14 -1
- package/dist/resources/RocksTransactionLogStore.js +57 -17
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.d.ts +112 -8
- package/dist/resources/Table.js +1073 -191
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/auditStore.d.ts +11 -2
- package/dist/resources/auditStore.js +325 -71
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/blob.d.ts +129 -9
- package/dist/resources/blob.js +992 -114
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/branchDatabase.d.ts +48 -0
- package/dist/resources/branchDatabase.js +892 -0
- package/dist/resources/branchDatabase.js.map +1 -0
- package/dist/resources/crdt.js +50 -12
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/databases.d.ts +150 -0
- package/dist/resources/databases.js +1200 -239
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/defineTable.d.ts +10 -2
- package/dist/resources/defineTable.js +9 -1
- package/dist/resources/defineTable.js.map +1 -1
- package/dist/resources/graphql.d.ts +1 -1
- package/dist/resources/graphql.js +50 -15
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +1 -0
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +6 -2
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/longLivedTransactions.d.ts +71 -0
- package/dist/resources/longLivedTransactions.js +358 -0
- package/dist/resources/longLivedTransactions.js.map +1 -0
- package/dist/resources/models/backendRegistry.d.ts +26 -0
- package/dist/resources/models/backendRegistry.js +60 -2
- package/dist/resources/models/backendRegistry.js.map +1 -1
- package/dist/resources/models/bootstrap.d.ts +33 -1
- package/dist/resources/models/bootstrap.js +416 -31
- package/dist/resources/models/bootstrap.js.map +1 -1
- package/dist/resources/recordLock.d.ts +80 -0
- package/dist/resources/recordLock.js +184 -0
- package/dist/resources/recordLock.js.map +1 -0
- package/dist/resources/replayLogs.d.ts +14 -1
- package/dist/resources/replayLogs.js +152 -24
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/replayLogsGuards.d.ts +94 -7
- package/dist/resources/replayLogsGuards.js +111 -7
- package/dist/resources/replayLogsGuards.js.map +1 -1
- package/dist/resources/search.js +158 -6
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transactionBroadcast.d.ts +1 -1
- package/dist/resources/transactionBroadcast.js +2 -2
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/auth.js +61 -30
- package/dist/security/auth.js.map +1 -1
- package/dist/security/authn/oidc/claims.d.ts +22 -0
- package/dist/security/authn/oidc/claims.js +71 -0
- package/dist/security/authn/oidc/claims.js.map +1 -0
- package/dist/security/authn/oidc/identityToken.d.ts +27 -0
- package/dist/security/authn/oidc/identityToken.js +111 -0
- package/dist/security/authn/oidc/identityToken.js.map +1 -0
- package/dist/security/authn/oidc/jwks.d.ts +25 -0
- package/dist/security/authn/oidc/jwks.js +261 -0
- package/dist/security/authn/oidc/jwks.js.map +1 -0
- package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
- package/dist/security/authn/oidc/providers/generic.js +34 -0
- package/dist/security/authn/oidc/providers/generic.js.map +1 -0
- package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
- package/dist/security/authn/oidc/providers/githubActions.js +129 -0
- package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
- package/dist/security/authn/oidc/providers/index.d.ts +37 -0
- package/dist/security/authn/oidc/providers/index.js +24 -0
- package/dist/security/authn/oidc/providers/index.js.map +1 -0
- package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
- package/dist/security/authn/oidc/tokenExchange.js +306 -0
- package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
- package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
- package/dist/security/authn/oidc/types.d.ts +38 -0
- package/dist/security/authn/oidc/types.js +6 -0
- package/dist/security/authn/oidc/types.js.map +1 -0
- package/dist/security/certificateVerification/index.js +40 -11
- package/dist/security/certificateVerification/index.js.map +1 -1
- package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
- package/dist/security/certificateVerification/trustedIssuers.js +79 -0
- package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
- package/dist/security/certificateVerification/types.d.ts +1 -0
- package/dist/security/credentialProvenance.d.ts +35 -0
- package/dist/security/credentialProvenance.js +51 -0
- package/dist/security/credentialProvenance.js.map +1 -0
- package/dist/security/credentialRejection.d.ts +4 -0
- package/dist/security/credentialRejection.js +24 -0
- package/dist/security/credentialRejection.js.map +1 -0
- package/dist/security/deferredAuthentication.d.ts +36 -0
- package/dist/security/deferredAuthentication.js +70 -0
- package/dist/security/deferredAuthentication.js.map +1 -0
- package/dist/security/impersonation.d.ts +21 -0
- package/dist/security/impersonation.js +108 -9
- package/dist/security/impersonation.js.map +1 -1
- package/dist/security/jsLoader.d.ts +6 -0
- package/dist/security/jsLoader.js +75 -15
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +301 -71
- package/dist/security/keys.js.map +1 -1
- package/dist/security/operationScope.d.ts +21 -0
- package/dist/security/operationScope.js +36 -0
- package/dist/security/operationScope.js.map +1 -0
- package/dist/security/permissionsTranslator.js +21 -0
- package/dist/security/permissionsTranslator.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +19 -1
- package/dist/security/tokenAuthentication.js +191 -10
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/security/user.js +4 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
- package/dist/server/DurableSubscriptionsSession.js +67 -10
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +73 -0
- package/dist/server/REST.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -0
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +11 -0
- package/dist/server/http.js +88 -15
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +8 -1
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/jobs/jobProcess.js +6 -2
- package/dist/server/jobs/jobProcess.js.map +1 -1
- package/dist/server/jobs/jobs.js +4 -1
- package/dist/server/jobs/jobs.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +26 -4
- package/dist/server/liveSubscriptionAuth.js +105 -39
- package/dist/server/liveSubscriptionAuth.js.map +1 -1
- package/dist/server/loadRootComponents.js +49 -10
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/mqtt.d.ts +2 -0
- package/dist/server/mqtt.js +165 -30
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/nodeName.d.ts +2 -0
- package/dist/server/nodeName.js +107 -23
- package/dist/server/nodeName.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +25 -0
- package/dist/server/serverHelpers/Headers.js +139 -1
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.d.ts +9 -0
- package/dist/server/serverHelpers/contentTypes.js +32 -25
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
- package/dist/server/serverHelpers/deployValidationState.js +9 -19
- package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
- package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
- package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
- package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
- package/dist/server/serverHelpers/registeredOperations.js +74 -21
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
- package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
- package/dist/server/serverHelpers/requestSanitization.js +20 -0
- package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +6 -3
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
- package/dist/server/serverHelpers/serverUtilities.js +90 -20
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
- package/dist/server/serverHelpers/uwsServer.js +19 -1
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/static.js +24 -28
- package/dist/server/static.js.map +1 -1
- package/dist/server/storageReclamation.d.ts +12 -0
- package/dist/server/storageReclamation.js +26 -0
- package/dist/server/storageReclamation.js.map +1 -1
- package/dist/server/threads/itc.d.ts +7 -2
- package/dist/server/threads/itc.js +5 -1
- package/dist/server/threads/itc.js.map +1 -1
- package/dist/server/threads/manageThreads.d.ts +26 -3
- package/dist/server/threads/manageThreads.js +517 -65
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.js +89 -26
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadHeapMemory.d.ts +2 -0
- package/dist/server/threads/threadHeapMemory.js +31 -0
- package/dist/server/threads/threadHeapMemory.js.map +1 -0
- package/dist/server/threads/threadServer.js +46 -15
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/sqlEngine/config.d.ts +1 -3
- package/dist/sqlEngine/config.js +19 -16
- package/dist/sqlEngine/config.js.map +1 -1
- package/dist/sqlTranslator/index.d.ts +1 -1
- package/dist/sqlTranslator/index.js +30 -7
- package/dist/sqlTranslator/index.js.map +1 -1
- package/dist/upgrade/directives/5-3-0.d.ts +7 -0
- package/dist/upgrade/directives/5-3-0.js +148 -0
- package/dist/upgrade/directives/5-3-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/OperationFunctionCaller.js +2 -1
- package/dist/utility/OperationFunctionCaller.js.map +1 -1
- package/dist/utility/common_utils.d.ts +16 -0
- package/dist/utility/common_utils.js +32 -6
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/componentNames.d.ts +8 -0
- package/dist/utility/componentNames.js +12 -1
- package/dist/utility/componentNames.js.map +1 -1
- package/dist/utility/environment/environmentManager.js +3 -6
- package/dist/utility/environment/environmentManager.js.map +1 -1
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/environment/systemInformation.js +1 -0
- package/dist/utility/environment/systemInformation.js.map +1 -1
- package/dist/utility/errors/commonErrors.d.ts +2 -0
- package/dist/utility/errors/commonErrors.js +2 -0
- package/dist/utility/errors/commonErrors.js.map +1 -1
- package/dist/utility/errors/hdbError.d.ts +17 -0
- package/dist/utility/errors/hdbError.js +30 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/globalSchema.d.ts +18 -0
- package/dist/utility/hdbTerms.d.ts +16 -0
- package/dist/utility/hdbTerms.js +18 -2
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +2 -0
- package/dist/utility/logging/harper_logger.js +91 -14
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logRotator.js +76 -49
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/nodeIdentity.d.ts +9 -0
- package/dist/utility/nodeIdentity.js +58 -0
- package/dist/utility/nodeIdentity.js.map +1 -0
- package/dist/utility/npmUtilities.js +9 -7
- package/dist/utility/npmUtilities.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +3 -1
- package/dist/utility/operationPermissions.js +16 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +10 -7
- package/dist/utility/operation_authorization.js +212 -41
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/watchPath.d.ts +29 -0
- package/dist/utility/watchPath.js +68 -0
- package/dist/utility/watchPath.js.map +1 -0
- package/dist/utility/watcherFallback.d.ts +86 -0
- package/dist/utility/watcherFallback.js +278 -1
- package/dist/utility/watcherFallback.js.map +1 -1
- package/dist/validation/configValidator.d.ts +12 -0
- package/dist/validation/configValidator.js +199 -72
- package/dist/validation/configValidator.js.map +1 -1
- package/dist/validation/installValidator.js +12 -0
- package/dist/validation/installValidator.js.map +1 -1
- package/dist/validation/validationWrapper.d.ts +11 -0
- package/dist/validation/validationWrapper.js +16 -3
- package/dist/validation/validationWrapper.js.map +1 -1
- package/index.ts +2 -0
- package/json/systemSchema.json +52 -0
- package/npm-shrinkwrap.json +196 -194
- package/package.json +7 -6
- package/resources/DESIGN.md +114 -52
- package/resources/DatabaseTransaction.ts +521 -43
- package/resources/LMDBTransaction.ts +26 -4
- package/resources/PrimaryRocksDatabase.ts +28 -1
- package/resources/RecordEncoder.ts +103 -8
- package/resources/RequestTarget.ts +2 -0
- package/resources/Resource.ts +17 -9
- package/resources/ResourceInterface.ts +23 -0
- package/resources/RocksIndexStore.ts +30 -9
- package/resources/RocksTransactionLogStore.ts +78 -21
- package/resources/Table.ts +1110 -159
- package/resources/auditStore.ts +331 -73
- package/resources/blob.ts +1029 -110
- package/resources/branchDatabase.ts +941 -0
- package/resources/crdt.ts +76 -12
- package/resources/databases.ts +1310 -252
- package/resources/defineTable.ts +18 -2
- package/resources/graphql.ts +70 -16
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +6 -2
- package/resources/longLivedTransactions.ts +360 -0
- package/resources/models/backendRegistry.ts +84 -2
- package/resources/models/bootstrap.ts +473 -28
- package/resources/recordLock.ts +253 -0
- package/resources/replayLogs.ts +147 -26
- package/resources/replayLogsGuards.ts +171 -8
- package/resources/search.ts +154 -7
- package/resources/transactionBroadcast.ts +3 -3
- package/security/auth.ts +68 -29
- package/security/authn/oidc/claims.ts +72 -0
- package/security/authn/oidc/identityToken.ts +129 -0
- package/security/authn/oidc/jwks.ts +260 -0
- package/security/authn/oidc/providers/generic.ts +40 -0
- package/security/authn/oidc/providers/githubActions.ts +137 -0
- package/security/authn/oidc/providers/index.ts +52 -0
- package/security/authn/oidc/tokenExchange.ts +300 -0
- package/security/authn/oidc/trustPolicyOperations.ts +343 -0
- package/security/authn/oidc/types.ts +41 -0
- package/security/certificateVerification/index.ts +54 -13
- package/security/certificateVerification/trustedIssuers.ts +76 -0
- package/security/certificateVerification/types.ts +1 -0
- package/security/credentialProvenance.ts +47 -0
- package/security/credentialRejection.ts +22 -0
- package/security/deferredAuthentication.ts +71 -0
- package/security/impersonation.ts +117 -12
- package/security/jsLoader.ts +81 -18
- package/security/keys.ts +298 -72
- package/security/operationScope.ts +33 -0
- package/security/permissionsTranslator.js +23 -0
- package/security/tokenAuthentication.ts +233 -12
- package/security/user.ts +4 -3
- package/server/DESIGN.md +183 -16
- package/server/DurableSubscriptionsSession.ts +71 -11
- package/server/REST.ts +79 -1
- package/server/graphqlQuerying.ts +4 -0
- package/server/http.ts +99 -18
- package/server/itc/serverHandlers.js +8 -1
- package/server/jobs/jobProcess.ts +8 -2
- package/server/jobs/jobs.ts +4 -1
- package/server/liveSubscriptionAuth.ts +129 -46
- package/server/loadRootComponents.js +50 -8
- package/server/mqtt.ts +179 -38
- package/server/nodeName.ts +103 -21
- package/server/serverHelpers/Headers.ts +131 -0
- package/server/serverHelpers/contentTypes.ts +29 -21
- package/server/serverHelpers/deployValidationState.ts +24 -13
- package/server/serverHelpers/operationAuthorizationState.ts +34 -3
- package/server/serverHelpers/registeredOperations.ts +79 -22
- package/server/serverHelpers/requestSanitization.ts +15 -0
- package/server/serverHelpers/serverHandlers.js +6 -3
- package/server/serverHelpers/serverUtilities.ts +104 -31
- package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
- package/server/serverHelpers/uwsServer.ts +17 -2
- package/server/static.ts +23 -29
- package/server/storageReclamation.ts +23 -0
- package/server/threads/itc.js +11 -1
- package/server/threads/manageThreads.js +526 -56
- package/server/threads/socketRouter.ts +81 -26
- package/server/threads/threadHeapMemory.ts +26 -0
- package/server/threads/threadServer.js +43 -15
- package/sqlTranslator/index.ts +31 -8
- package/studio/web/assets/Chat-BnCBegQz.js +2267 -0
- package/studio/web/assets/FloatingChat-CoDW1ySS.js +23 -0
- package/studio/web/assets/{abnfDiagram-VCTEODGH-C0_BAZyO.js → abnfDiagram-VCTEODGH-B0BebmD2.js} +1 -1
- package/studio/web/assets/{alertDialog-DIHt7Z0r.js → alertDialog-CQyAJJhl.js} +1 -1
- package/studio/web/assets/{apiToken-c3Rd-w6g.js → apiToken-Bwk5BLXW.js} +1 -1
- package/studio/web/assets/applications-DHxGi7JH.js +296 -0
- package/studio/web/assets/architecture-7GRP2DOG-LB-MLAAb.js +1 -0
- package/studio/web/assets/{architectureDiagram-5GKGNRK7-BWzrASgm.js → architectureDiagram-5GKGNRK7-7SW3GD-K.js} +1 -1
- package/studio/web/assets/authStore-C3Nfubqr.js +3 -0
- package/studio/web/assets/{blockDiagram-NRAW4CY4-BdJX9Khj.js → blockDiagram-I7D4REHJ-BqguiadH.js} +2 -2
- package/studio/web/assets/{button-DhiX-njv.js → button-BIsUKRZq.js} +2 -2
- package/studio/web/assets/{c4Diagram-UCG6FXSJ-CI6MzGmQ.js → c4Diagram-7LVT6UL2-LBNf8t_X.js} +1 -1
- package/studio/web/assets/channel-yictG-U-.js +1 -0
- package/studio/web/assets/{chevron-up-Df2c8uoU.js → chevron-up-DtKGqDn3.js} +1 -1
- package/studio/web/assets/{chunk-TEH6E4GO-P87k5mNi.js → chunk-4HAMMTFA-DWtTut21.js} +1 -1
- package/studio/web/assets/{chunk-75Z2AOVW-BT8tVmks.js → chunk-75Z2AOVW-QGQD6th2.js} +1 -1
- package/studio/web/assets/{chunk-DU6HZSFF-9kAOOmI6.js → chunk-DU6HZSFF-Chq20Ba5.js} +1 -1
- package/studio/web/assets/{chunk-F27PBJKO-BW7ao8AY.js → chunk-F27PBJKO-BVA5EPhV.js} +1 -1
- package/studio/web/assets/{chunk-GMAD6QVW-BNyXpoQO.js → chunk-GMAD6QVW-BeS7S07A.js} +1 -1
- package/studio/web/assets/{chunk-OBVCFTLP-D4wWOqDQ.js → chunk-GVQU2GXP-sbwVIQ8i.js} +1 -1
- package/studio/web/assets/{chunk-G27WJ6UU-COyLMcgK.js → chunk-IMKFNOWR-Bnh3tAVd.js} +1 -1
- package/studio/web/assets/{chunk-JQ64N6SF-Cyz1IeLf.js → chunk-L3NEJ4N5-COfUyKII.js} +1 -1
- package/studio/web/assets/chunk-OSK3NFVY-ByciRftO.js +10 -0
- package/studio/web/assets/{chunk-P2QGCYS3-DmIFY4d7.js → chunk-P2QGCYS3-CP1VhG_c.js} +1 -1
- package/studio/web/assets/{chunk-POPQ4Y6H-BPrvMyKz.js → chunk-POPQ4Y6H-ClWhhkwW.js} +1 -1
- package/studio/web/assets/{chunk-PWAF6VOD-2zB6IW9i.js → chunk-PWAF6VOD-1z1THyS5.js} +1 -1
- package/studio/web/assets/{chunk-RHFEMEQ7-2FgyI8YU.js → chunk-SHT3W25Y-LpQkMsah.js} +2 -2
- package/studio/web/assets/{chunk-SVP7TREG-FwtbH2QC.js → chunk-SVP7TREG-jtdAHw0S.js} +1 -1
- package/studio/web/assets/{chunk-LCL6LL3I-HOzK_ppE.js → chunk-TICWLB2K-VOwzetX-.js} +1 -1
- package/studio/web/assets/classDiagram-ZZMXUADV-VaEwSy_g.js +1 -0
- package/studio/web/assets/classDiagram-v2-VYDZK3BY-VaEwSy_g.js +1 -0
- package/studio/web/assets/{createLucideIcon-BKGPfjm2.js → createLucideIcon-CzW9508A.js} +1 -1
- package/studio/web/assets/{cssMode-CEN2mzSA.js → cssMode-s0cWI-_M.js} +1 -1
- package/studio/web/assets/{cynefin-OW5HDTMX-BRkpLFQV.js → cynefin-OW5HDTMX-BbdbCvub.js} +1 -1
- package/studio/web/assets/{cynefinDiagram-5FMLGOSQ-CHT1DaX6.js → cynefinDiagram-5FMLGOSQ-TP-aIqbt.js} +1 -1
- package/studio/web/assets/{dagre-3AP2YEHR-DpUXBh63.js → dagre-GXQ25YYZ-DShnGpGo.js} +1 -1
- package/studio/web/assets/{diagram-S7CK7UJ4-BuymVFZT.js → diagram-S7CK7UJ4-aoCVTtcy.js} +1 -1
- package/studio/web/assets/{diagram-UQ7AKVKN-CyP148RM.js → diagram-UQ7AKVKN-DglXtQ6x.js} +1 -1
- package/studio/web/assets/{diagram-VSXAHHWV-CoCAg3M9.js → diagram-VSXAHHWV-fhEdmkwM.js} +1 -1
- package/studio/web/assets/{diagram-VX7I27RA-BpOqCFca.js → diagram-VX7I27RA-DccVJet6.js} +1 -1
- package/studio/web/assets/{diagram-Z3DM3KII-Bfpw7Vbj.js → diagram-Z3DM3KII-D-RyJJb7.js} +1 -1
- package/studio/web/assets/{dialog-CBf0Mr1d.js → dialog-Cn2uWgD4.js} +1 -1
- package/studio/web/assets/{dist-lkA3O3eM.js → dist-DP8UjMB_.js} +1 -1
- package/studio/web/assets/{download-BtTOBem-.js → download-B5T5r7ss.js} +1 -1
- package/studio/web/assets/{ebnfDiagram-PWID7BFC-DS_6aWqL.js → ebnfDiagram-PWID7BFC-DJGpIpz_.js} +1 -1
- package/studio/web/assets/{editor-D8oDeCTL.js → editor-DNcRHK54.js} +1 -1
- package/studio/web/assets/{erDiagram-SSCWMZ5O-DJNk6Fgw.js → erDiagram-RLTQ6QDP-CIfNlgkC.js} +1 -1
- package/studio/web/assets/eventmodeling-NTZA5JFV-CLxnp2CR.js +1 -0
- package/studio/web/assets/flowDiagram-HODETNUW-BIbhmz9f.js +1 -0
- package/studio/web/assets/{ganttDiagram-EL5Y4UJY-2pOExxMY.js → ganttDiagram-EL5Y4UJY-BxToTzzD.js} +1 -1
- package/studio/web/assets/{getAnalytics-D4LKGeVy.js → getAnalytics-GHK8ORfM.js} +1 -1
- package/studio/web/assets/{gitGraph-4MIJSDKK-CH5ZxwzF.js → gitGraph-4MIJSDKK-D2s2w8lE.js} +1 -1
- package/studio/web/assets/{gitGraphDiagram-WWUBYQGX-DVIsIhbO.js → gitGraphDiagram-WWUBYQGX-Dwntd4-x.js} +1 -1
- package/studio/web/assets/{html-u3vOg7LJ.js → html-Bdssedlg.js} +1 -1
- package/studio/web/assets/{htmlMode-DyO31v-P.js → htmlMode-CoDlJ3fw.js} +1 -1
- package/studio/web/assets/{index-Cxj2_wsl.css → index-7RMEgVG1.css} +1 -1
- package/studio/web/assets/index-D6sxmFLR.js +824 -0
- package/studio/web/assets/index.lazy-tmU5BS8s.js +2 -0
- package/studio/web/assets/{info-A6RAGUB7-CPQfTnaG.js → info-A6RAGUB7-DYjkvb0C.js} +1 -1
- package/studio/web/assets/{infoDiagram-RXCK75RN-DlwLYlwm.js → infoDiagram-27XIBGKW-Bnp1FJE5.js} +1 -1
- package/studio/web/assets/{ishikawaDiagram-5VMMS53U-BRXRp29U.js → ishikawaDiagram-5VMMS53U-D9Xh2r6X.js} +1 -1
- package/studio/web/assets/{javascript-CUvxOyTC.js → javascript-B8meVSTH.js} +1 -1
- package/studio/web/assets/{journeyDiagram-EYS64GPL-B0ou8k0n.js → journeyDiagram-3NMN7TZE-CokIi6ll.js} +2 -2
- package/studio/web/assets/{jsonMode-f_IwbF3D.js → jsonMode-DpIPd35T.js} +1 -1
- package/studio/web/assets/{kanban-definition-3QL26DDD-uYg7iYzp.js → kanban-definition-UXKFOSKX-CukSFJfX.js} +1 -1
- package/studio/web/assets/{languageServices-DXtZ6rEF.js → languageServices-C_5FMJzQ.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-B4pCF1zO.js → lspLanguageFeatures-BIzNBzPK.js} +1 -1
- package/studio/web/assets/{mermaid-parser.core-Ck-fC8b7.js → mermaid-parser.core-BlEsOWNO.js} +3 -3
- package/studio/web/assets/{mermaid.core-CP8aNNYm.js → mermaid.core-BlkGaMIH.js} +5 -5
- package/studio/web/assets/{mindmap-definition-FBJOCRG2-CgTZ-rit.js → mindmap-definition-YA3MSWOX-IprMc_0j.js} +1 -1
- package/studio/web/assets/{notifications-D3tIQ4sg.js → notifications-CQf18QKb.js} +1 -1
- package/studio/web/assets/notifications-CvZivSbh.js +1 -0
- package/studio/web/assets/{packet-AYTQ26CC-DEyoPtPb.js → packet-AYTQ26CC-Bi3V04Zi.js} +1 -1
- package/studio/web/assets/{pegDiagram-XKGWAZYB-DrD-7sD9.js → pegDiagram-XKGWAZYB-BNuPDLZY.js} +1 -1
- package/studio/web/assets/{pie-WAS4IAKB-wjj-EI1d.js → pie-WAS4IAKB-_6DoDbng.js} +1 -1
- package/studio/web/assets/{pieDiagram-E7YTZNPT-GntqDCzv.js → pieDiagram-E7YTZNPT-DqNb6Ht2.js} +1 -1
- package/studio/web/assets/{profile-DZWU7MgT.js → profile-DdOwtntb.js} +1 -1
- package/studio/web/assets/{quadrantDiagram-AXDQQJYC-0UeqXQGd.js → quadrantDiagram-AXDQQJYC-BGH9E2YR.js} +1 -1
- package/studio/web/assets/{radar-RG4KPBEZ-DFSA5h7k.js → radar-RG4KPBEZ-DDdVczcL.js} +1 -1
- package/studio/web/assets/{railroad-74A4TZTK-CaOUG9wR.js → railroad-74A4TZTK-BJUP4Jds.js} +1 -1
- package/studio/web/assets/railroad-abnf-HS5TGJTU-Bm3L1L0h.js +1 -0
- package/studio/web/assets/railroad-ebnf-LZEXJU2U-CKzLGlkw.js +1 -0
- package/studio/web/assets/railroad-peg-WCYAUIDC-S8xLjslx.js +1 -0
- package/studio/web/assets/{railroadDiagram-O6MQD6OU-yHUELZaV.js → railroadDiagram-O6MQD6OU-DGTPh2KZ.js} +1 -1
- package/studio/web/assets/{regions-CkyurXzE.js → regions-n69fwagr.js} +1 -1
- package/studio/web/assets/{register-BUyhWjBO.js → register-PfWTCXWB.js} +3 -3
- package/studio/web/assets/{requirementDiagram-EFPCY7ZU-DNEGFjuW.js → requirementDiagram-BXWQKSXE-BJnO6uLz.js} +1 -1
- package/studio/web/assets/{sankeyDiagram-P5KCCOFB-DpyAmSVR.js → sankeyDiagram-P5KCCOFB-0vSOdymH.js} +1 -1
- package/studio/web/assets/{sequenceDiagram-WJ2MYXX4-TyaT7xNk.js → sequenceDiagram-WJ2MYXX4-hETizDWE.js} +1 -1
- package/studio/web/assets/{setComponentFile-CeyKSZAa.js → setComponentFile-Bg6O7X0S.js} +1 -1
- package/studio/web/assets/{setup-D2kn7cAA.js → setup-CUx_aUDl.js} +2 -2
- package/studio/web/assets/{stateDiagram-HBIQ2CUA-CeEdTArZ.js → stateDiagram-D77RDMKH-CdYQ_KtC.js} +1 -1
- package/studio/web/assets/stateDiagram-v2-MP3YSRHH-CdKuzQMT.js +1 -0
- package/studio/web/assets/status-D7BVKqX9.js +61 -0
- package/studio/web/assets/{swimlanes-XN3QIQJK-B54FmF46.js → swimlanes-42K2YHIH-B8cHIpU4.js} +1 -1
- package/studio/web/assets/swimlanesDiagram-VR7AAH4N-DmOSJwaH.js +8 -0
- package/studio/web/assets/{tabs-B_G5zscN.js → tabs-BrHu7gJi.js} +1 -1
- package/studio/web/assets/{timeline-definition-24CTP7MA-D-a9ujbo.js → timeline-definition-24CTP7MA-BJWYSXqF.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-C0UW6rI2.js → toggleHighContrast-DBSyXzMr.js} +1 -1
- package/studio/web/assets/{treeView-Q6P3EWNA-CrW_6JnS.js → treeView-Q6P3EWNA-qxe_v6CQ.js} +1 -1
- package/studio/web/assets/{treemap-WGGIJYW6-BxyYLdP_.js → treemap-WGGIJYW6-dDo97XXF.js} +1 -1
- package/studio/web/assets/{tsMode-DBC0zmDx.js → tsMode-BByKCjBS.js} +1 -1
- package/studio/web/assets/{typescript-DApRQir3.js → typescript-DDLnLpw9.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-DB6JStU1.js → useEntityRestURL-31CHGaHk.js} +1 -1
- package/studio/web/assets/{useLocalStorage-Dtj1QS8_.js → useLocalStorage-BqMR3D8_.js} +1 -1
- package/studio/web/assets/vendor-core-c2JRRJpV.js +58 -0
- package/studio/web/assets/vendor-datadog-CLUcJXOo.js +6 -0
- package/studio/web/assets/{vendor-react-Dyj4O3HE.js → vendor-react-CJV_K1u4.js} +1 -1
- package/studio/web/assets/vendor-tanstack-DxzraizX.js +1 -0
- package/studio/web/assets/{vendor-ui-vhu-UHhF.js → vendor-ui-BUjK0h8a.js} +2 -2
- package/studio/web/assets/{vennDiagram-4TSXK5OY-Cy7s7Mpy.js → vennDiagram-4TSXK5OY-A3i-lCdl.js} +1 -1
- package/studio/web/assets/{wardley-WFR3VGLG-BeBL35g2.js → wardley-WFR3VGLG-B0ik-_6g.js} +1 -1
- package/studio/web/assets/{wardleyDiagram-VM6X3IG4-BylmIGSg.js → wardleyDiagram-VM6X3IG4-CjrkKWUR.js} +1 -1
- package/studio/web/assets/{workers-C0bFIedw.js → workers-pR3jRY9D.js} +1 -1
- package/studio/web/assets/x-DIzaLEdK.js +1 -0
- package/studio/web/assets/{xml-HWd01lU-.js → xml-2iRnMhQO.js} +1 -1
- package/studio/web/assets/{xychartDiagram-S5SC5T6Z-CoKALMXr.js → xychartDiagram-S5SC5T6Z-Biok4GYV.js} +1 -1
- package/studio/web/assets/{yaml-CIH0Nt-h.js → yaml-Bf92gJpd.js} +1 -1
- package/studio/web/index.html +14 -14
- package/upgrade/directives/5-3-0.ts +132 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/OperationFunctionCaller.ts +2 -1
- package/utility/common_utils.ts +30 -5
- package/utility/componentNames.ts +12 -0
- package/utility/environment/environmentManager.ts +3 -7
- package/utility/environment/systemInformation.ts +7 -0
- package/utility/errors/commonErrors.ts +4 -0
- package/utility/errors/hdbError.ts +29 -0
- package/utility/hdbTerms.ts +17 -0
- package/utility/logging/harper_logger.ts +87 -14
- package/utility/logging/logRotator.ts +72 -45
- package/utility/nodeIdentity.ts +45 -0
- package/utility/npmUtilities.ts +10 -8
- package/utility/operationPermissions.ts +18 -1
- package/utility/operation_authorization.ts +231 -42
- package/utility/watchPath.ts +63 -0
- package/utility/watcherFallback.ts +270 -0
- package/validation/configValidator.ts +209 -72
- package/validation/installValidator.ts +15 -0
- package/validation/validationWrapper.ts +18 -4
- package/studio/web/assets/Chat-JpO8EtUu.js +0 -2067
- package/studio/web/assets/FloatingChat-Bcj3xSZu.js +0 -23
- package/studio/web/assets/applications-ByqLRKyZ.js +0 -296
- package/studio/web/assets/architecture-7GRP2DOG-DNdx5tEU.js +0 -1
- package/studio/web/assets/authStore-qKmCZcaf.js +0 -3
- package/studio/web/assets/channel-DtCV8PTL.js +0 -1
- package/studio/web/assets/chunk-R7TYR2AO-Irip67yr.js +0 -10
- package/studio/web/assets/classDiagram-DTDB5LWJ-DbO_dCNE.js +0 -1
- package/studio/web/assets/classDiagram-v2-JRS7N3AN-DbO_dCNE.js +0 -1
- package/studio/web/assets/eventmodeling-NTZA5JFV-5jbe4A5P.js +0 -1
- package/studio/web/assets/flowDiagram-A5DVABFB-Dp9Ezlow.js +0 -1
- package/studio/web/assets/index-aSt5tY-L.js +0 -824
- package/studio/web/assets/index.lazy-B9jiPwT8.js +0 -2
- package/studio/web/assets/notifications-CUoYgU98.js +0 -1
- package/studio/web/assets/railroad-abnf-HS5TGJTU-Bc0Qi0WH.js +0 -1
- package/studio/web/assets/railroad-ebnf-LZEXJU2U-G8rVVZ2C.js +0 -1
- package/studio/web/assets/railroad-peg-WCYAUIDC-CrehKBhC.js +0 -1
- package/studio/web/assets/stateDiagram-v2-4QOOHH4V-D4tuw9Su.js +0 -1
- package/studio/web/assets/status-D7Xn5ePA.js +0 -61
- package/studio/web/assets/swimlanesDiagram-VK2B7HYN-XOhmNEvq.js +0 -8
- package/studio/web/assets/vendor-core-RCcadM3e.js +0 -73
- package/studio/web/assets/vendor-datadog-BRv-mOv1.js +0 -6
- package/studio/web/assets/vendor-tanstack-BiFWSB3W.js +0 -1
- package/studio/web/assets/x-B9o9hsep.js +0 -1
- /package/studio/web/assets/{sizeCapture-X5ZJPWSS-B0uUizjq.js → sizeCapture-INFHLROL-B0uUizjq.js} +0 -0
package/dist/resources/Table.js
CHANGED
|
@@ -41,7 +41,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
41
41
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
42
42
|
};
|
|
43
43
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
44
|
-
exports.EVICTED = exports.INVALIDATED = void 0;
|
|
44
|
+
exports.EVICTED = exports.INVALIDATED = exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT = exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT = void 0;
|
|
45
|
+
exports.acquireUpdateAttributesLock = acquireUpdateAttributesLock;
|
|
46
|
+
exports.releaseUpdateAttributesLock = releaseUpdateAttributesLock;
|
|
47
|
+
exports.withUpdateAttributesLock = withUpdateAttributesLock;
|
|
45
48
|
exports.ignoreAlreadyDropped = ignoreAlreadyDropped;
|
|
46
49
|
exports.freezeRecord = freezeRecord;
|
|
47
50
|
exports.frozenRecordView = frozenRecordView;
|
|
@@ -50,6 +53,7 @@ exports.coerceType = coerceType;
|
|
|
50
53
|
const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
|
|
51
54
|
const node_vm_1 = require("node:vm");
|
|
52
55
|
const node_crypto_1 = require("node:crypto");
|
|
56
|
+
const node_perf_hooks_1 = require("node:perf_hooks");
|
|
53
57
|
const commonUtility_ts_1 = require("../utility/lmdb/commonUtility.js");
|
|
54
58
|
const nodeIdMapping_ts_1 = require("./nodeIdMapping.js");
|
|
55
59
|
const lodash_1 = __importDefault(require("lodash"));
|
|
@@ -58,6 +62,7 @@ const lmdbProcessRows_js_1 = __importDefault(require("../dataLayer/harperBridge/
|
|
|
58
62
|
const Resource_ts_1 = require("./Resource.js");
|
|
59
63
|
const when_ts_1 = require("../utility/when.js");
|
|
60
64
|
const DatabaseTransaction_ts_1 = require("./DatabaseTransaction.js");
|
|
65
|
+
const recordLock_ts_1 = require("./recordLock.js");
|
|
61
66
|
const envMngr = __importStar(require("../utility/environment/environmentManager.js"));
|
|
62
67
|
const transactionBroadcast_ts_1 = require("./transactionBroadcast.js");
|
|
63
68
|
const hdbError_ts_1 = require("../utility/errors/hdbError.js");
|
|
@@ -93,7 +98,10 @@ const { validateAttribute } = lmdbProcessRows_js_1.default;
|
|
|
93
98
|
const NULL_WITH_TIMESTAMP = new Uint8Array(9);
|
|
94
99
|
NULL_WITH_TIMESTAMP[8] = 0xc0; // null
|
|
95
100
|
const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification
|
|
101
|
+
const MAX_DATE_TIMESTAMP = 8.64e15;
|
|
96
102
|
const RECORD_PRUNING_INTERVAL = 60000; // one minute
|
|
103
|
+
const MAX_CONCURRENT_HISTORY_REMOVALS = 10;
|
|
104
|
+
const MAX_CONCURRENT_LMDB_HISTORY_REMOVALS = 1000;
|
|
97
105
|
// RocksDB-only: number of eviction/tombstone removals coalesced into a single transaction commit.
|
|
98
106
|
// Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already
|
|
99
107
|
// coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path.
|
|
@@ -102,9 +110,75 @@ const EVICTION_BATCH_SIZE = 100;
|
|
|
102
110
|
// letting an unbounded number of open transactions (and their snapshots) accumulate.
|
|
103
111
|
const MAX_INFLIGHT_EVICTION_BATCHES = 4;
|
|
104
112
|
const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
|
|
113
|
+
// Guardrails for `Prefer: count=exact`: once the requested page has been collected, counting the rest
|
|
114
|
+
// of the match set is bounded by BOTH a row cap and a wall-clock budget, so a paginated read can't turn
|
|
115
|
+
// into an unbounded scan. Exceeding either reports an unknown total (Content-Range `.../*`) rather than
|
|
116
|
+
// truncating the page. These bound the count tail, not the page itself; a genuinely expensive query
|
|
117
|
+
// (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure.
|
|
118
|
+
const MAX_EXACT_COUNT_SCAN = 1_000_000;
|
|
119
|
+
const MAX_EXACT_COUNT_MS = 1_000;
|
|
120
|
+
// Largest page a `Prefer: count=` request will materialize. A request whose limit exceeds this (or is
|
|
121
|
+
// not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal
|
|
122
|
+
// streaming path with no count, so a count request can't be coerced into buffering an unbounded page.
|
|
123
|
+
const MAX_COUNT_PAGE = 10_000;
|
|
124
|
+
// How often the exact-count drain yields to the macrotask queue (must be a power of two for the bit-mask
|
|
125
|
+
// check). Keeps a large scan from monopolizing the event loop without adding a yield per row.
|
|
126
|
+
const COUNT_YIELD_INTERVAL = 2_048;
|
|
105
127
|
envMngr.initSync();
|
|
106
128
|
const LMDB_PREFETCH_WRITES = envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
|
|
107
129
|
const LOCK_TIMEOUT = 10000;
|
|
130
|
+
// This bounds schema-lock acquisition; LOCK_TIMEOUT bounds in-flight record writes during a drop.
|
|
131
|
+
exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT = 10000;
|
|
132
|
+
const UPDATE_ATTRIBUTES_LOCK = 'update-attributes';
|
|
133
|
+
// Contention is otherwise only visible once it becomes a timeout (harper#2251).
|
|
134
|
+
exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT = 1000;
|
|
135
|
+
// raw ASCII bytes are ordered-binary's encoding of the string, so this addresses the same native
|
|
136
|
+
// lock as string-keyed tryLock/unlock calls
|
|
137
|
+
const updateAttributesLockKey = Buffer.from(UPDATE_ATTRIBUTES_LOCK);
|
|
138
|
+
const lockWait = new Int32Array(new SharedArrayBuffer(4));
|
|
139
|
+
/** The wait blocks the event loop, so the locked section must stay synchronous. */
|
|
140
|
+
function acquireUpdateAttributesLock(rootStore, scopeDescription, timeout = exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT) {
|
|
141
|
+
if (rootStore.tryLock(updateAttributesLockKey))
|
|
142
|
+
return;
|
|
143
|
+
const startTime = node_perf_hooks_1.performance.now();
|
|
144
|
+
let waitTime = 1;
|
|
145
|
+
while (!rootStore.tryLock(updateAttributesLockKey)) {
|
|
146
|
+
const elapsed = node_perf_hooks_1.performance.now() - startTime;
|
|
147
|
+
if (elapsed >= timeout) {
|
|
148
|
+
throw new hdbError_ts_1.UpdateAttributesLockTimeoutError(`Timed out after ${Math.round(elapsed)}ms waiting for the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription}; the lock holder did not release it before the deadline, so this schema/attribute update cannot proceed`);
|
|
149
|
+
}
|
|
150
|
+
if (elapsed >= 2) {
|
|
151
|
+
Atomics.wait(lockWait, 0, 0, Math.min(waitTime, timeout - elapsed));
|
|
152
|
+
if (waitTime < 16)
|
|
153
|
+
waitTime *= 2;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const waited = node_perf_hooks_1.performance.now() - startTime;
|
|
157
|
+
// The caller cannot register its release until we return, so a throw here would leak the lock
|
|
158
|
+
// with no `finally` able to reach it.
|
|
159
|
+
if (waited >= exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT)
|
|
160
|
+
try {
|
|
161
|
+
logger_ts_1.logger.warn?.(`Acquired the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription} after waiting ${Math.round(waited)}ms; this worker's event loop was blocked for that wait, and a holder that runs past ${exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT}ms fails the update outright`);
|
|
162
|
+
}
|
|
163
|
+
catch { }
|
|
164
|
+
}
|
|
165
|
+
function releaseUpdateAttributesLock(rootStore) {
|
|
166
|
+
rootStore.unlock(updateAttributesLockKey);
|
|
167
|
+
}
|
|
168
|
+
function withUpdateAttributesLock(rootStore, scopeDescription, callback) {
|
|
169
|
+
acquireUpdateAttributesLock(rootStore, scopeDescription);
|
|
170
|
+
try {
|
|
171
|
+
const result = callback();
|
|
172
|
+
if (typeof result?.then === 'function') {
|
|
173
|
+
Promise.resolve(result).catch((error) => logger_ts_1.logger.error?.(`Async update-attributes callback rejected after its lock was released (${scopeDescription})`, error));
|
|
174
|
+
throw new TypeError(`withUpdateAttributesLock callback must be synchronous (${scopeDescription}); asynchronous work may continue after the lock is released`);
|
|
175
|
+
}
|
|
176
|
+
return result;
|
|
177
|
+
}
|
|
178
|
+
finally {
|
|
179
|
+
releaseUpdateAttributesLock(rootStore);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
108
182
|
// Tolerate a redundant column family drop. Drops are broadcast to every worker
|
|
109
183
|
// thread and each holds its own handle to the same underlying family, so a
|
|
110
184
|
// concurrent worker may already have dropped it; the storage engine reports
|
|
@@ -335,8 +409,24 @@ function detectCyclicEnumerable(start) {
|
|
|
335
409
|
function chainKeyForId(id) {
|
|
336
410
|
return typeof id === 'string' ? 's' + id : 'k' + (0, DatabaseTransaction_ts_1.writeKeyId)(id);
|
|
337
411
|
}
|
|
412
|
+
/** Normalizes a passed `context` argument as `transactional()` does; undefined means fall back to ambient. */
|
|
413
|
+
function contextArgument(context) {
|
|
414
|
+
if (!context || (0, DatabaseTransaction_ts_1.isReleasedTransaction)(context))
|
|
415
|
+
return undefined;
|
|
416
|
+
const resolved = context.getContext?.() || context;
|
|
417
|
+
return resolved instanceof DatabaseTransaction_ts_1.DatabaseTransaction ? { transaction: resolved } : resolved;
|
|
418
|
+
}
|
|
419
|
+
/** Distinguishes bare lock options from a record target (id, URL, {id:...}). */
|
|
420
|
+
function isPlainOptions(value) {
|
|
421
|
+
return (typeof value === 'object' &&
|
|
422
|
+
value !== null &&
|
|
423
|
+
!Array.isArray(value) &&
|
|
424
|
+
!(value instanceof URLSearchParams) &&
|
|
425
|
+
value.id === undefined);
|
|
426
|
+
}
|
|
338
427
|
function makeTable(options) {
|
|
339
|
-
|
|
428
|
+
var _a;
|
|
429
|
+
const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, isBranch, } = options;
|
|
340
430
|
let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
|
|
341
431
|
evictionMs ??= 0;
|
|
342
432
|
// Eviction without explicit expiration means expiration:0. Apply at construction so
|
|
@@ -350,6 +440,7 @@ function makeTable(options) {
|
|
|
350
440
|
properties = (0, jsonSchemaTypes_ts_1.projectAttributesToProperties)(attributes);
|
|
351
441
|
const updateRecord = (0, RecordEncoder_ts_1.recordUpdater)(primaryStore, tableId, auditStore);
|
|
352
442
|
let warnedNullSourcePut = false; // latched: one warn per table per worker (see _writeUpdate)
|
|
443
|
+
let warnedFutureSourceVersion = false; // likewise (see getFromSource)
|
|
353
444
|
let sourceLoad; // if a source has a load function (replicator), record it here
|
|
354
445
|
let hasSourceGet;
|
|
355
446
|
let primaryKeyAttribute;
|
|
@@ -383,6 +474,10 @@ function makeTable(options) {
|
|
|
383
474
|
let cleanupPriority = 0;
|
|
384
475
|
let lastCleanupInterval;
|
|
385
476
|
let cleanupTimer;
|
|
477
|
+
let recordExpirationInterval;
|
|
478
|
+
// a reclamation pass awaits a scheduled cleanup, which only settles from its timer
|
|
479
|
+
const pendingCleanupResolvers = new Set();
|
|
480
|
+
let disposed = false;
|
|
386
481
|
// true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup
|
|
387
482
|
let expirationScanScheduled = false;
|
|
388
483
|
// set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table
|
|
@@ -431,10 +526,11 @@ function makeTable(options) {
|
|
|
431
526
|
const MAX_PREFETCH_BUNDLE = 6;
|
|
432
527
|
if (audit)
|
|
433
528
|
addDeleteRemoval();
|
|
434
|
-
|
|
529
|
+
const reclamationHandler = (priority) => {
|
|
435
530
|
if (hasSourceGet)
|
|
436
531
|
return scheduleCleanup(priority);
|
|
437
|
-
}
|
|
532
|
+
};
|
|
533
|
+
(0, storageReclamation_ts_1.onStorageReclamation)(primaryStore.path, reclamationHandler);
|
|
438
534
|
class Updatable extends tracked_ts_1.GenericTrackedObject {
|
|
439
535
|
getUpdatedTime() {
|
|
440
536
|
return RecordEncoder_ts_1.entryMap.get(this.getRecord())?.version;
|
|
@@ -533,12 +629,68 @@ function makeTable(options) {
|
|
|
533
629
|
},
|
|
534
630
|
});
|
|
535
631
|
}
|
|
632
|
+
function resolveAuditHead(id, version, nodeId, refs) {
|
|
633
|
+
if (!refs?.length)
|
|
634
|
+
return { txnLogKey: version, nodeId };
|
|
635
|
+
const visited = new Set();
|
|
636
|
+
function findHead(candidateRefs) {
|
|
637
|
+
if (!candidateRefs)
|
|
638
|
+
return;
|
|
639
|
+
const pending = candidateRefs.slice().reverse();
|
|
640
|
+
while (pending.length > 0) {
|
|
641
|
+
const ref = pending.pop();
|
|
642
|
+
const identity = `${ref.nodeId ?? 0}:${ref.version}`;
|
|
643
|
+
if (visited.has(identity))
|
|
644
|
+
continue;
|
|
645
|
+
visited.add(identity);
|
|
646
|
+
const entry = auditStore.getSync(ref.version, tableId, id, ref.nodeId);
|
|
647
|
+
if (!entry)
|
|
648
|
+
continue;
|
|
649
|
+
if (entry.version === version && (nodeId == null || (entry.nodeId ?? 0) === nodeId))
|
|
650
|
+
return { txnLogKey: ref.version, nodeId: ref.nodeId };
|
|
651
|
+
const previousRefs = entry.previousAdditionalAuditRefs;
|
|
652
|
+
if (previousRefs) {
|
|
653
|
+
for (let index = previousRefs.length - 1; index >= 0; index--)
|
|
654
|
+
pending.push(previousRefs[index]);
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
const referencedHead = findHead(refs);
|
|
659
|
+
if (referencedHead)
|
|
660
|
+
return referencedHead;
|
|
661
|
+
if (version != null) {
|
|
662
|
+
const directHead = auditStore.getSync(version, tableId, id, nodeId);
|
|
663
|
+
if (directHead?.version === version && (nodeId == null || (directHead.nodeId ?? 0) === nodeId))
|
|
664
|
+
return { txnLogKey: version, nodeId };
|
|
665
|
+
}
|
|
666
|
+
return { txnLogKey: version, nodeId };
|
|
667
|
+
}
|
|
536
668
|
class TableResource extends Resource_ts_1.Resource {
|
|
537
669
|
#record; // the stored/frozen record from the database and stored in the cache (should not be modified directly)
|
|
538
670
|
#changes; // the changes to the record that have been made (should not be modified directly)
|
|
539
671
|
#version; // version of the record
|
|
540
672
|
#entry; // the entry from the database
|
|
541
673
|
#savingOperation; // operation for the record is currently being saved
|
|
674
|
+
#lockHandle; // the record lock acquired by lock() — scoped or hold
|
|
675
|
+
#lockWritable; // set by #reloadLocked to let save() stage lock-writable updates
|
|
676
|
+
/**
|
|
677
|
+
* Shared guard: if this instance is lock-writable but the handle is gone (expired or
|
|
678
|
+
* released), throw 409 before staging any write. Covers update/invalidate/relocate/delete
|
|
679
|
+
* in addition to the save() path. Every lock-writable instance carries its own handle in
|
|
680
|
+
* #lockHandle (scoped and hold alike), so we never need to search the registry here.
|
|
681
|
+
*/
|
|
682
|
+
#assertLiveHandle(id) {
|
|
683
|
+
if (!this.#lockWritable)
|
|
684
|
+
return;
|
|
685
|
+
const handle = this.#lockHandle;
|
|
686
|
+
// Off-key writes through the same resource instance are ordinary; only guard the
|
|
687
|
+
// exact key the lock was acquired for.
|
|
688
|
+
if (handle.keyId !== (0, DatabaseTransaction_ts_1.writeKeyId)(id))
|
|
689
|
+
return;
|
|
690
|
+
if (handle.expired || handle.released) {
|
|
691
|
+
throw (0, recordLock_ts_1.lockNotHeldError)(handle);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
542
694
|
// #section: static-config
|
|
543
695
|
static name = tableName; // for display/debugging purposes
|
|
544
696
|
static primaryStore = primaryStore;
|
|
@@ -635,7 +787,7 @@ function makeTable(options) {
|
|
|
635
787
|
// perform the write of an individual write event
|
|
636
788
|
const writeUpdate = async (event, context) => {
|
|
637
789
|
const value = event.value;
|
|
638
|
-
const Table = event.table ? databases_ts_1.databases[databaseName][event.table] :
|
|
790
|
+
const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : _a;
|
|
639
791
|
if (databaseName === hdbTerms_ts_1.SYSTEM_SCHEMA_NAME &&
|
|
640
792
|
(event.table === hdbTerms_ts_1.SYSTEM_TABLE_NAMES.ROLE_TABLE_NAME || event.table === hdbTerms_ts_1.SYSTEM_TABLE_NAMES.USER_TABLE_NAME)) {
|
|
641
793
|
userRoleUpdate = true;
|
|
@@ -652,6 +804,9 @@ function makeTable(options) {
|
|
|
652
804
|
ensureLoaded: false,
|
|
653
805
|
nodeId: event.nodeId,
|
|
654
806
|
viaNodeId: event.viaNodeId,
|
|
807
|
+
// the origin's record version, stored as-is so every replica holds the version the
|
|
808
|
+
// origin holds; the transaction's own timestamp stays the origin's log key
|
|
809
|
+
version: event.version,
|
|
655
810
|
// use per-event expiresAt: batched txn context only holds the first event's expiration
|
|
656
811
|
expiresAt: event.expiresAt,
|
|
657
812
|
// bulk base-copy snapshot frame: apply current-state directly, without an audit/transaction-log
|
|
@@ -690,7 +845,7 @@ function makeTable(options) {
|
|
|
690
845
|
const stageWrite = (event, context) => {
|
|
691
846
|
let chainKey;
|
|
692
847
|
try {
|
|
693
|
-
const Table = event.table ? databases_ts_1.databases[databaseName][event.table] :
|
|
848
|
+
const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : _a;
|
|
694
849
|
const id = event.id ?? (event.value ? event.value[Table?.primaryKey] : undefined);
|
|
695
850
|
if (id != null && typeof id !== 'symbol')
|
|
696
851
|
chainKey = `${event.table ?? tableName} ${chainKeyForId(id)}`;
|
|
@@ -890,7 +1045,9 @@ function makeTable(options) {
|
|
|
890
1045
|
continue;
|
|
891
1046
|
}
|
|
892
1047
|
}
|
|
893
|
-
//
|
|
1048
|
+
// A source that reports no log position of its own (no `timestamp`) has only one clock,
|
|
1049
|
+
// so its record version doubles as the apply transaction's timestamp. A replication
|
|
1050
|
+
// receiver always sets `timestamp` from the origin's log key and never reaches this.
|
|
894
1051
|
if (!event.timestamp && event.version)
|
|
895
1052
|
event.timestamp = event.version;
|
|
896
1053
|
const commitResolution = (0, transaction_ts_1.transaction)(event, () => {
|
|
@@ -991,7 +1148,7 @@ function makeTable(options) {
|
|
|
991
1148
|
/** Indicates if the events should be revalidated when they are received. By default we do this if the get
|
|
992
1149
|
* method is overriden */
|
|
993
1150
|
static get shouldRevalidateEvents() {
|
|
994
|
-
return this.prototype.get !==
|
|
1151
|
+
return this.prototype.get !== _a.prototype.get;
|
|
995
1152
|
}
|
|
996
1153
|
/**
|
|
997
1154
|
* Gets a resource instance, as defined by the Resource class, adding the table-specific handling
|
|
@@ -1027,7 +1184,7 @@ function makeTable(options) {
|
|
|
1027
1184
|
}
|
|
1028
1185
|
return loadLocalRecord(id, request, { transaction: readTxn, ensureLoaded: resourceOptions?.ensureLoaded }, sync, (entry) => {
|
|
1029
1186
|
if (entry) {
|
|
1030
|
-
|
|
1187
|
+
_a._updateResource(this, entry);
|
|
1031
1188
|
}
|
|
1032
1189
|
else
|
|
1033
1190
|
this.#record = null;
|
|
@@ -1045,7 +1202,7 @@ function makeTable(options) {
|
|
|
1045
1202
|
if (loadingFromSource) {
|
|
1046
1203
|
txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
|
|
1047
1204
|
return (0, when_ts_1.when)(loadingFromSource, (entry) => {
|
|
1048
|
-
|
|
1205
|
+
_a._updateResource(this, entry);
|
|
1049
1206
|
return this;
|
|
1050
1207
|
});
|
|
1051
1208
|
}
|
|
@@ -1265,7 +1422,7 @@ function makeTable(options) {
|
|
|
1265
1422
|
return dbisDb.getSync([Symbol.for('residency_by_id'), id]);
|
|
1266
1423
|
}
|
|
1267
1424
|
static setResidency(getResidency) {
|
|
1268
|
-
|
|
1425
|
+
_a.getResidency =
|
|
1269
1426
|
getResidency &&
|
|
1270
1427
|
((record, context) => {
|
|
1271
1428
|
try {
|
|
@@ -1278,7 +1435,7 @@ function makeTable(options) {
|
|
|
1278
1435
|
});
|
|
1279
1436
|
}
|
|
1280
1437
|
static setResidencyById(getResidencyById) {
|
|
1281
|
-
|
|
1438
|
+
_a.getResidencyById =
|
|
1282
1439
|
getResidencyById &&
|
|
1283
1440
|
((id) => {
|
|
1284
1441
|
try {
|
|
@@ -1291,8 +1448,8 @@ function makeTable(options) {
|
|
|
1291
1448
|
});
|
|
1292
1449
|
}
|
|
1293
1450
|
static getResidency(record, context) {
|
|
1294
|
-
if (
|
|
1295
|
-
return
|
|
1451
|
+
if (_a.getResidencyById) {
|
|
1452
|
+
return _a.getResidencyById(record[primaryKey]);
|
|
1296
1453
|
}
|
|
1297
1454
|
let count = replicateToCount;
|
|
1298
1455
|
if (context.replicateTo != undefined) {
|
|
@@ -1334,7 +1491,7 @@ function makeTable(options) {
|
|
|
1334
1491
|
return; // already enabled
|
|
1335
1492
|
audit = true;
|
|
1336
1493
|
addDeleteRemoval();
|
|
1337
|
-
|
|
1494
|
+
_a.audit = true;
|
|
1338
1495
|
}
|
|
1339
1496
|
/**
|
|
1340
1497
|
* Coerce the id as a string to the correct type for the primary key
|
|
@@ -1346,7 +1503,24 @@ function makeTable(options) {
|
|
|
1346
1503
|
return null;
|
|
1347
1504
|
return coerceType(id, primaryKeyAttribute);
|
|
1348
1505
|
}
|
|
1506
|
+
/**
|
|
1507
|
+
* A branch's Table classes deliberately carry the BASE's logical database name so an
|
|
1508
|
+
* application's schema and code resolve unchanged (harper#643). That makes every schema
|
|
1509
|
+
* mutation resolve against the global catalog — a `dropTable()` through a branch would delete
|
|
1510
|
+
* the live base table. Reads and writes are per-branch and unaffected; DDL is refused until a
|
|
1511
|
+
* branch owns a schema identity of its own.
|
|
1512
|
+
*/
|
|
1513
|
+
static assertSchemaMutable(operation) {
|
|
1514
|
+
if (!isBranch)
|
|
1515
|
+
return;
|
|
1516
|
+
const error = new Error(`Cannot ${operation} through a branched database: '${tableName}' resolves to the schema of base ` +
|
|
1517
|
+
`database '${databaseName}', so the change would apply to the base rather than the branch`);
|
|
1518
|
+
error.statusCode = 400;
|
|
1519
|
+
throw error;
|
|
1520
|
+
}
|
|
1349
1521
|
static async dropTable() {
|
|
1522
|
+
_a.assertSchemaMutable('drop a table');
|
|
1523
|
+
const rootStore = primaryStore.rootStore;
|
|
1350
1524
|
if (databaseName === databasePath) {
|
|
1351
1525
|
// Persist a drop tombstone on the primary catalog entry BEFORE any
|
|
1352
1526
|
// destructive work. If the process dies or a column family drop fails
|
|
@@ -1354,9 +1528,11 @@ function makeTable(options) {
|
|
|
1354
1528
|
// the next startup (or a same-name create) completes the drop via
|
|
1355
1529
|
// completeInterruptedDrop in databases.ts instead of resurrecting
|
|
1356
1530
|
// the table.
|
|
1357
|
-
const primaryCatalogKey =
|
|
1358
|
-
const
|
|
1359
|
-
|
|
1531
|
+
const primaryCatalogKey = _a.tableName + '/';
|
|
1532
|
+
const writeTombstone = () => {
|
|
1533
|
+
const primaryMeta = dbisDb.getSync(primaryCatalogKey);
|
|
1534
|
+
if (!primaryMeta || primaryMeta.dropping)
|
|
1535
|
+
return;
|
|
1360
1536
|
primaryMeta.dropping = true;
|
|
1361
1537
|
// Stamps this drop's identity so the interrupted-drop retry budget in
|
|
1362
1538
|
// databases.ts can be scoped to THIS drop rather than the table name: a
|
|
@@ -1366,11 +1542,23 @@ function makeTable(options) {
|
|
|
1366
1542
|
// the budget by generation instead makes the new drop's tombstone carry
|
|
1367
1543
|
// its own fresh key regardless of what any worker last observed.
|
|
1368
1544
|
primaryMeta.dropGeneration = (0, node_crypto_1.randomUUID)();
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1545
|
+
return dbisDb.put(primaryCatalogKey, primaryMeta);
|
|
1546
|
+
};
|
|
1547
|
+
if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
|
|
1548
|
+
// withUpdateAttributesLock's locked section cannot be held across an await, so a durable
|
|
1549
|
+
// tombstone depends on put being rebound to putSync for RocksDB primary stores (see
|
|
1550
|
+
// createOpenDBIObject). Check that BEFORE writing anything: a tombstone left behind by a
|
|
1551
|
+
// refused drop would delete the table on the next load.
|
|
1552
|
+
if (dbisDb.put !== dbisDb.putSync)
|
|
1553
|
+
throw new Error(`Cannot drop ${databaseName}.${_a.tableName}: the catalog store's put is asynchronous, so the drop tombstone cannot be made durable before the column families are dropped`);
|
|
1554
|
+
withUpdateAttributesLock(rootStore, `drop table '${databaseName}.${_a.tableName}'`, writeTombstone);
|
|
1555
|
+
}
|
|
1556
|
+
else {
|
|
1557
|
+
let tombstoneWrite;
|
|
1558
|
+
rootStore.transactionSync(() => {
|
|
1559
|
+
tombstoneWrite = writeTombstone();
|
|
1560
|
+
});
|
|
1561
|
+
if (typeof tombstoneWrite?.then === 'function')
|
|
1374
1562
|
await tombstoneWrite;
|
|
1375
1563
|
}
|
|
1376
1564
|
}
|
|
@@ -1441,28 +1629,25 @@ function makeTable(options) {
|
|
|
1441
1629
|
// same-name create completes the interrupted drop and writes fresh
|
|
1442
1630
|
// catalog rows, and clobbering those would orphan the new table.
|
|
1443
1631
|
const removeTombstonedCatalog = () => {
|
|
1444
|
-
const currentPrimary = dbisDb.getSync(
|
|
1632
|
+
const currentPrimary = dbisDb.getSync(_a.tableName + '/');
|
|
1445
1633
|
if (!currentPrimary?.dropping)
|
|
1446
1634
|
return false;
|
|
1447
1635
|
for (const attribute of attributes) {
|
|
1448
|
-
dbisDb.remove(
|
|
1636
|
+
dbisDb.remove(_a.tableName + '/' + attribute.name);
|
|
1449
1637
|
}
|
|
1450
|
-
dbisDb.remove(
|
|
1638
|
+
dbisDb.remove(_a.tableName + '/');
|
|
1451
1639
|
return true;
|
|
1452
1640
|
};
|
|
1453
|
-
const rootStore = primaryStore.rootStore;
|
|
1454
1641
|
if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
|
|
1455
1642
|
// Serialize the drops + catalog removal against a concurrent
|
|
1456
1643
|
// same-name create (and completeInterruptedDrop) under the database's
|
|
1457
1644
|
// 'update-attributes' exclusive lock - the same lock the create path
|
|
1458
|
-
// holds. It is a synchronous
|
|
1645
|
+
// holds. It is a synchronous lock wait that blocks the event loop, so
|
|
1459
1646
|
// the locked section MUST stay synchronous: drop with dropSync (as
|
|
1460
1647
|
// completeInterruptedDrop does), never an awaited drop(), or a
|
|
1461
|
-
// concurrent create's
|
|
1462
|
-
//
|
|
1463
|
-
|
|
1464
|
-
let removed = false;
|
|
1465
|
-
try {
|
|
1648
|
+
// concurrent create's wait would be stuck on a drop that the blocked
|
|
1649
|
+
// event loop can never resolve, burning its full deadline before failing.
|
|
1650
|
+
const removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
|
|
1466
1651
|
for (const attribute of attributes) {
|
|
1467
1652
|
const index = indices[attribute.name];
|
|
1468
1653
|
if (index)
|
|
@@ -1479,11 +1664,8 @@ function makeTable(options) {
|
|
|
1479
1664
|
catch (error) {
|
|
1480
1665
|
ignoreAlreadyDropped(error);
|
|
1481
1666
|
}
|
|
1482
|
-
|
|
1483
|
-
}
|
|
1484
|
-
finally {
|
|
1485
|
-
rootStore.unlock('update-attributes');
|
|
1486
|
-
}
|
|
1667
|
+
return removeTombstonedCatalog();
|
|
1668
|
+
});
|
|
1487
1669
|
if (removed)
|
|
1488
1670
|
await dbisDb.committed;
|
|
1489
1671
|
}
|
|
@@ -1504,7 +1686,12 @@ function makeTable(options) {
|
|
|
1504
1686
|
}
|
|
1505
1687
|
}
|
|
1506
1688
|
else {
|
|
1507
|
-
// legacy table per database
|
|
1689
|
+
// legacy table per database. The store to retire is this table's own audit store: nothing
|
|
1690
|
+
// assigns `primaryStore.auditStore` — openAuditStore() assigns `rootStore.auditStore`, and
|
|
1691
|
+
// this is the reference makeTable() was handed. Awaited so a pass suspended mid-removal has
|
|
1692
|
+
// released the primary DBI before it is closed and unlinked.
|
|
1693
|
+
await auditStore?.stopAuditCleanup?.();
|
|
1694
|
+
(0, storageReclamation_ts_1.removeStorageReclamation)(primaryStore.path);
|
|
1508
1695
|
await primaryStore.close();
|
|
1509
1696
|
node_fs_1.default.unlinkSync(primaryStore.path);
|
|
1510
1697
|
}
|
|
@@ -1535,7 +1722,7 @@ function makeTable(options) {
|
|
|
1535
1722
|
estimatedRecordRange: undefined,
|
|
1536
1723
|
};
|
|
1537
1724
|
if (this.getContext()?.includeExpensiveRecordCountEstimates) {
|
|
1538
|
-
return
|
|
1725
|
+
return _a.getRecordCount().then((recordCount) => {
|
|
1539
1726
|
description.recordCount = recordCount.recordCount;
|
|
1540
1727
|
description.estimatedRecordRange = recordCount.estimatedRange;
|
|
1541
1728
|
return description;
|
|
@@ -1819,14 +2006,71 @@ function makeTable(options) {
|
|
|
1819
2006
|
* Save any changes into this instance to the current transaction
|
|
1820
2007
|
*/
|
|
1821
2008
|
save() {
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
2009
|
+
this.#assertLiveHandle(this.getId()); // a write through a released or expired lock never lands
|
|
2010
|
+
const operation = this.#savingOperation;
|
|
2011
|
+
if ((!operation || operation.dropped) && this.#lockWritable && this.#lockHandle?.hold) {
|
|
2012
|
+
// A held lock's record stages its update here rather than at lock() time: it is often
|
|
2013
|
+
// written after the acquiring transaction has already completed, which would have
|
|
2014
|
+
// dropped an update staged then. Nothing set means nothing to stage — a held-but-untouched
|
|
2015
|
+
// id stays untouched. Scoped locks do not take this branch: #reloadLocked stages their
|
|
2016
|
+
// TransactionWrite at lock() time (exactly like update()), so #savingOperation is always
|
|
2017
|
+
// set for a live scoped lock and the ordinary path below applies.
|
|
2018
|
+
// Verify the hold is still alive: if #lockWritable is set but the handle expired or was
|
|
2019
|
+
// released between lock acquisition and this save(), throw 409 rather than silently
|
|
2020
|
+
// committing stale data. Every lock-writable instance carries its own handle.
|
|
2021
|
+
const saveHandle = this.#lockHandle;
|
|
2022
|
+
if (saveHandle.expired || saveHandle.released) {
|
|
2023
|
+
throw (0, recordLock_ts_1.lockNotHeldError)(saveHandle);
|
|
2024
|
+
}
|
|
2025
|
+
const changes = this.#changes;
|
|
2026
|
+
if (changes && Object.keys(changes).length > 0) {
|
|
2027
|
+
this.#savingOperation = null;
|
|
2028
|
+
return (0, when_ts_1.when)(this._writeUpdate(this.getId(), changes, false), () => {
|
|
2029
|
+
const op = this.#savingOperation;
|
|
2030
|
+
if (op?.dropped) {
|
|
2031
|
+
this.#changes = undefined;
|
|
2032
|
+
return;
|
|
2033
|
+
}
|
|
2034
|
+
// Clear #savingOperation so the next sequential save() enters the lock-writable
|
|
2035
|
+
// path and creates a fresh write (otherwise a non-null #savingOperation makes
|
|
2036
|
+
// save() take the #saveOperation branch with an already-committed write, which
|
|
2037
|
+
// is a no-op, silently dropping the new change).
|
|
2038
|
+
// op.innerCommit is the real native-transaction commit Promise set on the
|
|
2039
|
+
// immediateCommit path in DatabaseTransaction.save(); await it to ensure
|
|
2040
|
+
// durability before resolving to the caller.
|
|
2041
|
+
if (op?.saved) {
|
|
2042
|
+
this.#savingOperation = null;
|
|
2043
|
+
return op?.innerCommit;
|
|
2044
|
+
}
|
|
2045
|
+
// op.saved = false means addWrite deferred the save; #saveOperation commits it
|
|
2046
|
+
// synchronously but ImmediateTransaction.save() returns undefined while the
|
|
2047
|
+
// inner rocksdb commit is still pending — return innerCommit so the caller
|
|
2048
|
+
// actually waits for durability.
|
|
2049
|
+
return (0, when_ts_1.when)(this.save(), () => op?.innerCommit);
|
|
2050
|
+
});
|
|
1825
2051
|
}
|
|
1826
|
-
|
|
2052
|
+
// No changes: nothing to stage. A dropped operation (detached at a scoped→hold
|
|
2053
|
+
// upgrade — see detachScopedUpgradeWrite) must not fall through to the ordinary
|
|
2054
|
+
// #saveOperation path below with its now-detached reference.
|
|
2055
|
+
if (!operation || operation.dropped) {
|
|
1827
2056
|
this.#savingOperation = null;
|
|
2057
|
+
return;
|
|
1828
2058
|
}
|
|
1829
2059
|
}
|
|
2060
|
+
if (this.#savingOperation) {
|
|
2061
|
+
const operation = this.#savingOperation;
|
|
2062
|
+
this.#savingOperation = null;
|
|
2063
|
+
// A write that lands via a nested immediateCommit (e.g. a second sequential save() on
|
|
2064
|
+
// the same ImmediateTransaction context, once the first has already closed it) sets
|
|
2065
|
+
// operation.innerCommit to the real native-commit promise, but the commit() sweep loop
|
|
2066
|
+
// that triggers it discards its own return value — #saveOperation()'s result can
|
|
2067
|
+
// resolve before that native commit actually settles. Chain on innerCommit (as the
|
|
2068
|
+
// lock-writable hold branch above already does) so callers awaiting save() see the
|
|
2069
|
+
// write durably land, not just the outer (possibly premature) resolution.
|
|
2070
|
+
const result = this.#saveOperation(operation);
|
|
2071
|
+
const innerCommit = operation.innerCommit;
|
|
2072
|
+
return innerCommit ? (0, when_ts_1.when)(innerCommit, () => result) : result;
|
|
2073
|
+
}
|
|
1830
2074
|
}
|
|
1831
2075
|
#saveOperation(operation) {
|
|
1832
2076
|
const transaction = txnForContext(this.getContext());
|
|
@@ -1904,6 +2148,7 @@ function makeTable(options) {
|
|
|
1904
2148
|
});
|
|
1905
2149
|
}
|
|
1906
2150
|
_writeInvalidate(id, partialRecord, options) {
|
|
2151
|
+
this.#assertLiveHandle(id);
|
|
1907
2152
|
const context = this.getContext();
|
|
1908
2153
|
checkValidId(id);
|
|
1909
2154
|
const transaction = txnForContext(this.getContext());
|
|
@@ -1912,9 +2157,12 @@ function makeTable(options) {
|
|
|
1912
2157
|
store: primaryStore,
|
|
1913
2158
|
invalidated: true,
|
|
1914
2159
|
entry: this.#entry,
|
|
2160
|
+
recordVersion: options?.version,
|
|
2161
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
|
|
1915
2162
|
commit: (txnTime, existingEntry, _retry, transaction) => {
|
|
2163
|
+
const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
1916
2164
|
write.skipped = false; // reset on each retry; cleanup happens after commit if still true
|
|
1917
|
-
if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId)
|
|
2165
|
+
if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
|
|
1918
2166
|
write.skipped = true;
|
|
1919
2167
|
return;
|
|
1920
2168
|
}
|
|
@@ -1935,7 +2183,13 @@ function makeTable(options) {
|
|
|
1935
2183
|
viaNodeId: options?.viaNodeId,
|
|
1936
2184
|
transaction,
|
|
1937
2185
|
tableToTrack: tableName,
|
|
2186
|
+
recordVersion: txnTime,
|
|
2187
|
+
additionalAuditRefs: isRocksDB && audit && txnLogKey !== txnTime
|
|
2188
|
+
? [{ version: txnLogKey, nodeId: options?.nodeId }]
|
|
2189
|
+
: undefined,
|
|
1938
2190
|
}, 'invalidate');
|
|
2191
|
+
if (write.trackRecordVersion)
|
|
2192
|
+
write.recordVersionApplied = true;
|
|
1939
2193
|
// TODO: recordDeletion?
|
|
1940
2194
|
},
|
|
1941
2195
|
};
|
|
@@ -1943,21 +2197,25 @@ function makeTable(options) {
|
|
|
1943
2197
|
transaction.addWrite(write);
|
|
1944
2198
|
}
|
|
1945
2199
|
_writeRelocate(id, options) {
|
|
2200
|
+
this.#assertLiveHandle(id);
|
|
1946
2201
|
const context = this.getContext();
|
|
1947
2202
|
checkValidId(id);
|
|
1948
2203
|
const transaction = txnForContext(this.getContext());
|
|
1949
|
-
|
|
2204
|
+
const write = {
|
|
1950
2205
|
key: id,
|
|
1951
2206
|
store: primaryStore,
|
|
1952
2207
|
invalidated: true,
|
|
1953
2208
|
entry: this.#entry,
|
|
2209
|
+
recordVersion: options?.version,
|
|
2210
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
|
|
1954
2211
|
before: this.constructor.source?.relocate && !context?.source
|
|
1955
2212
|
? this.constructor.source.relocate.bind(this.constructor.source, id, undefined, context)
|
|
1956
2213
|
: undefined,
|
|
1957
2214
|
commit: (txnTime, existingEntry, _retry, transaction) => {
|
|
1958
|
-
|
|
2215
|
+
const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
2216
|
+
if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0)
|
|
1959
2217
|
return;
|
|
1960
|
-
const residency =
|
|
2218
|
+
const residency = _a.getResidencyRecord(options.residencyId);
|
|
1961
2219
|
let metadata = 0;
|
|
1962
2220
|
let newRecord = null;
|
|
1963
2221
|
const existingRecord = existingEntry?.value;
|
|
@@ -1981,9 +2239,16 @@ function makeTable(options) {
|
|
|
1981
2239
|
viaNodeId: options?.viaNodeId,
|
|
1982
2240
|
expiresAt: options.expiresAt,
|
|
1983
2241
|
transaction,
|
|
2242
|
+
recordVersion: txnTime,
|
|
2243
|
+
additionalAuditRefs: isRocksDB && audit && txnLogKey !== txnTime
|
|
2244
|
+
? [{ version: txnLogKey, nodeId: options?.nodeId }]
|
|
2245
|
+
: undefined,
|
|
1984
2246
|
}, 'relocate', false, null);
|
|
2247
|
+
if (write.trackRecordVersion)
|
|
2248
|
+
write.recordVersionApplied = true;
|
|
1985
2249
|
},
|
|
1986
|
-
}
|
|
2250
|
+
};
|
|
2251
|
+
transaction.addWrite(write);
|
|
1987
2252
|
}
|
|
1988
2253
|
/**
|
|
1989
2254
|
* Record the relocation of an entry (when a record is moved to a different node), return true if it is now located locally
|
|
@@ -2102,10 +2367,253 @@ function makeTable(options) {
|
|
|
2102
2367
|
}
|
|
2103
2368
|
}
|
|
2104
2369
|
/**
|
|
2105
|
-
*
|
|
2370
|
+
* Static entry point: `Table.lock(id, options?, context?)` — creates an instance in the given,
|
|
2371
|
+
* ambient, or a fresh context and delegates to the instance lock(). This shadows Resource.static
|
|
2372
|
+
* lock so that both callers share the same transaction link (required for cross-instance upgrade
|
|
2373
|
+
* detection). lock() is an in-process API with no authorization hook of its own; it is not
|
|
2374
|
+
* protocol-dispatched, so no allowUpdate/allowCreate check runs on acquisition.
|
|
2375
|
+
*
|
|
2376
|
+
* Dropping the trailing `context` leaks the key: the bare `{}` fallback is an
|
|
2377
|
+
* ImmediateTransaction, which releases no record locks.
|
|
2378
|
+
*/
|
|
2379
|
+
static async lock(target, options, context) {
|
|
2380
|
+
if (!isRocksDB)
|
|
2381
|
+
throw new hdbError_ts_1.ClientError('Record locks are not supported on LMDB', 501);
|
|
2382
|
+
if (options === undefined && isPlainOptions(target)) {
|
|
2383
|
+
options = target;
|
|
2384
|
+
target = undefined;
|
|
2385
|
+
}
|
|
2386
|
+
const id = target != null ? requestTargetToId(target) : null;
|
|
2387
|
+
const resolvedContext = contextArgument(context) ?? transaction_ts_1.contextStorage.getStore() ?? {};
|
|
2388
|
+
const resource = new _a(id, resolvedContext);
|
|
2389
|
+
return resource.lock(target, options);
|
|
2390
|
+
}
|
|
2391
|
+
/**
|
|
2392
|
+
* Acquire an exclusive lock on this record (or on `target`'s) and return it ready for updates
|
|
2393
|
+
* (harper#483, Phase 0: exclusive across every worker thread of this node). The lock is held
|
|
2394
|
+
* in process memory only — no durable writes. Phase 0 contract: lock() is mutually exclusive
|
|
2395
|
+
* with other lock() calls on the same key; plain writes (put/patch/delete/create) are never
|
|
2396
|
+
* gated or blocked. The generation expires after `lease` if it is never released.
|
|
2397
|
+
*
|
|
2398
|
+
* Transaction-scoped (default): write through the returned record (or the table's static verbs
|
|
2399
|
+
* in the same transaction), and the commit or abort releases it. `{ hold: true }`: the lock
|
|
2400
|
+
* outlives the transaction; write through the returned record and release with `unlock()`, or
|
|
2401
|
+
* let the lease expire.
|
|
2402
|
+
*/
|
|
2403
|
+
// async so option/id validation rejects rather than throwing past a caller's `.catch()`; the
|
|
2404
|
+
// body still runs to completion synchronously, which is what keeps concurrent lock() calls
|
|
2405
|
+
// on one key coalescing instead of racing to tryLock.
|
|
2406
|
+
async lock(target, options) {
|
|
2407
|
+
if (!isRocksDB)
|
|
2408
|
+
throw new hdbError_ts_1.ClientError('Record locks are not supported on LMDB', 501);
|
|
2409
|
+
if (options === undefined && isPlainOptions(target)) {
|
|
2410
|
+
options = target;
|
|
2411
|
+
target = undefined;
|
|
2412
|
+
}
|
|
2413
|
+
const id = target != null ? requestTargetToId(target) : this.getId();
|
|
2414
|
+
checkValidId(id);
|
|
2415
|
+
const resolved = (0, recordLock_ts_1.resolveLockOptions)(options);
|
|
2416
|
+
const context = this.getContext();
|
|
2417
|
+
const link = txnForContext(context);
|
|
2418
|
+
const keyId = (0, DatabaseTransaction_ts_1.writeKeyId)(id);
|
|
2419
|
+
const held = this.#lockHandle;
|
|
2420
|
+
if (held && !held.released && !held.expired && held.keyId === keyId) {
|
|
2421
|
+
// Re-entrant: upgrade to hold if requested, then preserve staged changes.
|
|
2422
|
+
if (resolved.hold && !held.hold) {
|
|
2423
|
+
held.upgradeToHold(resolved.lease);
|
|
2424
|
+
// The scoped phase eagerly staged a TransactionWrite (see #reloadLocked); hold
|
|
2425
|
+
// staging is deferred and explicit-save-only, so an unsaved scoped write left
|
|
2426
|
+
// dangling here would otherwise auto-commit at the transaction sweep and clobber
|
|
2427
|
+
// whatever the hold write lands. detachScopedUpgradeWrite marks it .dropped so a
|
|
2428
|
+
// later save() on this instance falls through to the hold branch instead of the
|
|
2429
|
+
// dead #savingOperation reference.
|
|
2430
|
+
detachScopedUpgradeWrite(link, keyId, held);
|
|
2431
|
+
}
|
|
2432
|
+
return Promise.resolve(this.#reloadLocked(id, undefined, true));
|
|
2433
|
+
}
|
|
2434
|
+
const scoped = link.recordLockFor(primaryStore, keyId);
|
|
2435
|
+
if (scoped && !scoped.released && !scoped.expired) {
|
|
2436
|
+
if (resolved.hold && !scoped.hold) {
|
|
2437
|
+
// Upgrade scoped → hold: flip the existing handle object to hold mode so every
|
|
2438
|
+
// instance that already references this handle stays valid. Retiring and creating a
|
|
2439
|
+
// new handle would invalidate those other references (their save() would then throw
|
|
2440
|
+
// 409 against a released handle). The native key stays locked throughout.
|
|
2441
|
+
scoped.upgradeToHold(resolved.lease);
|
|
2442
|
+
detachScopedUpgradeWrite(link, keyId, scoped);
|
|
2443
|
+
return Promise.resolve(this.#reloadLocked(id, scoped, true));
|
|
2444
|
+
}
|
|
2445
|
+
// Already held with the same type: re-entrant return. Preserve any staged changes.
|
|
2446
|
+
return Promise.resolve(this.#reloadLocked(id, scoped, true));
|
|
2447
|
+
}
|
|
2448
|
+
const key = (0, recordLock_ts_1.lockAttemptKey)(tableId, id);
|
|
2449
|
+
// Coalesce concurrent lock() calls for the same key inside one link so they don't
|
|
2450
|
+
// self-block: Promise.all([T.lock(id), T.lock(id)]) would otherwise have both calls
|
|
2451
|
+
// reach tryLock before either registers, making the second park against the first.
|
|
2452
|
+
const pending = link.pendingLockFor(primaryStore, keyId);
|
|
2453
|
+
if (pending) {
|
|
2454
|
+
// Wait for the in-flight acquisition, then take the re-entrant path as if
|
|
2455
|
+
// recordLockFor had found it. If the first attempt timed out, re-enter so
|
|
2456
|
+
// the second caller gets its own timeout.
|
|
2457
|
+
// The follower waits on the leader's acquisition, but only for its own timeout.
|
|
2458
|
+
let followerTimer;
|
|
2459
|
+
const followerTimedOut = Symbol('follower timeout');
|
|
2460
|
+
const followerStart = Date.now();
|
|
2461
|
+
const followerDeadline = new Promise((_, reject) => {
|
|
2462
|
+
followerTimer = setTimeout(() => reject(followerTimedOut), resolved.timeout).unref();
|
|
2463
|
+
});
|
|
2464
|
+
// Try again on this caller's own terms with the budget it has left.
|
|
2465
|
+
const retryOnRemainingBudget = () => {
|
|
2466
|
+
// The enclosing transaction ended while we were parked. A retry re-resolves the
|
|
2467
|
+
// context, which no longer points at this link, so the handle it acquired would be
|
|
2468
|
+
// registered on a fresh transaction that no commit or abort ever releases — the
|
|
2469
|
+
// same abandonment the leader's own post-acquisition guard below rejects.
|
|
2470
|
+
if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.CLOSED && !link.saveCommits)
|
|
2471
|
+
throw new hdbError_ts_1.ServerError('Transaction was closed while waiting for a record lock', 500);
|
|
2472
|
+
const remaining = resolved.timeout - (Date.now() - followerStart);
|
|
2473
|
+
if (remaining <= 0)
|
|
2474
|
+
throw new hdbError_ts_1.ClientError(`Record is locked and was not released in time`, 423);
|
|
2475
|
+
return this.lock(target, { ...resolved, timeout: remaining });
|
|
2476
|
+
};
|
|
2477
|
+
return Promise.race([pending, followerDeadline]).then(() => {
|
|
2478
|
+
clearTimeout(followerTimer);
|
|
2479
|
+
const acquired = link.recordLockFor(primaryStore, keyId);
|
|
2480
|
+
if (acquired && !acquired.released && !acquired.expired) {
|
|
2481
|
+
if (resolved.hold && !acquired.hold) {
|
|
2482
|
+
detachScopedUpgradeWrite(link, keyId, acquired);
|
|
2483
|
+
acquired.upgradeToHold(resolved.lease);
|
|
2484
|
+
}
|
|
2485
|
+
return this.#reloadLocked(id, acquired, true);
|
|
2486
|
+
}
|
|
2487
|
+
return retryOnRemainingBudget();
|
|
2488
|
+
}, (error) => {
|
|
2489
|
+
clearTimeout(followerTimer);
|
|
2490
|
+
if (error === followerTimedOut)
|
|
2491
|
+
throw new hdbError_ts_1.ClientError(`Record is locked and was not released in time`, 423);
|
|
2492
|
+
return retryOnRemainingBudget();
|
|
2493
|
+
});
|
|
2494
|
+
}
|
|
2495
|
+
const pendingPromise = (0, recordLock_ts_1.acquireRecordKey)(link, primaryStore, key, keyId, resolved.timeout, resolved.lease, resolved.hold);
|
|
2496
|
+
link.registerPendingLock(primaryStore, keyId, pendingPromise);
|
|
2497
|
+
return pendingPromise.then((handle) => {
|
|
2498
|
+
link.unregisterPendingLock(primaryStore, keyId);
|
|
2499
|
+
if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.CLOSED && !link.saveCommits) {
|
|
2500
|
+
// The transaction was aborted while this call waited; nothing would ever release the handle.
|
|
2501
|
+
handle.release();
|
|
2502
|
+
throw new hdbError_ts_1.ServerError('Transaction was closed while waiting for a record lock', 500);
|
|
2503
|
+
}
|
|
2504
|
+
link.registerRecordLock(handle);
|
|
2505
|
+
if (link.saveCommits && context?.timestamp)
|
|
2506
|
+
handle.noteCandidateFloor(context.timestamp);
|
|
2507
|
+
if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.OPEN && !link.saveCommits) {
|
|
2508
|
+
// Explicit transaction() (not ImmediateTransaction): pin the clock to
|
|
2509
|
+
// acquiredAt when no writes have been staged yet. When writes already
|
|
2510
|
+
// exist, leave the clock alone (ordering is best-effort; write held records
|
|
2511
|
+
// in their own transaction for the guarantee). ImmediateTransaction is
|
|
2512
|
+
// excluded (saveCommits=true) — its clock is never pinned in lock();
|
|
2513
|
+
// each save() stamps from the handle's committed version floor instead.
|
|
2514
|
+
if (link.writes.length === 0 && !link.timestamp) {
|
|
2515
|
+
link.timestamp = handle.acquiredAt;
|
|
2516
|
+
}
|
|
2517
|
+
if (!resolved.hold && link.transaction) {
|
|
2518
|
+
// Scoped lock: the read snapshot may predate the lock; drop it so the
|
|
2519
|
+
// scope reads what it locked. Hold locks use acquiredAt directly and
|
|
2520
|
+
// do not update the read snapshot.
|
|
2521
|
+
// The timestamp guard matches DatabaseTransaction's own setTimestamp calls: a
|
|
2522
|
+
// deferred update() write leaves the clock at 0, which rocksdb-js rejects.
|
|
2523
|
+
if (link.writes.length === 0 && link.readTxnsUsed <= 1) {
|
|
2524
|
+
link.releaseReadTxn();
|
|
2525
|
+
link.snapshotFree = true;
|
|
2526
|
+
}
|
|
2527
|
+
else if (link.timestamp)
|
|
2528
|
+
link.transaction.setTimestamp(link.timestamp);
|
|
2529
|
+
}
|
|
2530
|
+
}
|
|
2531
|
+
// ImmediateTransaction: no clock pinning in lock(); save() stamps each write
|
|
2532
|
+
// from the committed handle floor for both scoped and hold handles.
|
|
2533
|
+
return this.#reloadLocked(id, handle);
|
|
2534
|
+
}, (err) => {
|
|
2535
|
+
link.unregisterPendingLock(primaryStore, keyId);
|
|
2536
|
+
throw err;
|
|
2537
|
+
});
|
|
2538
|
+
}
|
|
2539
|
+
#reloadLocked(id, holdHandle, preserveChanges = false) {
|
|
2540
|
+
// For freshness, read the committed entry (snapshot-free) so a hold lock sees concurrent
|
|
2541
|
+
// committed writes rather than a stale snapshot. A write earlier in THIS explicit
|
|
2542
|
+
// transaction has not landed in that committed entry yet (harper#1968: Harper defers an
|
|
2543
|
+
// explicit transaction's writes until the writing call actually runs them), so pull the
|
|
2544
|
+
// current value the same way a chained write picks up its basis (priorStagedWrite): the
|
|
2545
|
+
// record comes from the prior staged write, the rest of the entry (version, audit chain,
|
|
2546
|
+
// blob metadata) stays the pre-transaction one.
|
|
2547
|
+
const link = txnForContext(this.getContext());
|
|
2548
|
+
let entryForReload = primaryStore.getEntry(id);
|
|
2549
|
+
if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.OPEN) {
|
|
2550
|
+
const keyId = (0, DatabaseTransaction_ts_1.writeKeyId)(id);
|
|
2551
|
+
const tailWrite = link.writesByKey?.get(primaryStore)?.get(keyId);
|
|
2552
|
+
const priorStaged = tailWrite && (tailWrite.stagedEntry !== undefined ? tailWrite : (0, DatabaseTransaction_ts_1.priorStagedWrite)(tailWrite));
|
|
2553
|
+
if (priorStaged?.stagedEntry !== undefined) {
|
|
2554
|
+
entryForReload = entryForReload
|
|
2555
|
+
? { ...entryForReload, value: priorStaged.stagedEntry.value }
|
|
2556
|
+
: { value: priorStaged.stagedEntry.value };
|
|
2557
|
+
if (entryForReload.value && typeof entryForReload.value === 'object') {
|
|
2558
|
+
// Register the merged entry in entryMap so getUpdatedTime() works.
|
|
2559
|
+
RecordEncoder_ts_1.entryMap.set(entryForReload.value, entryForReload);
|
|
2560
|
+
}
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
if ((0, DatabaseTransaction_ts_1.writeKeyId)(id) !== (0, DatabaseTransaction_ts_1.writeKeyId)(this.getId())) {
|
|
2564
|
+
// lock(target) where target differs from this record: return a separate instance.
|
|
2565
|
+
const fresh = new this.constructor(id, this.getContext());
|
|
2566
|
+
_a._updateResource(fresh, entryForReload);
|
|
2567
|
+
if (holdHandle != null) {
|
|
2568
|
+
fresh.#lockHandle = holdHandle;
|
|
2569
|
+
// Do not clear this.#lockHandle: the original instance keeps its own lock on its
|
|
2570
|
+
// own id; the fresh instance owns the lock on the target id independently.
|
|
2571
|
+
}
|
|
2572
|
+
fresh.#lockWritable = true;
|
|
2573
|
+
// Scoped (not hold) stages exactly like update(): create the TransactionWrite now so
|
|
2574
|
+
// save() is the ordinary #savingOperation path. Hold keeps deferred staging (the
|
|
2575
|
+
// acquiring transaction may commit before the holder ever writes).
|
|
2576
|
+
if (!fresh.#lockHandle.hold)
|
|
2577
|
+
fresh._writeUpdate(id, fresh.#changes, false);
|
|
2578
|
+
return fresh;
|
|
2579
|
+
}
|
|
2580
|
+
// Store the handle for both scoped and hold locks; undefined (re-entrant hold fast-path)
|
|
2581
|
+
// must not clear a handle already set.
|
|
2582
|
+
if (holdHandle != null)
|
|
2583
|
+
this.#lockHandle = holdHandle;
|
|
2584
|
+
_a._updateResource(this, entryForReload);
|
|
2585
|
+
// Preserve staged changes when upgrading the same instance from scoped to hold so that
|
|
2586
|
+
// set() calls made under the scoped lock survive the reload.
|
|
2587
|
+
if (!preserveChanges)
|
|
2588
|
+
this.#changes = undefined;
|
|
2589
|
+
this.#lockWritable = true;
|
|
2590
|
+
// Scoped (not hold): stage now, same as update() would. Skip if a write from an earlier
|
|
2591
|
+
// lock() cycle on this instance is still pending (re-entrant call before its save()).
|
|
2592
|
+
if (!this.#lockHandle.hold && !this.#savingOperation)
|
|
2593
|
+
this._writeUpdate(id, this.#changes, false);
|
|
2594
|
+
return this;
|
|
2595
|
+
}
|
|
2596
|
+
/**
|
|
2597
|
+
* Release the lock this instance holds. Resolves true when this call cleared the native key lock.
|
|
2598
|
+
* Works for both held (`{ hold: true }`) and transaction-scoped locks. After unlock() the
|
|
2599
|
+
* instance is no longer lock-writable; writes through it require a fresh lock.
|
|
2106
2600
|
*/
|
|
2107
|
-
|
|
2108
|
-
|
|
2601
|
+
unlock() {
|
|
2602
|
+
// Always clear the local lock-writable state so subsequent writes on this instance are
|
|
2603
|
+
// ungated, regardless of whether the handle was already released.
|
|
2604
|
+
const handle = this.#lockHandle;
|
|
2605
|
+
this.#lockHandle = undefined;
|
|
2606
|
+
this.#lockWritable = false;
|
|
2607
|
+
if (!handle || handle.released)
|
|
2608
|
+
return Promise.resolve(false);
|
|
2609
|
+
const link = txnForContext(this.getContext());
|
|
2610
|
+
// A scoped lock staged its write at lock() time; released before commit, that write must not
|
|
2611
|
+
// run into the released-handle guard at the sweep.
|
|
2612
|
+
if (this.#savingOperation && !this.#savingOperation.saved && this.#savingOperation.lockHandle === handle)
|
|
2613
|
+
this.#savingOperation = null;
|
|
2614
|
+
detachScopedUpgradeWrite(link, (0, DatabaseTransaction_ts_1.writeKeyId)(this.getId()), handle);
|
|
2615
|
+
link.unregisterRecordLock(handle);
|
|
2616
|
+
return Promise.resolve(handle.release());
|
|
2109
2617
|
}
|
|
2110
2618
|
static operation(operation, context) {
|
|
2111
2619
|
operation.table ||= tableName;
|
|
@@ -2221,8 +2729,10 @@ function makeTable(options) {
|
|
|
2221
2729
|
// a notification that a write has already occurred in the canonical data source, we need to update our
|
|
2222
2730
|
// local copy
|
|
2223
2731
|
_writeUpdate(id, recordUpdate, fullUpdate, options) {
|
|
2732
|
+
this.#assertLiveHandle(id);
|
|
2224
2733
|
const context = this.getContext();
|
|
2225
2734
|
const transaction = txnForContext(context);
|
|
2735
|
+
const replaying = transaction.isReplay === true;
|
|
2226
2736
|
checkValidId(id);
|
|
2227
2737
|
if (fullUpdate && recordUpdate == null && options?.isNotification) {
|
|
2228
2738
|
// A source/replication-applied put must carry the record; these applies skip record
|
|
@@ -2265,6 +2775,13 @@ function makeTable(options) {
|
|
|
2265
2775
|
nodeName: context?.nodeName,
|
|
2266
2776
|
fullUpdate,
|
|
2267
2777
|
deferSave: true,
|
|
2778
|
+
// the origin's record version on an applied write; absent for a locally-originated one
|
|
2779
|
+
recordVersion: options?.version,
|
|
2780
|
+
// Include the lock handle (if any) so the expired-handle guard in
|
|
2781
|
+
// DatabaseTransaction.save() can throw 409 when the lease has lapsed.
|
|
2782
|
+
// Only attach the hold handle when it covers exactly this key; off-key writes
|
|
2783
|
+
// are ordinary and must not carry an unrelated hold's handle.
|
|
2784
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
|
|
2268
2785
|
validate: (txnTime, committedBy = transaction) => {
|
|
2269
2786
|
if (!recordUpdate)
|
|
2270
2787
|
recordUpdate = this.#changes;
|
|
@@ -2389,6 +2906,7 @@ function makeTable(options) {
|
|
|
2389
2906
|
this.#savingOperation = null;
|
|
2390
2907
|
write.stagedIn = undefined; // nothing may pin this write's transaction past its commit
|
|
2391
2908
|
let omitLocalRecord = false;
|
|
2909
|
+
const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
2392
2910
|
// we use optimistic locking to only commit if the existing record state still holds true.
|
|
2393
2911
|
// this is superior to using an async transaction since it doesn't require JS execution
|
|
2394
2912
|
// during the write transaction.
|
|
@@ -2449,8 +2967,8 @@ function makeTable(options) {
|
|
|
2449
2967
|
// best-effort keyed lookup in the capped block below — see #1148. precedesExistingVersion(...)
|
|
2450
2968
|
// === 0 is the identity tie: same version AND same node (the local node is id 0, so an undefined
|
|
2451
2969
|
// options?.nodeId resolves to the same 0 the ref stored).
|
|
2452
|
-
if (existingEntry.additionalAuditRefs?.some((ref) => ref.version ===
|
|
2453
|
-
precedesExistingVersion(txnTime, { version: txnTime, localTime:
|
|
2970
|
+
if (existingEntry.additionalAuditRefs?.some((ref) => ref.version === txnLogKey &&
|
|
2971
|
+
precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: ref.nodeId }, options?.nodeId) === 0)) {
|
|
2454
2972
|
write.skipped = true;
|
|
2455
2973
|
return; // out-of-order write already folded into this record
|
|
2456
2974
|
}
|
|
@@ -2478,10 +2996,10 @@ function makeTable(options) {
|
|
|
2478
2996
|
if (!oldestRetainedAuditTimeResolved) {
|
|
2479
2997
|
oldestRetainedAuditTimeResolved = true;
|
|
2480
2998
|
// getRange yields ascending by audit-log key, so the first entry is the oldest retained.
|
|
2481
|
-
// Mirror replicationConnection's retention check and the cleanup key basis (
|
|
2482
|
-
//
|
|
2999
|
+
// Mirror replicationConnection's retention check and the cleanup key basis (`txnLogKey`).
|
|
3000
|
+
// Fall back to the nominal time-based purge floor when the log is empty/unavailable.
|
|
2483
3001
|
for (const entry of auditStore.getRange({ start: 1, log: options?.nodeId })) {
|
|
2484
|
-
oldestRetainedAuditTime = entry.
|
|
3002
|
+
oldestRetainedAuditTime = entry.txnLogKey;
|
|
2485
3003
|
break;
|
|
2486
3004
|
}
|
|
2487
3005
|
oldestRetainedAuditTime ??= Date.now() - auditStore_ts_1.auditRetention;
|
|
@@ -2494,43 +3012,67 @@ function makeTable(options) {
|
|
|
2494
3012
|
// depth-cap block. This is the same keyed lookup that block performs, hoisted ahead of the walk.
|
|
2495
3013
|
// It is what catches transitive/proxied re-deliveries: they arrive buried below the record head
|
|
2496
3014
|
// (so replication's head-tie fast-skip can't see them) yet are exact duplicates. Keyed by nodeId,
|
|
2497
|
-
// so it is correct across multiple source nodes.
|
|
2498
|
-
//
|
|
2499
|
-
//
|
|
3015
|
+
// so it is correct across multiple source nodes. The lookup key is this write's LOG key, not its
|
|
3016
|
+
// record version — a replication apply commits under the origin's log key while storing the
|
|
3017
|
+
// origin's version, and only the log key addresses the entry (harper#2412).
|
|
3018
|
+
// RocksDB-only: LMDB audit entries are keyed by local audit time, so this lookup doesn't apply
|
|
3019
|
+
// there (LMDB keeps the exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
|
|
2500
3020
|
// simply falls through to the walk, so this never changes correctness; the additionalAuditRefs
|
|
2501
3021
|
// check above remains the read-your-writes guard. Never when this write staged in a prior
|
|
2502
3022
|
// failed attempt: that attempt already appended this write's own audit entry, so the lookup
|
|
2503
3023
|
// would find it and skip the write as "already applied" when the record was never committed.
|
|
2504
3024
|
// A recommit of the same transaction survived that skip only because the old write batch
|
|
2505
3025
|
// still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
|
|
2506
|
-
if (isRocksDB && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(
|
|
2507
|
-
const priorAudit = auditStore.get(
|
|
3026
|
+
if (isRocksDB && !replaying && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnLogKey)) {
|
|
3027
|
+
const priorAudit = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
|
|
2508
3028
|
if (priorAudit &&
|
|
2509
|
-
priorAudit.
|
|
2510
|
-
precedesExistingVersion(txnTime, { version: txnTime, localTime:
|
|
3029
|
+
priorAudit.txnLogKey === txnLogKey &&
|
|
3030
|
+
precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: priorAudit.nodeId }, options?.nodeId) === 0) {
|
|
2511
3031
|
write.skipped = true;
|
|
2512
3032
|
return; // duplicate already applied; avoid the resequencing walk
|
|
2513
3033
|
}
|
|
2514
3034
|
}
|
|
2515
3035
|
// incremental CRDT updates are only available with audit logging on
|
|
2516
|
-
|
|
3036
|
+
const initialAuditHead = isRocksDB
|
|
3037
|
+
? resolveAuditHead(id, existingEntry.version, existingEntry.nodeId, existingEntry.additionalAuditRefs)
|
|
3038
|
+
: { txnLogKey: existingEntry.localTime, nodeId: existingEntry.nodeId };
|
|
3039
|
+
let localTime = initialAuditHead.txnLogKey;
|
|
2517
3040
|
let auditedVersion = existingEntry.version;
|
|
2518
3041
|
logger_ts_1.logger.debug?.('Applying CRDT update to record with id: ', id, 'txn time', new Date(txnTime), 'applying later update from:', new Date(auditedVersion), 'local recorded time', new Date(localTime));
|
|
2519
|
-
let nodeId =
|
|
3042
|
+
let nodeId = initialAuditHead.nodeId;
|
|
2520
3043
|
const succeedingUpdates = []; // record the "future" updates, as we need to apply the updates in reverse order
|
|
2521
3044
|
const auditRefsToVisit = existingEntry.additionalAuditRefs
|
|
2522
3045
|
? existingEntry.additionalAuditRefs.map((ref) => ({ localTime: ref.version, nodeId: ref.nodeId }))
|
|
2523
3046
|
: [];
|
|
2524
|
-
//
|
|
3047
|
+
// Out-of-order merges retain every existing branch head; per-origin log keys are not globally ordered.
|
|
2525
3048
|
if (existingEntry.additionalAuditRefs) {
|
|
2526
3049
|
for (const ref of existingEntry.additionalAuditRefs) {
|
|
2527
|
-
|
|
2528
|
-
additionalAuditRefs.push(ref);
|
|
2529
|
-
}
|
|
3050
|
+
additionalAuditRefs.push(ref);
|
|
2530
3051
|
}
|
|
2531
3052
|
}
|
|
2532
3053
|
let addedAuditRef = false;
|
|
2533
3054
|
let nextRef;
|
|
3055
|
+
const visitedAuditRefs = new Set();
|
|
3056
|
+
const queuePreviousAuditRefs = (auditRecord) => {
|
|
3057
|
+
const previousRefs = auditRecord.previousAdditionalAuditRefs;
|
|
3058
|
+
if (previousRefs) {
|
|
3059
|
+
for (const ref of previousRefs) {
|
|
3060
|
+
auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
|
|
3061
|
+
logger_ts_1.logger.debug?.('Adding audit ref from audit record to visit queue', {
|
|
3062
|
+
version: ref.version,
|
|
3063
|
+
nodeId: ref.nodeId,
|
|
3064
|
+
});
|
|
3065
|
+
}
|
|
3066
|
+
}
|
|
3067
|
+
};
|
|
3068
|
+
const advanceToPreviousAudit = (auditRecord) => {
|
|
3069
|
+
const previousRefs = auditRecord.previousAdditionalAuditRefs;
|
|
3070
|
+
const previousHead = isRocksDB && previousRefs?.length
|
|
3071
|
+
? resolveAuditHead(id, auditRecord.previousVersion, auditRecord.previousNodeId, previousRefs)
|
|
3072
|
+
: { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
|
|
3073
|
+
localTime = previousHead.txnLogKey;
|
|
3074
|
+
nodeId = previousHead.nodeId;
|
|
3075
|
+
};
|
|
2534
3076
|
let walkSteps = 0;
|
|
2535
3077
|
let auditWalkCapped = false;
|
|
2536
3078
|
// Early-out residual: as we walk the chain newest-first, fold each succeeding patch into a
|
|
@@ -2549,17 +3091,21 @@ function makeTable(options) {
|
|
|
2549
3091
|
// appended this write's own audit entry, so the lookup would match it while the record was
|
|
2550
3092
|
// never committed (see the up-front keyed dedup above).
|
|
2551
3093
|
const isReDeliveredDuplicate = () => {
|
|
2552
|
-
if (stagedOwnAuditEntry)
|
|
3094
|
+
if (replaying || stagedOwnAuditEntry)
|
|
2553
3095
|
return false;
|
|
2554
|
-
if (!dedupVersionCouldBeRetained(
|
|
2555
|
-
return false; // pre-retention
|
|
2556
|
-
const duplicate = auditStore.get(
|
|
3096
|
+
if (!dedupVersionCouldBeRetained(txnLogKey))
|
|
3097
|
+
return false; // pre-retention log key — skip the end-of-log scan (best-effort; see above)
|
|
3098
|
+
const duplicate = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
|
|
2557
3099
|
return (duplicate &&
|
|
2558
|
-
duplicate.
|
|
2559
|
-
precedesExistingVersion(txnTime, { version: txnTime, localTime:
|
|
3100
|
+
duplicate.txnLogKey === txnLogKey &&
|
|
3101
|
+
precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: duplicate.nodeId }, options?.nodeId) === 0);
|
|
2560
3102
|
};
|
|
2561
3103
|
do {
|
|
2562
3104
|
while (localTime > txnTime || (auditedVersion >= txnTime && localTime > 0)) {
|
|
3105
|
+
const auditIdentity = `${nodeId ?? 0}:${localTime}`;
|
|
3106
|
+
if (visitedAuditRefs.has(auditIdentity))
|
|
3107
|
+
break;
|
|
3108
|
+
visitedAuditRefs.add(auditIdentity);
|
|
2563
3109
|
// Bound the walk only for RocksDB, where the OOM was observed (issue #1114): each step
|
|
2564
3110
|
// is a transaction-log range scan + msgpackr decode, and the per-node logs can be huge.
|
|
2565
3111
|
// LMDB audit entries are keyed by local audit time (not version), so the duplicate
|
|
@@ -2571,19 +3117,39 @@ function makeTable(options) {
|
|
|
2571
3117
|
const auditRecord = auditStore.get(localTime, tableId, id, nodeId);
|
|
2572
3118
|
if (!auditRecord)
|
|
2573
3119
|
break;
|
|
3120
|
+
queuePreviousAuditRefs(auditRecord);
|
|
3121
|
+
if (isRocksDB &&
|
|
3122
|
+
!replaying &&
|
|
3123
|
+
!stagedOwnAuditEntry &&
|
|
3124
|
+
localTime === txnLogKey &&
|
|
3125
|
+
precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: auditRecord.nodeId }, options?.nodeId) === 0) {
|
|
3126
|
+
write.skipped = true;
|
|
3127
|
+
return;
|
|
3128
|
+
}
|
|
2574
3129
|
auditedVersion = auditRecord.version;
|
|
2575
3130
|
if (auditedVersion >= txnTime) {
|
|
2576
3131
|
if (auditedVersion === txnTime) {
|
|
2577
3132
|
precedesExisting = precedesExistingVersion(txnTime, { version: auditedVersion, localTime: localTime, key: id, nodeId: auditRecord.nodeId }, options?.nodeId);
|
|
2578
3133
|
if (precedesExisting === 0) {
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
3134
|
+
if (isRocksDB && localTime !== txnLogKey) {
|
|
3135
|
+
// Same origin and record version, but a distinct write. Its per-origin log key
|
|
3136
|
+
// orders the otherwise non-unique record clock without comparing keys across origins.
|
|
3137
|
+
precedesExisting = txnLogKey > localTime ? 1 : -1;
|
|
3138
|
+
}
|
|
3139
|
+
else if (replaying || stagedOwnAuditEntry) {
|
|
3140
|
+
// The log entry being replayed (or staged by this write's failed attempt) is
|
|
3141
|
+
// the write itself, not proof that its primary-store mutation committed.
|
|
3142
|
+
precedesExisting = 1;
|
|
3143
|
+
}
|
|
3144
|
+
else {
|
|
3145
|
+
logger_ts_1.logger.debug?.('The transaction time and log key match the existing write, treating as duplicate', id);
|
|
3146
|
+
write.skipped = true;
|
|
3147
|
+
return;
|
|
3148
|
+
}
|
|
2582
3149
|
}
|
|
2583
3150
|
if (precedesExisting > 0) {
|
|
2584
3151
|
// if the existing version is older, we can skip this update
|
|
2585
|
-
|
|
2586
|
-
nodeId = auditRecord.previousNodeId;
|
|
3152
|
+
advanceToPreviousAudit(auditRecord);
|
|
2587
3153
|
continue;
|
|
2588
3154
|
}
|
|
2589
3155
|
}
|
|
@@ -2618,23 +3184,16 @@ function makeTable(options) {
|
|
|
2618
3184
|
}
|
|
2619
3185
|
if (!addedAuditRef && isRocksDB) {
|
|
2620
3186
|
addedAuditRef = true;
|
|
2621
|
-
// Add a reference to this older audit record if we had out-of-order writes
|
|
2622
|
-
|
|
3187
|
+
// Add a reference to this older audit record if we had out-of-order writes. The stored
|
|
3188
|
+
// value is a LOG key, not a record version: every consumer follows it straight into
|
|
3189
|
+
// `auditStore.get` (see the `auditRefsToVisit` mapping above and below), and on an
|
|
3190
|
+
// applied write those two clocks differ.
|
|
3191
|
+
additionalAuditRefs.push({ version: txnLogKey, nodeId: options?.nodeId });
|
|
2623
3192
|
logger_ts_1.logger.debug?.('Adding additional audit ref for out-of-order write', {
|
|
2624
|
-
|
|
3193
|
+
txnLogKey,
|
|
2625
3194
|
nodeId: options?.nodeId,
|
|
2626
3195
|
});
|
|
2627
3196
|
}
|
|
2628
|
-
// Collect any additional audit refs from this audit record to traverse other branches
|
|
2629
|
-
if (auditRecord.previousAdditionalAuditRefs) {
|
|
2630
|
-
for (const ref of auditRecord.previousAdditionalAuditRefs) {
|
|
2631
|
-
auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
|
|
2632
|
-
logger_ts_1.logger.debug?.('Adding audit ref from audit record to visit queue', {
|
|
2633
|
-
version: ref.version,
|
|
2634
|
-
nodeId: ref.nodeId,
|
|
2635
|
-
});
|
|
2636
|
-
}
|
|
2637
|
-
}
|
|
2638
3197
|
// Every field of this write is overwritten by newer writes, and there is no alternate
|
|
2639
3198
|
// audit branch left to scan, so it is fully superseded — the same outcome as walking to
|
|
2640
3199
|
// the end and taking the `writeCommit(false)` escape below, reached without paying the rest
|
|
@@ -2652,8 +3211,7 @@ function makeTable(options) {
|
|
|
2652
3211
|
}
|
|
2653
3212
|
return writeCommit(false);
|
|
2654
3213
|
}
|
|
2655
|
-
|
|
2656
|
-
nodeId = auditRecord.previousNodeId;
|
|
3214
|
+
advanceToPreviousAudit(auditRecord);
|
|
2657
3215
|
}
|
|
2658
3216
|
// Check if we need to scan additional audit refs from this record
|
|
2659
3217
|
if (auditWalkCapped)
|
|
@@ -2748,14 +3306,14 @@ function makeTable(options) {
|
|
|
2748
3306
|
throw new Error('Can not assign a record to a record, check for circular references');
|
|
2749
3307
|
if (residencyId == undefined) {
|
|
2750
3308
|
if (entry?.residencyId)
|
|
2751
|
-
context.previousResidency =
|
|
2752
|
-
const residency = residencyFromFunction(
|
|
3309
|
+
context.previousResidency = _a.getResidencyRecord(entry.residencyId);
|
|
3310
|
+
const residency = residencyFromFunction(_a.getResidency(recordToStore, context));
|
|
2753
3311
|
if (residency) {
|
|
2754
3312
|
if (!residency.includes(server.hostname)) {
|
|
2755
3313
|
// if we aren't in the residency list, specify that our local record should be omitted or be partial
|
|
2756
3314
|
auditRecordToStore ??= recordToStore;
|
|
2757
3315
|
omitLocalRecord = true;
|
|
2758
|
-
if (
|
|
3316
|
+
if (_a.getResidencyById) {
|
|
2759
3317
|
// complete omission of the record that doesn't belong here
|
|
2760
3318
|
recordToStore = undefined;
|
|
2761
3319
|
}
|
|
@@ -2825,7 +3383,17 @@ function makeTable(options) {
|
|
|
2825
3383
|
}
|
|
2826
3384
|
})());
|
|
2827
3385
|
updateIndices(id, existingRecord, recordToStore, transaction && { transaction });
|
|
3386
|
+
// Preserve an addressable audit head when the record and log clocks diverge.
|
|
3387
|
+
if (isRocksDB && audit && !isCopyApply && txnLogKey !== txnTime) {
|
|
3388
|
+
const headIndex = additionalAuditRefs.findIndex((ref) => ref.version === txnLogKey && (ref.nodeId ?? 0) === (options?.nodeId ?? 0));
|
|
3389
|
+
if (headIndex > 0)
|
|
3390
|
+
additionalAuditRefs.unshift(additionalAuditRefs.splice(headIndex, 1)[0]);
|
|
3391
|
+
else if (headIndex < 0)
|
|
3392
|
+
additionalAuditRefs.unshift({ version: txnLogKey, nodeId: options?.nodeId });
|
|
3393
|
+
}
|
|
2828
3394
|
writeCommit(true);
|
|
3395
|
+
if (write.trackRecordVersion)
|
|
3396
|
+
write.recordVersionApplied = true;
|
|
2829
3397
|
if (expiresAt >= 0) {
|
|
2830
3398
|
scheduleCleanup(); // arm for replicated writes too, not just local-context writes
|
|
2831
3399
|
// A runtime per-record expiresAt on a table with no table-level expiration/eviction, no expiresAt
|
|
@@ -2852,6 +3420,8 @@ function makeTable(options) {
|
|
|
2852
3420
|
user: context?.user,
|
|
2853
3421
|
residencyId,
|
|
2854
3422
|
expiresAt,
|
|
3423
|
+
recordVersion: txnTime,
|
|
3424
|
+
recordNodeId: precedesExisting < 0 ? existingEntry?.nodeId : options?.nodeId,
|
|
2855
3425
|
nodeId: options?.nodeId,
|
|
2856
3426
|
viaNodeId: options?.viaNodeId,
|
|
2857
3427
|
originatingOperation: context?.originatingOperation,
|
|
@@ -2889,7 +3459,7 @@ function makeTable(options) {
|
|
|
2889
3459
|
// calls the backend, and a tracked-instance mutation (update(id,{}); row.source=…;
|
|
2890
3460
|
// save()) that sets the source via accessors after update() won't re-embed. A
|
|
2891
3461
|
// resource-layer re-embed is the proper fix; tracked as a follow-up.
|
|
2892
|
-
const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(recordUpdate, context, options,
|
|
3462
|
+
const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(recordUpdate, context, options, _a.embedAttributes, _a.userEmbedders);
|
|
2893
3463
|
const proceed = () => {
|
|
2894
3464
|
// On a source/replication apply (`isNotification`), the record's already-saved blobs were
|
|
2895
3465
|
// received out-of-band for THIS write, so track them for skip/abort cleanup (harper-pro#406).
|
|
@@ -2945,6 +3515,7 @@ function makeTable(options) {
|
|
|
2945
3515
|
return Boolean(this.#record);
|
|
2946
3516
|
}
|
|
2947
3517
|
_writeDelete(id, options) {
|
|
3518
|
+
this.#assertLiveHandle(id);
|
|
2948
3519
|
const context = this.getContext();
|
|
2949
3520
|
const transaction = txnForContext(context);
|
|
2950
3521
|
checkValidId(id);
|
|
@@ -2955,6 +3526,8 @@ function makeTable(options) {
|
|
|
2955
3526
|
entry,
|
|
2956
3527
|
chainsStagedState: true,
|
|
2957
3528
|
nodeName: context?.nodeName,
|
|
3529
|
+
recordVersion: options?.version,
|
|
3530
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
|
|
2958
3531
|
before: this.constructor.source?.delete && !context?.source
|
|
2959
3532
|
? this.constructor.source.delete.bind(this.constructor.source, id, undefined, context)
|
|
2960
3533
|
: undefined,
|
|
@@ -2966,10 +3539,11 @@ function makeTable(options) {
|
|
|
2966
3539
|
const priorStagedOp = (0, DatabaseTransaction_ts_1.priorStagedWrite)(write);
|
|
2967
3540
|
const priorStaged = priorStagedOp?.stagedEntry;
|
|
2968
3541
|
const existingRecord = priorStaged ? priorStaged.value : existingEntry?.value;
|
|
3542
|
+
const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
2969
3543
|
if (retry) {
|
|
2970
3544
|
if (context && existingEntry?.version > (context.lastModified || 0))
|
|
2971
3545
|
context.lastModified = existingEntry.version;
|
|
2972
|
-
|
|
3546
|
+
_a._updateResource(this, existingEntry);
|
|
2973
3547
|
}
|
|
2974
3548
|
// a strictly newer record exists locally, so this delete loses. An earlier write in this
|
|
2975
3549
|
// transaction can never trip this guard — it shares this transaction's timestamp and
|
|
@@ -2987,6 +3561,10 @@ function makeTable(options) {
|
|
|
2987
3561
|
viaNodeId: options?.viaNodeId,
|
|
2988
3562
|
transaction,
|
|
2989
3563
|
tableToTrack: tableName,
|
|
3564
|
+
recordVersion: txnTime,
|
|
3565
|
+
additionalAuditRefs: isRocksDB && audit && txnLogKey !== txnTime
|
|
3566
|
+
? [{ version: txnLogKey, nodeId: options?.nodeId }]
|
|
3567
|
+
: undefined,
|
|
2990
3568
|
}, 'delete');
|
|
2991
3569
|
if (!audit || isRocksDB)
|
|
2992
3570
|
scheduleCleanup();
|
|
@@ -2996,6 +3574,8 @@ function makeTable(options) {
|
|
|
2996
3574
|
(0, RecordEncoder_ts_1.removeEntry)(primaryStore, existingEntry, isRocksDB && transaction ? { transaction } : undefined);
|
|
2997
3575
|
}
|
|
2998
3576
|
write.stagedEntry = { value: undefined }; // the key holds no record for the rest of this transaction
|
|
3577
|
+
if (write.trackRecordVersion)
|
|
3578
|
+
write.recordVersionApplied = true;
|
|
2999
3579
|
// the removal supersedes the nearest record an earlier write in this transaction stored
|
|
3000
3580
|
// (older ones were already marked by their staged successors), so its saved blobs are
|
|
3001
3581
|
// cleaned up post-commit unless its audit entry references them
|
|
@@ -3254,7 +3834,7 @@ function makeTable(options) {
|
|
|
3254
3834
|
// Note, that we do allow users to disable condition re-ordering, in case they have knowledge of a preferred
|
|
3255
3835
|
// order for their query.
|
|
3256
3836
|
if (conditions.length > 1 && operator !== 'or')
|
|
3257
|
-
return sortBy(conditions, (0, search_ts_1.estimateCondition)(
|
|
3837
|
+
return sortBy(conditions, (0, search_ts_1.estimateCondition)(_a));
|
|
3258
3838
|
else
|
|
3259
3839
|
return conditions;
|
|
3260
3840
|
}
|
|
@@ -3329,6 +3909,10 @@ function makeTable(options) {
|
|
|
3329
3909
|
}
|
|
3330
3910
|
}
|
|
3331
3911
|
const select = target.select;
|
|
3912
|
+
// Whether the caller supplied real filter conditions — read from the raw request, NOT the
|
|
3913
|
+
// planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and
|
|
3914
|
+
// injected full-scan condition). Used to pick the count-estimate source below.
|
|
3915
|
+
const hasUserConditions = Array.isArray(target.conditions) && target.conditions.length > 0;
|
|
3332
3916
|
if (conditions.length === 0) {
|
|
3333
3917
|
conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }];
|
|
3334
3918
|
}
|
|
@@ -3365,7 +3949,7 @@ function makeTable(options) {
|
|
|
3365
3949
|
const recordAccess = boundRowFilter || typeof target.vectorFilter === 'function'
|
|
3366
3950
|
? { rowFilter: boundRowFilter, vectorFilter: target.vectorFilter }
|
|
3367
3951
|
: undefined;
|
|
3368
|
-
const entries = (0, search_ts_1.executeConditions)(conditions, operator,
|
|
3952
|
+
const entries = (0, search_ts_1.executeConditions)(conditions, operator, _a, readTxn, target, context, (results, filters) => transformToEntries(results, select, context, readTxn, filters), filtered, recordAccess);
|
|
3369
3953
|
const ensure_loaded = target.ensureLoaded !== false;
|
|
3370
3954
|
// The guards inside executeConditions evaluate the
|
|
3371
3955
|
// LOCAL record, but on a caching table transformEntryForSelect may then revalidate an
|
|
@@ -3382,11 +3966,126 @@ function makeTable(options) {
|
|
|
3382
3966
|
// since those never run the ensureLoaded-gated freshness check this transform
|
|
3383
3967
|
// otherwise applies unconditionally to every read.
|
|
3384
3968
|
const includeExpired = target.includeExpired === true;
|
|
3385
|
-
const transformToRecord =
|
|
3386
|
-
let results =
|
|
3969
|
+
const transformToRecord = _a.transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded, true, boundRowFilter, includeExpired, postOrdering);
|
|
3970
|
+
let results = _a.transformToOrderedSelect(entries, select, postOrdering, context, readTxn, transformToRecord);
|
|
3971
|
+
const offset = target.offset || 0;
|
|
3972
|
+
const end = target.limit !== undefined ? offset + target.limit : undefined;
|
|
3973
|
+
// `Prefer: count=` (REST pagination): materialize the requested page and attach a total record
|
|
3974
|
+
// count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once,
|
|
3975
|
+
// windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/
|
|
3976
|
+
// table estimate. Opt-in only — the default streaming path below is untouched.
|
|
3977
|
+
//
|
|
3978
|
+
// Requires a bounded page AND window. Counting is a pagination feature; both the limit and the
|
|
3979
|
+
// offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the
|
|
3980
|
+
// window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/
|
|
3981
|
+
// oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity),
|
|
3982
|
+
// limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the
|
|
3983
|
+
// normal streaming path with no count. This bounds the offset too: without it a huge offset would
|
|
3984
|
+
// postpone the exact guardrail (which only engages past the page) until that offset was scanned.
|
|
3985
|
+
const pageLimit = target.limit;
|
|
3986
|
+
if (target.count &&
|
|
3987
|
+
Number.isInteger(pageLimit) &&
|
|
3988
|
+
pageLimit >= 0 &&
|
|
3989
|
+
pageLimit <= MAX_COUNT_PAGE &&
|
|
3990
|
+
Number.isInteger(offset) &&
|
|
3991
|
+
offset >= 0 &&
|
|
3992
|
+
offset + pageLimit <= MAX_EXACT_COUNT_SCAN) {
|
|
3993
|
+
const wantExact = target.count === 'exact';
|
|
3994
|
+
const pageEnd = offset + pageLimit;
|
|
3995
|
+
const countStart = node_perf_hooks_1.performance.now();
|
|
3996
|
+
// A custom-index (vector/HNSW) traversal returns a bounded, approximate candidate set whose size is
|
|
3997
|
+
// chosen from `minResults` (offset + limit), so `scanned` over it tracks the requested page size, not
|
|
3998
|
+
// the true match count — the same query at limit(5) vs limit(200) would otherwise advertise two
|
|
3999
|
+
// different `count=exact` totals. Any query whose execution touches a custom index is affected: a
|
|
4000
|
+
// custom-index sort (its aligned pseudo-condition lands in `conditions`), a custom-index threshold
|
|
4001
|
+
// filter (an HNSW `lt`/`le` is the same minResults-widened traversal as a sort), or an opaque vector
|
|
4002
|
+
// filter. Report the total as unavailable for those rather than advertising it as count=exact
|
|
4003
|
+
// (mirroring how the estimated branch below bails to null for an opaque row/vector filter). A vector
|
|
4004
|
+
// sort applied as in-memory post-ordering leaves no custom-index condition here and stays exact.
|
|
4005
|
+
const touchesCustomIndex = (conds) => conds.some((c) => {
|
|
4006
|
+
if (!c)
|
|
4007
|
+
return false;
|
|
4008
|
+
if (c.conditions)
|
|
4009
|
+
return touchesCustomIndex(c.conditions);
|
|
4010
|
+
const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]);
|
|
4011
|
+
return typeof attr === 'string' && Boolean(indices[attr]?.customIndex);
|
|
4012
|
+
});
|
|
4013
|
+
const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions);
|
|
4014
|
+
return (async () => {
|
|
4015
|
+
const page = [];
|
|
4016
|
+
let scanned = 0;
|
|
4017
|
+
let exact = true;
|
|
4018
|
+
try {
|
|
4019
|
+
for await (const record of results) {
|
|
4020
|
+
if (scanned >= offset && scanned < pageEnd)
|
|
4021
|
+
page.push(record);
|
|
4022
|
+
scanned++;
|
|
4023
|
+
// A store whose async iterator settles synchronously (the common indexed-scan case) would
|
|
4024
|
+
// otherwise let this drain spin as one uninterrupted microtask run, blocking the event loop
|
|
4025
|
+
// for the whole count. Yield to the macrotask queue periodically so concurrent requests and
|
|
4026
|
+
// I/O still make progress during a large exact scan.
|
|
4027
|
+
if ((scanned & (COUNT_YIELD_INTERVAL - 1)) === 0)
|
|
4028
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
4029
|
+
// The page window [offset, pageEnd) is always collected in full first — the guardrail
|
|
4030
|
+
// only ever abandons the running TOTAL, never truncates the page body.
|
|
4031
|
+
if (scanned >= pageEnd) {
|
|
4032
|
+
// `estimated` needs nothing past the page; an approximate (vector) exact total is going to
|
|
4033
|
+
// be reported unavailable anyway, so don't drain its tail for a number we won't publish.
|
|
4034
|
+
if (!wantExact || approximateResultSet)
|
|
4035
|
+
break;
|
|
4036
|
+
// `exact` keeps counting the tail, bounded by a row cap AND a time budget so a
|
|
4037
|
+
// large match set can't turn a bounded page fetch into an unbounded scan.
|
|
4038
|
+
if (scanned > MAX_EXACT_COUNT_SCAN || node_perf_hooks_1.performance.now() - countStart > MAX_EXACT_COUNT_MS) {
|
|
4039
|
+
exact = false;
|
|
4040
|
+
break;
|
|
4041
|
+
}
|
|
4042
|
+
}
|
|
4043
|
+
}
|
|
4044
|
+
}
|
|
4045
|
+
finally {
|
|
4046
|
+
// We own the iteration here (no results.onDone consumer), so release the read
|
|
4047
|
+
// transaction unconditionally — including when the drain throws — or the snapshot leaks.
|
|
4048
|
+
txn.doneReadTxn();
|
|
4049
|
+
}
|
|
4050
|
+
let total;
|
|
4051
|
+
if (wantExact) {
|
|
4052
|
+
// `scanned` is only an authoritative total when the iteration was exhaustive and deterministic;
|
|
4053
|
+
// an approximate (vector/HNSW) result set is neither, so report the total as unavailable.
|
|
4054
|
+
total = exact && !approximateResultSet ? scanned : null;
|
|
4055
|
+
}
|
|
4056
|
+
else if (boundRowFilter || typeof target.vectorFilter === 'function') {
|
|
4057
|
+
// An opaque row/vector filter shapes the result but isn't reflected in the index/condition
|
|
4058
|
+
// estimate; guessing would both mislead and disclose cardinality the filter hides.
|
|
4059
|
+
total = null;
|
|
4060
|
+
}
|
|
4061
|
+
else if (!hasUserConditions) {
|
|
4062
|
+
total = (0, search_ts_1.estimatedEntryCount)(primaryStore);
|
|
4063
|
+
}
|
|
4064
|
+
else {
|
|
4065
|
+
// Estimate from the real conditions only — drop the planner's synthetic `sort`
|
|
4066
|
+
// pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality.
|
|
4067
|
+
const est = (0, search_ts_1.estimateCondition)(_a)({
|
|
4068
|
+
conditions: conditions.filter((c) => c.comparator !== 'sort'),
|
|
4069
|
+
operator: operator ? String(operator).toLowerCase() : 'and',
|
|
4070
|
+
});
|
|
4071
|
+
total = isFinite(est) ? Math.round(est) : null;
|
|
4072
|
+
}
|
|
4073
|
+
// For an estimate, never report a total below the last row actually returned — keeps the
|
|
4074
|
+
// Content-Range valid (start-end/total) when an estimate undershoots a non-empty page.
|
|
4075
|
+
// Exact totals are authoritative (and an empty page past the end must not be clamped up).
|
|
4076
|
+
if (!wantExact && total != null && page.length > 0 && total < offset + page.length) {
|
|
4077
|
+
total = offset + page.length;
|
|
4078
|
+
}
|
|
4079
|
+
page.recordCount = total;
|
|
4080
|
+
page.recordCountExact = wantExact && exact && !approximateResultSet;
|
|
4081
|
+
page.selectApplied = true;
|
|
4082
|
+
page.getColumns = getColumns;
|
|
4083
|
+
return page;
|
|
4084
|
+
})();
|
|
4085
|
+
}
|
|
3387
4086
|
// apply any offset/limit after all the sorting and filtering
|
|
3388
4087
|
if (target.offset || target.limit !== undefined)
|
|
3389
|
-
results = results.slice(
|
|
4088
|
+
results = results.slice(offset, end);
|
|
3390
4089
|
results.onDone = () => {
|
|
3391
4090
|
results.onDone = null; // ensure that it isn't called twice
|
|
3392
4091
|
txn.doneReadTxn();
|
|
@@ -3701,12 +4400,12 @@ function makeTable(options) {
|
|
|
3701
4400
|
if (resolver.directReturn)
|
|
3702
4401
|
return callback(value, attribute_name);
|
|
3703
4402
|
if (value && typeof value === 'object') {
|
|
3704
|
-
const targetTable = resolver.definition?.tableClass ||
|
|
4403
|
+
const targetTable = resolver.definition?.tableClass || _a;
|
|
3705
4404
|
if (!transformCache)
|
|
3706
4405
|
transformCache = {};
|
|
3707
4406
|
// Use the target table's own read transaction; each table's readTxn is
|
|
3708
4407
|
// scoped to its RocksDB column family and cannot read another table's store.
|
|
3709
|
-
const targetReadTxn = targetTable ===
|
|
4408
|
+
const targetReadTxn = targetTable === _a ? readTxn : targetTable._readTxnForContext(context);
|
|
3710
4409
|
const transform = transformCache[attribute_name] ||
|
|
3711
4410
|
(transformCache[attribute_name] = targetTable.transformEntryForSelect(
|
|
3712
4411
|
// if it is a simple string, there is no select for the next level,
|
|
@@ -3759,7 +4458,7 @@ function makeTable(options) {
|
|
|
3759
4458
|
else {
|
|
3760
4459
|
value = record[attribute_name];
|
|
3761
4460
|
if (value && typeof value === 'object' && attribute_name !== attribute) {
|
|
3762
|
-
const subTransform =
|
|
4461
|
+
const subTransform = _a.transformEntryForSelect(attribute.select || attribute, context, readTxn, null);
|
|
3763
4462
|
// Plain JSON nested values: arrays project per-element so that
|
|
3764
4463
|
// `select: [{ name: 'addresses', select: ['city'] }]` returns
|
|
3765
4464
|
// `addresses: [{ city }, { city }]` rather than a single object.
|
|
@@ -3838,6 +4537,10 @@ function makeTable(options) {
|
|
|
3838
4537
|
if (!auditStore)
|
|
3839
4538
|
throw new Error('Can not subscribe to a table without an audit log');
|
|
3840
4539
|
if (!audit) {
|
|
4540
|
+
// Turning auditing on is a schema write, and a branch's Table classes carry the base's
|
|
4541
|
+
// logical name: without this a subscribe through a branched application would enable
|
|
4542
|
+
// auditing on the live base table for every other consumer, with no DDL call involved.
|
|
4543
|
+
_a.assertSchemaMutable('enable auditing for a subscription');
|
|
3841
4544
|
(0, databases_ts_1.table)({ table: tableName, database: databaseName, schemaDefined, attributes, audit: true });
|
|
3842
4545
|
}
|
|
3843
4546
|
const getFullRecord = !request.rawEvents;
|
|
@@ -3893,7 +4596,7 @@ function makeTable(options) {
|
|
|
3893
4596
|
return evaluateFilter(rowFilter, event.value, 'rowFilter');
|
|
3894
4597
|
}
|
|
3895
4598
|
: null;
|
|
3896
|
-
const subscription = (0, transactionBroadcast_ts_1.addSubscription)(
|
|
4599
|
+
const subscription = (0, transactionBroadcast_ts_1.addSubscription)(_a, thisId, function (id, auditRecord, txnLogKey, beginTxn) {
|
|
3897
4600
|
if (dropDuringReplay)
|
|
3898
4601
|
return;
|
|
3899
4602
|
try {
|
|
@@ -3920,8 +4623,7 @@ function makeTable(options) {
|
|
|
3920
4623
|
// been written, so are fresh in memory.
|
|
3921
4624
|
const entry = primaryStore.getEntry(id);
|
|
3922
4625
|
if (entry) {
|
|
3923
|
-
|
|
3924
|
-
if (entry.version !== (auditRecord.recordVersion ?? auditRecord.version))
|
|
4626
|
+
if (entry.version !== auditRecord.version)
|
|
3925
4627
|
return; // out of order event, with old update, don't send anything
|
|
3926
4628
|
value = entry.value;
|
|
3927
4629
|
type = entry.metadataFlags & exports.INVALIDATED ? 'invalidate' : value ? 'put' : 'delete';
|
|
@@ -3932,7 +4634,7 @@ function makeTable(options) {
|
|
|
3932
4634
|
}
|
|
3933
4635
|
const event = {
|
|
3934
4636
|
id,
|
|
3935
|
-
localTime,
|
|
4637
|
+
localTime: txnLogKey,
|
|
3936
4638
|
value,
|
|
3937
4639
|
version: auditRecord.version,
|
|
3938
4640
|
type,
|
|
@@ -4001,10 +4703,10 @@ function makeTable(options) {
|
|
|
4001
4703
|
continue;
|
|
4002
4704
|
const id = auditRecord.recordId;
|
|
4003
4705
|
if (thisId == null || isDescendantId(thisId, id)) {
|
|
4004
|
-
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.
|
|
4706
|
+
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
|
|
4005
4707
|
if (!send({
|
|
4006
4708
|
id,
|
|
4007
|
-
localTime: auditRecord.
|
|
4709
|
+
localTime: auditRecord.txnLogKey,
|
|
4008
4710
|
value,
|
|
4009
4711
|
version: auditRecord.version,
|
|
4010
4712
|
type: auditRecord.type,
|
|
@@ -4017,7 +4719,7 @@ function makeTable(options) {
|
|
|
4017
4719
|
return;
|
|
4018
4720
|
}
|
|
4019
4721
|
}
|
|
4020
|
-
subscription.startTime = auditRecord.
|
|
4722
|
+
subscription.startTime = auditRecord.txnLogKey; // update so we don't double send
|
|
4021
4723
|
}
|
|
4022
4724
|
}
|
|
4023
4725
|
finally {
|
|
@@ -4051,10 +4753,10 @@ function makeTable(options) {
|
|
|
4051
4753
|
logger_ts_1.logger.warn?.(`previousCount backfill on ${tableName} stopped after inspecting ${MAX_PREVIOUS_COUNT_SCAN} in-scope audit records without collecting ${request.previousCount} accepted event(s); returning ${history.length} instead`);
|
|
4052
4754
|
break;
|
|
4053
4755
|
}
|
|
4054
|
-
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.
|
|
4756
|
+
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
|
|
4055
4757
|
const historyEntry = {
|
|
4056
4758
|
id,
|
|
4057
|
-
localTime: auditRecord.
|
|
4759
|
+
localTime: auditRecord.txnLogKey,
|
|
4058
4760
|
value,
|
|
4059
4761
|
version: auditRecord.version,
|
|
4060
4762
|
type: auditRecord.type,
|
|
@@ -4071,7 +4773,7 @@ function makeTable(options) {
|
|
|
4071
4773
|
}
|
|
4072
4774
|
}
|
|
4073
4775
|
catch (error) {
|
|
4074
|
-
logger_ts_1.logger.error?.('Error getting history entry', auditRecord.
|
|
4776
|
+
logger_ts_1.logger.error?.('Error getting history entry', auditRecord.txnLogKey, error);
|
|
4075
4777
|
}
|
|
4076
4778
|
}
|
|
4077
4779
|
for (let i = history.length; i > 0;) {
|
|
@@ -4159,6 +4861,12 @@ function makeTable(options) {
|
|
|
4159
4861
|
logger_ts_1.logger.trace?.('re-retrieved record', localTime, this.#entry?.localTime);
|
|
4160
4862
|
localTime = entry?.localTime;
|
|
4161
4863
|
}
|
|
4864
|
+
let nodeId = entry?.nodeId;
|
|
4865
|
+
if (isRocksDB && entry) {
|
|
4866
|
+
const head = resolveAuditHead(thisId, entry.version, nodeId, entry.additionalAuditRefs);
|
|
4867
|
+
localTime = head.txnLogKey;
|
|
4868
|
+
nodeId = head.nodeId;
|
|
4869
|
+
}
|
|
4162
4870
|
logger_ts_1.logger.trace?.('Subscription from', startTime, 'from', thisId, localTime);
|
|
4163
4871
|
if (startTime < localTime) {
|
|
4164
4872
|
// start time specified, get the audit history for this record. Set startTime up
|
|
@@ -4169,7 +4877,6 @@ function makeTable(options) {
|
|
|
4169
4877
|
const history = [];
|
|
4170
4878
|
let inspected = 0;
|
|
4171
4879
|
let nextTime = localTime;
|
|
4172
|
-
let nodeId = entry?.nodeId;
|
|
4173
4880
|
do {
|
|
4174
4881
|
if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
|
|
4175
4882
|
recordsSinceYield = 0;
|
|
@@ -4200,8 +4907,11 @@ function makeTable(options) {
|
|
|
4200
4907
|
else if (!isActive())
|
|
4201
4908
|
return;
|
|
4202
4909
|
}
|
|
4203
|
-
|
|
4204
|
-
|
|
4910
|
+
const previousHead = isRocksDB
|
|
4911
|
+
? resolveAuditHead(thisId, auditRecord.previousVersion, auditRecord.previousNodeId, auditRecord.previousAdditionalAuditRefs)
|
|
4912
|
+
: { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
|
|
4913
|
+
nextTime = previousHead.txnLogKey;
|
|
4914
|
+
nodeId = previousHead.nodeId;
|
|
4205
4915
|
}
|
|
4206
4916
|
else
|
|
4207
4917
|
break;
|
|
@@ -4396,6 +5106,7 @@ function makeTable(options) {
|
|
|
4396
5106
|
store: primaryStore,
|
|
4397
5107
|
entry: this.#entry,
|
|
4398
5108
|
nodeName: context?.nodeName,
|
|
5109
|
+
recordVersion: options?.version,
|
|
4399
5110
|
validate: () => {
|
|
4400
5111
|
if (!context?.source) {
|
|
4401
5112
|
transaction.checkOverloaded();
|
|
@@ -4609,6 +5320,7 @@ function makeTable(options) {
|
|
|
4609
5320
|
return this.#version;
|
|
4610
5321
|
}
|
|
4611
5322
|
static async addAttributes(attributesToAdd) {
|
|
5323
|
+
_a.assertSchemaMutable('add attributes');
|
|
4612
5324
|
const new_attributes = attributes.slice(0);
|
|
4613
5325
|
for (const attribute of attributesToAdd) {
|
|
4614
5326
|
if (!attribute.name)
|
|
@@ -4624,9 +5336,10 @@ function makeTable(options) {
|
|
|
4624
5336
|
schemaDefined,
|
|
4625
5337
|
attributes: new_attributes,
|
|
4626
5338
|
});
|
|
4627
|
-
return
|
|
5339
|
+
return _a.indexingOperation;
|
|
4628
5340
|
}
|
|
4629
5341
|
static async removeAttributes(names) {
|
|
5342
|
+
_a.assertSchemaMutable('remove attributes');
|
|
4630
5343
|
const new_attributes = attributes.filter((attribute) => !names.includes(attribute.name));
|
|
4631
5344
|
(0, databases_ts_1.table)({
|
|
4632
5345
|
table: tableName,
|
|
@@ -4634,7 +5347,7 @@ function makeTable(options) {
|
|
|
4634
5347
|
schemaDefined,
|
|
4635
5348
|
attributes: new_attributes,
|
|
4636
5349
|
});
|
|
4637
|
-
return
|
|
5350
|
+
return _a.indexingOperation;
|
|
4638
5351
|
}
|
|
4639
5352
|
/**
|
|
4640
5353
|
* Get the size of the table in bytes (based on amount of pages stored in the database)
|
|
@@ -4647,6 +5360,10 @@ function makeTable(options) {
|
|
|
4647
5360
|
const stats = primaryStore.getStats();
|
|
4648
5361
|
return (stats.treeBranchPageCount + stats.treeLeafPageCount + stats.overflowPages) * stats.pageSize;
|
|
4649
5362
|
}
|
|
5363
|
+
/** Sizes of this table's durable record-structure dictionaries. */
|
|
5364
|
+
static getStructureCounts() {
|
|
5365
|
+
return primaryStore.encoder?.getStructureCounts?.();
|
|
5366
|
+
}
|
|
4650
5367
|
static getAuditSize() {
|
|
4651
5368
|
const stats = auditStore?.getStats();
|
|
4652
5369
|
return (stats &&
|
|
@@ -4665,7 +5382,7 @@ function makeTable(options) {
|
|
|
4665
5382
|
// iterate through the metadata entries to exclude their count and exclude the deletion counts
|
|
4666
5383
|
const exactCount = options?.exactCount;
|
|
4667
5384
|
const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms
|
|
4668
|
-
const start = performance.now();
|
|
5385
|
+
const start = node_perf_hooks_1.performance.now();
|
|
4669
5386
|
// `entryCount` (the exact key count) is only needed once the scan blows the time budget --
|
|
4670
5387
|
// to decide whether to estimate and as the extrapolation base. On RocksDB it is a full
|
|
4671
5388
|
// key-only scan, so we defer it: tables that finish within budget (the common case) and
|
|
@@ -4682,7 +5399,7 @@ function makeTable(options) {
|
|
|
4682
5399
|
recordCount++;
|
|
4683
5400
|
entriesScanned++;
|
|
4684
5401
|
await rest();
|
|
4685
|
-
if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) {
|
|
5402
|
+
if (!exactCount && !completeForExact && node_perf_hooks_1.performance.now() - start > TIME_LIMIT) {
|
|
4686
5403
|
if (!counted) {
|
|
4687
5404
|
counted = true;
|
|
4688
5405
|
entryCount = isRocksDB
|
|
@@ -4806,7 +5523,7 @@ function makeTable(options) {
|
|
|
4806
5523
|
const computed = attribute.computed;
|
|
4807
5524
|
// Register the default embedder unless an author override is set. Sits outside
|
|
4808
5525
|
// the resolver chain below so `@embed` fields still flow through auto-HNSW indexing.
|
|
4809
|
-
if (attribute.embed && !
|
|
5526
|
+
if (attribute.embed && !_a.userSetEmbedders.has(attribute.name)) {
|
|
4810
5527
|
this.userEmbedders[attribute.name] = (0, embedHook_ts_1.createDefaultEmbedder)(attribute.embed);
|
|
4811
5528
|
}
|
|
4812
5529
|
if (relationship) {
|
|
@@ -4872,7 +5589,7 @@ function makeTable(options) {
|
|
|
4872
5589
|
const options = { transaction: txnForContext(context).getReadTxn() };
|
|
4873
5590
|
const results = normalizedIds.map((id) => {
|
|
4874
5591
|
const value = store[method](id, options);
|
|
4875
|
-
if (
|
|
5592
|
+
if (_a.loadAsInstance === false)
|
|
4876
5593
|
freezeRecord(returnEntry ? value?.value : value);
|
|
4877
5594
|
return value;
|
|
4878
5595
|
});
|
|
@@ -4881,7 +5598,7 @@ function makeTable(options) {
|
|
|
4881
5598
|
const value = definition.tableClass.primaryStore[returnEntry ? 'getEntry' : 'getSync'](ids, {
|
|
4882
5599
|
transaction: txnForContext(context).getReadTxn(),
|
|
4883
5600
|
});
|
|
4884
|
-
if (
|
|
5601
|
+
if (_a.loadAsInstance === false)
|
|
4885
5602
|
freezeRecord(returnEntry ? value?.value : value);
|
|
4886
5603
|
return value;
|
|
4887
5604
|
};
|
|
@@ -5036,30 +5753,85 @@ function makeTable(options) {
|
|
|
5036
5753
|
this.userSetEmbedders.add(attribute_name);
|
|
5037
5754
|
}
|
|
5038
5755
|
static async deleteHistory(endTime = 0, cleanupDeletedRecords = false) {
|
|
5039
|
-
|
|
5756
|
+
const maxConcurrentRemovals = isRocksDB ? MAX_CONCURRENT_HISTORY_REMOVALS : MAX_CONCURRENT_LMDB_HISTORY_REMOVALS;
|
|
5757
|
+
const inFlightRemovals = new Set();
|
|
5758
|
+
const removalSlotWaiters = [];
|
|
5759
|
+
let removalsAttempted = 0;
|
|
5760
|
+
let removalsSucceeded = 0;
|
|
5761
|
+
let firstRemovalError;
|
|
5762
|
+
function startRemoval(remove, errorMessage, onSuccess) {
|
|
5763
|
+
removalsAttempted++;
|
|
5764
|
+
const removal = new Promise((resolve) => resolve(remove()))
|
|
5765
|
+
.then(() => {
|
|
5766
|
+
removalsSucceeded++;
|
|
5767
|
+
onSuccess?.();
|
|
5768
|
+
}, (error) => {
|
|
5769
|
+
// capture before logging: a throwing logger must not cost us the error we may rethrow
|
|
5770
|
+
if (firstRemovalError === undefined)
|
|
5771
|
+
firstRemovalError = error;
|
|
5772
|
+
harper_logger_ts_1.default.warn(errorMessage, error);
|
|
5773
|
+
})
|
|
5774
|
+
.catch(() => undefined)
|
|
5775
|
+
.finally(() => {
|
|
5776
|
+
inFlightRemovals.delete(removal);
|
|
5777
|
+
removalSlotWaiters.shift()?.();
|
|
5778
|
+
});
|
|
5779
|
+
inFlightRemovals.add(removal);
|
|
5780
|
+
}
|
|
5781
|
+
function queueRemoval(remove, errorMessage, onSuccess) {
|
|
5782
|
+
if (inFlightRemovals.size >= maxConcurrentRemovals) {
|
|
5783
|
+
return new Promise((resolve) => {
|
|
5784
|
+
removalSlotWaiters.push(resolve);
|
|
5785
|
+
}).then(() => startRemoval(remove, errorMessage, onSuccess));
|
|
5786
|
+
}
|
|
5787
|
+
startRemoval(remove, errorMessage, onSuccess);
|
|
5788
|
+
}
|
|
5789
|
+
const drainRemovals = () => Promise.all(inFlightRemovals);
|
|
5040
5790
|
let entriesDeleted = 0;
|
|
5041
|
-
|
|
5042
|
-
|
|
5043
|
-
|
|
5044
|
-
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5791
|
+
try {
|
|
5792
|
+
for (const auditRecord of auditStore.getRange({
|
|
5793
|
+
start: 1, // must not be zero; see getHistory below for why
|
|
5794
|
+
end: endTime,
|
|
5795
|
+
})) {
|
|
5796
|
+
await rest(); // yield to other async operations
|
|
5797
|
+
if (auditRecord.tableId !== tableId)
|
|
5798
|
+
continue;
|
|
5799
|
+
const backpressure = queueRemoval(() => (0, auditStore_ts_1.removeAuditEntry)(auditStore, auditRecord), 'Error removing audit entry during deleteHistory', () => {
|
|
5800
|
+
entriesDeleted++;
|
|
5801
|
+
});
|
|
5802
|
+
if (backpressure)
|
|
5803
|
+
await backpressure;
|
|
5804
|
+
}
|
|
5805
|
+
}
|
|
5806
|
+
finally {
|
|
5807
|
+
await drainRemovals();
|
|
5050
5808
|
}
|
|
5051
5809
|
if (cleanupDeletedRecords) {
|
|
5052
5810
|
// this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't
|
|
5053
5811
|
// ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records
|
|
5054
|
-
|
|
5055
|
-
const {
|
|
5056
|
-
|
|
5057
|
-
|
|
5058
|
-
|
|
5812
|
+
try {
|
|
5813
|
+
for (const entry of primaryStore.getRange({ start: 0, versions: true })) {
|
|
5814
|
+
const { key, value, localTime, version } = entry;
|
|
5815
|
+
await rest(); // yield to other async operations
|
|
5816
|
+
const auditTime = isRocksDB && version != null
|
|
5817
|
+
? resolveAuditHead(key, version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
|
|
5818
|
+
: localTime;
|
|
5819
|
+
if (value === null && version != null && auditTime < endTime) {
|
|
5820
|
+
const backpressure = queueRemoval(() => primaryStore.remove(key, version), 'Error removing deleted record during deleteHistory');
|
|
5821
|
+
if (backpressure)
|
|
5822
|
+
await backpressure;
|
|
5823
|
+
}
|
|
5059
5824
|
}
|
|
5060
5825
|
}
|
|
5826
|
+
finally {
|
|
5827
|
+
await drainRemovals();
|
|
5828
|
+
}
|
|
5829
|
+
}
|
|
5830
|
+
if (removalsAttempted > 0 && removalsSucceeded === 0) {
|
|
5831
|
+
// zero progress must not report the same success as "nothing was eligible" (see DESIGN.md);
|
|
5832
|
+
// partial failures stay best-effort, logged and excluded from the returned count
|
|
5833
|
+
throw firstRemovalError ?? new Error('Every removal attempted during deleteHistory failed');
|
|
5061
5834
|
}
|
|
5062
|
-
await completion;
|
|
5063
5835
|
return entriesDeleted;
|
|
5064
5836
|
}
|
|
5065
5837
|
static async *getHistory(startTime = 0, endTime = Infinity) {
|
|
@@ -5072,10 +5844,11 @@ function makeTable(options) {
|
|
|
5072
5844
|
continue;
|
|
5073
5845
|
yield {
|
|
5074
5846
|
id: auditRecord.recordId,
|
|
5075
|
-
|
|
5847
|
+
// Compatibility-facing LMDB history has always reported/grouped by record version.
|
|
5848
|
+
localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
|
|
5076
5849
|
version: auditRecord.version,
|
|
5077
5850
|
type: auditRecord.type,
|
|
5078
|
-
value: auditRecord.getValue(primaryStore, true, auditRecord.
|
|
5851
|
+
value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
|
|
5079
5852
|
user: auditRecord.user,
|
|
5080
5853
|
operation: auditRecord.originatingOperation,
|
|
5081
5854
|
};
|
|
@@ -5088,7 +5861,9 @@ function makeTable(options) {
|
|
|
5088
5861
|
const entry = primaryStore.getEntry(id);
|
|
5089
5862
|
if (!entry)
|
|
5090
5863
|
return history;
|
|
5091
|
-
let nextVersion =
|
|
5864
|
+
let nextVersion = isRocksDB
|
|
5865
|
+
? resolveAuditHead(id, entry.version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
|
|
5866
|
+
: entry.localTime;
|
|
5092
5867
|
if (!nextVersion)
|
|
5093
5868
|
throw new Error('The entry does not have a local audit time');
|
|
5094
5869
|
const count = 0;
|
|
@@ -5102,17 +5877,20 @@ function makeTable(options) {
|
|
|
5102
5877
|
if (auditRecord.tableId === tableId && (0, ordered_binary_1.compareKeys)(auditRecord.recordId, id) === 0) {
|
|
5103
5878
|
history.splice(insertionPoint, 0, {
|
|
5104
5879
|
id: auditRecord.recordId,
|
|
5105
|
-
localTime: auditRecord.version,
|
|
5880
|
+
localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
|
|
5106
5881
|
version: auditRecord.version,
|
|
5107
5882
|
type: auditRecord.type,
|
|
5108
|
-
// reconstruct each entry's record image as of its own
|
|
5883
|
+
// reconstruct each entry's record image as of its own log position, not the audit
|
|
5109
5884
|
// window boundary (nextVersion), matching getHistory (issue #1330)
|
|
5110
|
-
value: auditRecord.getValue(primaryStore, true, auditRecord.
|
|
5885
|
+
value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
|
|
5111
5886
|
user: auditRecord.user,
|
|
5112
5887
|
operation: auditRecord.originatingOperation,
|
|
5113
5888
|
});
|
|
5114
|
-
|
|
5115
|
-
|
|
5889
|
+
const previousVersion = isRocksDB
|
|
5890
|
+
? resolveAuditHead(id, auditRecord.previousVersion, auditRecord.previousNodeId, auditRecord.previousAdditionalAuditRefs).txnLogKey
|
|
5891
|
+
: auditRecord.previousVersion;
|
|
5892
|
+
if (previousVersion > highestPreviousVersion && previousVersion < start) {
|
|
5893
|
+
highestPreviousVersion = previousVersion;
|
|
5116
5894
|
}
|
|
5117
5895
|
}
|
|
5118
5896
|
}
|
|
@@ -5131,13 +5909,20 @@ function makeTable(options) {
|
|
|
5131
5909
|
}
|
|
5132
5910
|
return Promise.all(promises);
|
|
5133
5911
|
}
|
|
5912
|
+
/** Release everything makeTable() registered process-wide; the class must not be used afterwards. */
|
|
5134
5913
|
static cleanup() {
|
|
5914
|
+
disposed = true;
|
|
5915
|
+
clearTimeout(cleanupTimer);
|
|
5916
|
+
settlePendingCleanup();
|
|
5917
|
+
clearInterval(recordExpirationInterval);
|
|
5135
5918
|
deleteCallbackHandle?.remove();
|
|
5919
|
+
(0, storageReclamation_ts_1.removeStorageReclamationHandler)(primaryStore.path, reclamationHandler);
|
|
5136
5920
|
}
|
|
5137
5921
|
static _readTxnForContext(context) {
|
|
5138
5922
|
return txnForContext(context).getReadTxn();
|
|
5139
5923
|
}
|
|
5140
5924
|
}
|
|
5925
|
+
_a = TableResource;
|
|
5141
5926
|
const throttledCallToSource = (0, throttle_ts_1.throttle)(async (source, id, sourceContext, existingEntry) => {
|
|
5142
5927
|
// call the data source if it exists and will fulfill our request for data
|
|
5143
5928
|
if (source && source.get && (!source.get.reliesOnPrototype || source.prototype.get)) {
|
|
@@ -5151,11 +5936,17 @@ function makeTable(options) {
|
|
|
5151
5936
|
}, () => {
|
|
5152
5937
|
throw new hdbError_ts_1.ServerError('Service unavailable, exceeded request queue limit for resolving cache record', 503);
|
|
5153
5938
|
});
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5939
|
+
try {
|
|
5940
|
+
TableResource.updatedAttributes(); // on creation, update accessors as well
|
|
5941
|
+
if (expirationMs)
|
|
5942
|
+
TableResource.setTTLExpiration(expirationMs / 1000);
|
|
5943
|
+
if (expiresAtProperty)
|
|
5944
|
+
runRecordExpirationEviction();
|
|
5945
|
+
}
|
|
5946
|
+
catch (error) {
|
|
5947
|
+
TableResource.cleanup();
|
|
5948
|
+
throw error;
|
|
5949
|
+
}
|
|
5159
5950
|
return TableResource;
|
|
5160
5951
|
function updateIndices(id, existingRecord, record, options) {
|
|
5161
5952
|
let hasChanges;
|
|
@@ -5582,6 +6373,22 @@ function makeTable(options) {
|
|
|
5582
6373
|
return transaction;
|
|
5583
6374
|
}
|
|
5584
6375
|
}
|
|
6376
|
+
/**
|
|
6377
|
+
* Detach an unsaved TransactionWrite that a scoped lock() eagerly staged (see #reloadLocked)
|
|
6378
|
+
* once its handle upgrades to hold: hold staging is deferred and explicit-save-only, so a
|
|
6379
|
+
* dangling scoped write would otherwise auto-commit at the transaction sweep and clobber
|
|
6380
|
+
* whatever the hold write lands. Marking it .dropped lets a later save() on the instance that
|
|
6381
|
+
* owns it (checked via #savingOperation === this write) fall through to the hold branch
|
|
6382
|
+
* instead of resolving a detached, dead reference.
|
|
6383
|
+
*/
|
|
6384
|
+
function detachScopedUpgradeWrite(link, keyId, handle) {
|
|
6385
|
+
for (const write of link.writes) {
|
|
6386
|
+
if (write && !write.saved && write.lockHandle === handle && (0, DatabaseTransaction_ts_1.writeKeyId)(write.key) === keyId) {
|
|
6387
|
+
write.dropped = true;
|
|
6388
|
+
link.detachWrite(write);
|
|
6389
|
+
}
|
|
6390
|
+
}
|
|
6391
|
+
}
|
|
5585
6392
|
function getAttributeValue(entry, attribute_name, context, sort) {
|
|
5586
6393
|
if (!entry) {
|
|
5587
6394
|
return;
|
|
@@ -5699,6 +6506,10 @@ function makeTable(options) {
|
|
|
5699
6506
|
async function getFromSource(source, id, existingEntry, context, target) {
|
|
5700
6507
|
const metadataFlags = existingEntry?.metadataFlags;
|
|
5701
6508
|
const existingVersion = existingEntry?.version;
|
|
6509
|
+
const existingRecord = existingEntry?.value;
|
|
6510
|
+
const inheritedTimestamp = context?.timestamp || context?.transaction?.timestamp;
|
|
6511
|
+
const sourceTimestamp = inheritedTimestamp ||
|
|
6512
|
+
(isRocksDB ? primaryStore.getMonotonicTimestamp() : (0, commonUtility_ts_1.getNextMonotonicTime)());
|
|
5702
6513
|
let whenResolved, timer;
|
|
5703
6514
|
// We start by locking the record so that there is only one resolution happening at once;
|
|
5704
6515
|
// if there is already a resolution in process, we want to use the results of that resolution
|
|
@@ -5733,10 +6544,8 @@ function makeTable(options) {
|
|
|
5733
6544
|
}
|
|
5734
6545
|
// lock acquired — this request will actually load from source
|
|
5735
6546
|
setLoadedFromSource(target, true);
|
|
5736
|
-
const existingRecord = existingEntry?.value;
|
|
5737
6547
|
// it is important to remember that this is _NOT_ part of the current transaction; nothing is changing
|
|
5738
|
-
// with the canonical data, we are simply fulfilling our local copy of the canonical data
|
|
5739
|
-
// want a timestamp later than the current transaction
|
|
6548
|
+
// with the canonical data, we are simply fulfilling our local copy of the canonical data.
|
|
5740
6549
|
// we create a new context for the source, we want to determine the timestamp and don't want to
|
|
5741
6550
|
// attribute this to the current user
|
|
5742
6551
|
const sourceContext = {
|
|
@@ -5771,15 +6580,36 @@ function makeTable(options) {
|
|
|
5771
6580
|
// belt to that suspenders, at the cost of a bounded wait on a merely slow source
|
|
5772
6581
|
// before the drain's fail-closed timeout below.
|
|
5773
6582
|
const commitPromise = (0, transaction_ts_1.transaction)(sourceContext, async (_txn) => {
|
|
5774
|
-
const start = performance.now();
|
|
5775
|
-
let updatedRecord;
|
|
6583
|
+
const start = node_perf_hooks_1.performance.now();
|
|
6584
|
+
let updatedRecord, assignCreatedTime, sourceVersion;
|
|
5776
6585
|
let hasChanges, invalidated;
|
|
5777
6586
|
try {
|
|
5778
6587
|
updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry);
|
|
5779
6588
|
invalidated = metadataFlags & exports.INVALIDATED;
|
|
5780
|
-
|
|
5781
|
-
|
|
5782
|
-
|
|
6589
|
+
const reportedVersion = sourceContext.lastModified;
|
|
6590
|
+
const validReportedVersion = typeof reportedVersion === 'number' &&
|
|
6591
|
+
Number.isFinite(reportedVersion) &&
|
|
6592
|
+
reportedVersion > 0 &&
|
|
6593
|
+
reportedVersion <= MAX_DATE_TIMESTAMP;
|
|
6594
|
+
if (validReportedVersion) {
|
|
6595
|
+
// A record version is also this node's ordering token (precedesExistingVersion), so a
|
|
6596
|
+
// source-reported version ahead of local time would make every subsequent local write look
|
|
6597
|
+
// out-of-order and be discarded until wall-clock caught up — freezing the row. Honor what
|
|
6598
|
+
// the source reports, but never beyond now.
|
|
6599
|
+
const versionCeiling = Math.max(sourceTimestamp, Date.now());
|
|
6600
|
+
sourceVersion = Math.min(reportedVersion, versionCeiling);
|
|
6601
|
+
if (sourceVersion !== reportedVersion) {
|
|
6602
|
+
logger_ts_1.logger.trace?.(`Capping future source version for ${tableName} id ${id}: ${reportedVersion} -> ${sourceVersion}`);
|
|
6603
|
+
if (!warnedFutureSourceVersion) {
|
|
6604
|
+
warnedFutureSourceVersion = true;
|
|
6605
|
+
logger_ts_1.logger.warn?.(`The source for ${tableName} reported a lastModified ahead of local time (${new Date(reportedVersion).toISOString()}) for id ${id}; capping cached record versions at local time`);
|
|
6606
|
+
}
|
|
6607
|
+
}
|
|
6608
|
+
}
|
|
6609
|
+
else
|
|
6610
|
+
sourceVersion = sourceTimestamp;
|
|
6611
|
+
hasChanges = invalidated || (validReportedVersion && reportedVersion > existingVersion) || !existingRecord;
|
|
6612
|
+
const resolveDuration = node_perf_hooks_1.performance.now() - start;
|
|
5783
6613
|
(0, write_ts_1.recordAction)(resolveDuration, 'cache-resolution', tableName, null, 'success');
|
|
5784
6614
|
if (responseHeaders)
|
|
5785
6615
|
(0, Headers_ts_1.appendHeader)(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
|
|
@@ -5794,7 +6624,7 @@ function makeTable(options) {
|
|
|
5794
6624
|
if (status === 304) {
|
|
5795
6625
|
// revalidation of our current cached record
|
|
5796
6626
|
updatedRecord = existingRecord;
|
|
5797
|
-
|
|
6627
|
+
sourceVersion = existingVersion;
|
|
5798
6628
|
}
|
|
5799
6629
|
else if (!CACHEABLE_STATUS_CODES.has(status)) {
|
|
5800
6630
|
// non-cacheable status - propagate to client without caching
|
|
@@ -5872,10 +6702,14 @@ function makeTable(options) {
|
|
|
5872
6702
|
if (primaryKey && updatedRecord[primaryKey] !== id)
|
|
5873
6703
|
updatedRecord[primaryKey] = id;
|
|
5874
6704
|
}
|
|
6705
|
+
assignCreatedTime = createdTimeProperty && updatedRecord?.[createdTimeProperty.name] == null;
|
|
5875
6706
|
resolved = true;
|
|
6707
|
+
const resolvedVersion = isRocksDB && updatedRecord && existingVersion != null
|
|
6708
|
+
? Math.max(sourceVersion, existingVersion)
|
|
6709
|
+
: sourceVersion;
|
|
5876
6710
|
const resolvedEntry = {
|
|
5877
6711
|
key: id,
|
|
5878
|
-
version,
|
|
6712
|
+
version: resolvedVersion,
|
|
5879
6713
|
value: updatedRecord,
|
|
5880
6714
|
expiresAt: sourceContext.expiresAt,
|
|
5881
6715
|
metadataFlags: 0,
|
|
@@ -5914,7 +6748,7 @@ function makeTable(options) {
|
|
|
5914
6748
|
}
|
|
5915
6749
|
else
|
|
5916
6750
|
reject(error);
|
|
5917
|
-
const resolveDuration = performance.now() - start;
|
|
6751
|
+
const resolveDuration = node_perf_hooks_1.performance.now() - start;
|
|
5918
6752
|
(0, write_ts_1.recordAction)(resolveDuration, 'cache-resolution', tableName, null, 'fail');
|
|
5919
6753
|
if (responseHeaders)
|
|
5920
6754
|
(0, Headers_ts_1.appendHeader)(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
|
|
@@ -5933,16 +6767,28 @@ function makeTable(options) {
|
|
|
5933
6767
|
const sourceWrite = {
|
|
5934
6768
|
key: id,
|
|
5935
6769
|
store: primaryStore,
|
|
5936
|
-
entry:
|
|
6770
|
+
entry: undefined,
|
|
5937
6771
|
nodeName: 'source',
|
|
5938
|
-
commit: (
|
|
6772
|
+
commit: (_txnTime, existingEntry, _retry, transaction) => {
|
|
5939
6773
|
sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true
|
|
5940
|
-
|
|
5941
|
-
|
|
6774
|
+
const racedVersion = existingEntry?.version;
|
|
6775
|
+
// A first fill may replace a record that raced it only when its candidate version strictly
|
|
6776
|
+
// orders after that record. The comparison has to be replica-independent, so a tie leaves the
|
|
6777
|
+
// raced record in place: precedesExistingVersion() would break the tie with *this* node's
|
|
6778
|
+
// name, and a fill from a shared source has no node identity of its own, so two replicas
|
|
6779
|
+
// resolving the same tie could keep different values at the same version.
|
|
6780
|
+
const replacesRacedRecord = racedVersion == null || sourceVersion > racedVersion;
|
|
6781
|
+
if (racedVersion !== existingVersion &&
|
|
6782
|
+
// Revalidations retain exact-CAS semantics; first fills use deterministic ordering.
|
|
6783
|
+
(existingVersion != null || !updatedRecord || !replacesRacedRecord)) {
|
|
6784
|
+
logger_ts_1.logger.trace?.(`Discarding resolved record from source with id: ${id}, source version: ${sourceVersion}, current version: ${racedVersion}`);
|
|
5942
6785
|
sourceWrite.skipped = true;
|
|
5943
6786
|
return;
|
|
5944
6787
|
}
|
|
5945
|
-
|
|
6788
|
+
const currentRecord = existingEntry?.value;
|
|
6789
|
+
const recordVersion = isRocksDB && racedVersion != null ? Math.max(sourceVersion, racedVersion) : sourceVersion;
|
|
6790
|
+
const txnLogKey = isRocksDB ? transaction?.getTimestamp?.() : recordVersion;
|
|
6791
|
+
updateIndices(id, currentRecord, updatedRecord, transaction && { transaction });
|
|
5946
6792
|
if (updatedRecord) {
|
|
5947
6793
|
if (existingEntry) {
|
|
5948
6794
|
context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
|
|
@@ -5953,23 +6799,23 @@ function makeTable(options) {
|
|
|
5953
6799
|
if (updatedTimeProperty) {
|
|
5954
6800
|
updatedRecord[updatedTimeProperty.name] =
|
|
5955
6801
|
updatedTimeProperty.type === 'Date'
|
|
5956
|
-
? new Date(
|
|
6802
|
+
? new Date(recordVersion)
|
|
5957
6803
|
: updatedTimeProperty.type === 'String'
|
|
5958
|
-
? new Date(
|
|
5959
|
-
:
|
|
6804
|
+
? new Date(recordVersion).toISOString()
|
|
6805
|
+
: recordVersion;
|
|
5960
6806
|
}
|
|
5961
|
-
if (
|
|
5962
|
-
const existingCreatedTime =
|
|
6807
|
+
if (assignCreatedTime) {
|
|
6808
|
+
const existingCreatedTime = currentRecord?.[createdTimeProperty.name];
|
|
5963
6809
|
if (existingCreatedTime != null) {
|
|
5964
6810
|
updatedRecord[createdTimeProperty.name] = existingCreatedTime;
|
|
5965
6811
|
}
|
|
5966
6812
|
else {
|
|
5967
6813
|
updatedRecord[createdTimeProperty.name] =
|
|
5968
6814
|
createdTimeProperty.type === 'Date'
|
|
5969
|
-
? new Date(
|
|
6815
|
+
? new Date(recordVersion)
|
|
5970
6816
|
: createdTimeProperty.type === 'String'
|
|
5971
|
-
? new Date(
|
|
5972
|
-
:
|
|
6817
|
+
? new Date(recordVersion).toISOString()
|
|
6818
|
+
: recordVersion;
|
|
5973
6819
|
}
|
|
5974
6820
|
}
|
|
5975
6821
|
const residency = residencyFromFunction(TableResource.getResidency(updatedRecord, context));
|
|
@@ -6002,23 +6848,33 @@ function makeTable(options) {
|
|
|
6002
6848
|
}
|
|
6003
6849
|
residencyId = getResidencyId(residency);
|
|
6004
6850
|
}
|
|
6005
|
-
logger_ts_1.logger.trace?.(`Writing resolved record from source with id: ${id}, timestamp: ${new Date(
|
|
6851
|
+
logger_ts_1.logger.trace?.(`Writing resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`);
|
|
6006
6852
|
// TODO: We are doing a double check for ifVersion that should probably be cleaned out
|
|
6007
|
-
|
|
6853
|
+
const writeAudit = (audit && (hasChanges || omitLocalRecord)) || null;
|
|
6854
|
+
updateRecord(id, updatedRecord, existingEntry, recordVersion, omitLocalRecord ? exports.INVALIDATED : 0, writeAudit, {
|
|
6008
6855
|
user: sourceContext?.user,
|
|
6009
6856
|
expiresAt: sourceContext.expiresAt,
|
|
6010
6857
|
residencyId,
|
|
6011
6858
|
transaction,
|
|
6012
6859
|
tableToTrack: tableName,
|
|
6860
|
+
additionalAuditRefs: writeAudit && txnLogKey !== recordVersion ? [{ version: txnLogKey, nodeId: 0 }] : undefined,
|
|
6013
6861
|
}, 'put', Boolean(invalidated), auditRecord);
|
|
6014
6862
|
// arm the eviction scanner, mirroring the .put() path
|
|
6015
6863
|
if (sourceContext.expiresAt)
|
|
6016
6864
|
scheduleCleanup();
|
|
6017
6865
|
}
|
|
6018
6866
|
else if (existingEntry) {
|
|
6019
|
-
logger_ts_1.logger.trace?.(`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(
|
|
6867
|
+
logger_ts_1.logger.trace?.(`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`);
|
|
6020
6868
|
if (audit || trackDeletes) {
|
|
6021
|
-
updateRecord(id, null, existingEntry,
|
|
6869
|
+
updateRecord(id, null, existingEntry, recordVersion, 0, (audit && hasChanges) || null, {
|
|
6870
|
+
user: sourceContext?.user,
|
|
6871
|
+
transaction,
|
|
6872
|
+
tableToTrack: tableName,
|
|
6873
|
+
recordVersion,
|
|
6874
|
+
additionalAuditRefs: audit && hasChanges && txnLogKey !== recordVersion
|
|
6875
|
+
? [{ version: txnLogKey, nodeId: 0 }]
|
|
6876
|
+
: undefined,
|
|
6877
|
+
}, 'delete', Boolean(invalidated));
|
|
6022
6878
|
}
|
|
6023
6879
|
else {
|
|
6024
6880
|
(0, RecordEncoder_ts_1.removeEntry)(primaryStore, existingEntry, existingVersion);
|
|
@@ -6178,7 +7034,15 @@ function makeTable(options) {
|
|
|
6178
7034
|
},
|
|
6179
7035
|
};
|
|
6180
7036
|
}
|
|
7037
|
+
function settlePendingCleanup() {
|
|
7038
|
+
for (const resolve of pendingCleanupResolvers)
|
|
7039
|
+
resolve();
|
|
7040
|
+
pendingCleanupResolvers.clear();
|
|
7041
|
+
}
|
|
6181
7042
|
function scheduleCleanup(priority) {
|
|
7043
|
+
// a reclamation run may still hold this class's handler after cleanup(); a promise here would never settle
|
|
7044
|
+
if (disposed)
|
|
7045
|
+
return;
|
|
6182
7046
|
let runImmediately = false;
|
|
6183
7047
|
if (priority) {
|
|
6184
7048
|
// run immediately if there is a big increase in priority
|
|
@@ -6194,9 +7058,19 @@ function makeTable(options) {
|
|
|
6194
7058
|
// run on the last thread so we aren't overloading lower-numbered threads
|
|
6195
7059
|
if (cleanupTimer)
|
|
6196
7060
|
clearTimeout(cleanupTimer);
|
|
6197
|
-
if (!cleanupInterval)
|
|
7061
|
+
if (!cleanupInterval) {
|
|
7062
|
+
// no replacement pass is being scheduled, so nothing is left to settle a superseded one
|
|
7063
|
+
settlePendingCleanup();
|
|
6198
7064
|
return;
|
|
7065
|
+
}
|
|
7066
|
+
// This pass adopts the awaiters of the pass whose timer it just cleared: they settle when
|
|
7067
|
+
// this pass's scan completes, so a reclamation run is never told the storage was reclaimed
|
|
7068
|
+
// before any scan ran. It has to run now, though — that run blocks its whole path on the
|
|
7069
|
+
// promise, and the replacement's own slot can be a full interval out.
|
|
7070
|
+
if (pendingCleanupResolvers.size > 0)
|
|
7071
|
+
runImmediately = true;
|
|
6199
7072
|
return new Promise((resolve) => {
|
|
7073
|
+
pendingCleanupResolvers.add(resolve);
|
|
6200
7074
|
const startOfYear = new Date();
|
|
6201
7075
|
startOfYear.setMonth(0);
|
|
6202
7076
|
startOfYear.setDate(1);
|
|
@@ -6209,6 +7083,8 @@ function makeTable(options) {
|
|
|
6209
7083
|
? Date.now()
|
|
6210
7084
|
: Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime();
|
|
6211
7085
|
const startNextTimer = (nextScheduled) => {
|
|
7086
|
+
if (disposed)
|
|
7087
|
+
return;
|
|
6212
7088
|
logger_ts_1.logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`);
|
|
6213
7089
|
// noinspection JSVoidFunctionReturnValueUsed
|
|
6214
7090
|
cleanupTimer = setTimeout(() => (lastEvictionCompletion = lastEvictionCompletion.then(async () => {
|
|
@@ -6217,8 +7093,11 @@ function makeTable(options) {
|
|
|
6217
7093
|
const rootStore = primaryStore.rootStore;
|
|
6218
7094
|
if (rootStore.status !== 'open') {
|
|
6219
7095
|
clearTimeout(cleanupTimer);
|
|
7096
|
+
settlePendingCleanup();
|
|
6220
7097
|
return;
|
|
6221
7098
|
}
|
|
7099
|
+
// snapshot: an awaiter that arrives during this scan belongs to the pass that supersedes it
|
|
7100
|
+
const settling = [...pendingCleanupResolvers];
|
|
6222
7101
|
const MAX_CLEANUP_CONCURRENCY = 50;
|
|
6223
7102
|
const outstandingCleanupOperations = new Array(MAX_CLEANUP_CONCURRENCY);
|
|
6224
7103
|
let cleanupIndex = 0;
|
|
@@ -6298,7 +7177,10 @@ function makeTable(options) {
|
|
|
6298
7177
|
catch (error) {
|
|
6299
7178
|
logger_ts_1.logger.warn?.(`Error in cleanup scan for ${tableName}:`, error);
|
|
6300
7179
|
}
|
|
6301
|
-
|
|
7180
|
+
for (const settle of settling) {
|
|
7181
|
+
pendingCleanupResolvers.delete(settle);
|
|
7182
|
+
settle();
|
|
7183
|
+
}
|
|
6302
7184
|
cleanupPriority = 0; // reset the priority
|
|
6303
7185
|
})), Math.min(nextScheduled - Date.now(), hdbTerms_ts_1.MAX_SET_TIMEOUT_MS) // make sure it can fit in 32-bit signed number
|
|
6304
7186
|
).unref(); // don't let this prevent closing the thread
|
|
@@ -6309,17 +7191,17 @@ function makeTable(options) {
|
|
|
6309
7191
|
}
|
|
6310
7192
|
function addDeleteRemoval() {
|
|
6311
7193
|
deleteCallbackHandle = auditStore?.addDeleteRemovalCallback(tableId, primaryStore, (id, version) => {
|
|
6312
|
-
primaryStore.remove(id, version);
|
|
7194
|
+
return primaryStore.remove(id, version);
|
|
6313
7195
|
});
|
|
6314
7196
|
}
|
|
6315
7197
|
function runRecordExpirationEviction() {
|
|
6316
7198
|
// Periodically evict expired records, searching for records who expiresAt timestamp is before now
|
|
6317
7199
|
if ((0, manageThreads_js_1.getWorkerIndex)() === 0) {
|
|
6318
7200
|
// we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting
|
|
6319
|
-
setInterval(async () => {
|
|
7201
|
+
recordExpirationInterval = setInterval(async () => {
|
|
6320
7202
|
// go through each database and table and then search for expired entries
|
|
6321
7203
|
// find any entries that are set to expire before now
|
|
6322
|
-
if (runningRecordExpiration)
|
|
7204
|
+
if (disposed || runningRecordExpiration)
|
|
6323
7205
|
return;
|
|
6324
7206
|
runningRecordExpiration = true;
|
|
6325
7207
|
try {
|