@harperfast/harper 5.2.10 → 5.3.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/mcpTools.ts +1 -1
- package/agent/session.ts +25 -14
- package/bin/cliOperations.ts +46 -9
- package/bin/copyDb.ts +282 -59
- package/bin/deploySetup.ts +16 -5
- package/bin/harper.ts +1 -1
- package/bin/help.ts +4 -1
- package/bin/lite.ts +4 -1
- package/bin/restart.ts +59 -3
- package/bin/run.ts +6 -11
- package/bin/upgrade.js +7 -3
- package/bin/workloadIdentity.ts +119 -0
- package/components/Application.ts +2414 -242
- package/components/ApplicationScope.ts +8 -0
- package/components/EntryHandler.ts +59 -39
- package/components/OptionsWatcher.ts +150 -74
- package/components/RuntimeModuleTracker.ts +38 -7
- package/components/Scope.ts +37 -15
- package/components/awaitRestart.ts +84 -0
- package/components/componentLoader.ts +340 -30
- package/components/componentPreparationLock.ts +16 -5
- package/components/mcp/adapters/harperHttp.ts +4 -0
- package/components/mcp/listChanged.ts +4 -0
- package/components/mcp/toolRegistry.ts +2 -0
- package/components/mcp/tools/operations.ts +9 -0
- package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
- package/components/operations.js +270 -78
- package/components/operationsValidation.js +49 -1
- package/components/status/ComponentStatusRegistry.ts +59 -0
- package/config/RootConfigWatcher.ts +80 -34
- package/config/configUtils.ts +291 -34
- package/config/harperConfigEnvVars.ts +170 -27
- package/config-root.schema.json +29 -0
- package/dataLayer/blobBackup.ts +160 -50
- package/dataLayer/delete.ts +6 -1
- package/dataLayer/harperBridge/ResourceBridge.ts +52 -8
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dataLayer/hdbInfoController.ts +34 -1
- package/dataLayer/insert.ts +44 -1
- package/dataLayer/rocksdbBackup.ts +53 -10
- package/dataLayer/schema.ts +11 -1
- package/dataLayer/schemaDescribe.ts +8 -1
- package/dist/agent/mcpTools.js +1 -1
- package/dist/agent/mcpTools.js.map +1 -1
- package/dist/agent/session.d.ts +22 -0
- package/dist/agent/session.js +26 -15
- package/dist/agent/session.js.map +1 -1
- package/dist/bin/cliOperations.js +48 -9
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/copyDb.d.ts +12 -1
- package/dist/bin/copyDb.js +248 -60
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/bin/deploySetup.d.ts +2 -0
- package/dist/bin/deploySetup.js +11 -3
- package/dist/bin/deploySetup.js.map +1 -1
- package/dist/bin/harper.js +1 -1
- package/dist/bin/harper.js.map +1 -1
- package/dist/bin/help.js +4 -1
- package/dist/bin/help.js.map +1 -1
- package/dist/bin/lite.js +4 -1
- package/dist/bin/lite.js.map +1 -1
- package/dist/bin/restart.js +54 -5
- package/dist/bin/restart.js.map +1 -1
- package/dist/bin/run.js +4 -10
- package/dist/bin/run.js.map +1 -1
- package/dist/bin/upgrade.js +4 -3
- package/dist/bin/upgrade.js.map +1 -1
- package/dist/bin/workloadIdentity.d.ts +18 -0
- package/dist/bin/workloadIdentity.js +100 -0
- package/dist/bin/workloadIdentity.js.map +1 -0
- package/dist/components/Application.d.ts +139 -16
- package/dist/components/Application.js +2215 -267
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +8 -0
- package/dist/components/ApplicationScope.js +7 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/EntryHandler.js +26 -10
- package/dist/components/EntryHandler.js.map +1 -1
- package/dist/components/OptionsWatcher.d.ts +1 -0
- package/dist/components/OptionsWatcher.js +141 -74
- package/dist/components/OptionsWatcher.js.map +1 -1
- package/dist/components/RuntimeModuleTracker.js +40 -6
- package/dist/components/RuntimeModuleTracker.js.map +1 -1
- package/dist/components/Scope.js +38 -13
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/awaitRestart.d.ts +33 -0
- package/dist/components/awaitRestart.js +61 -0
- package/dist/components/awaitRestart.js.map +1 -0
- package/dist/components/componentLoader.d.ts +38 -1
- package/dist/components/componentLoader.js +279 -22
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentPreparationLock.d.ts +5 -0
- package/dist/components/componentPreparationLock.js +14 -6
- package/dist/components/componentPreparationLock.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.js +4 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/listChanged.js +5 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +1 -0
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +5 -0
- package/dist/components/mcp/tools/operations.js +9 -0
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/operations.js +231 -77
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.js +49 -1
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
- package/dist/components/status/ComponentStatusRegistry.js +63 -0
- package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
- package/dist/config/RootConfigWatcher.d.ts +7 -1
- package/dist/config/RootConfigWatcher.js +64 -27
- package/dist/config/RootConfigWatcher.js.map +1 -1
- package/dist/config/configUtils.d.ts +9 -1
- package/dist/config/configUtils.js +254 -33
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/harperConfigEnvVars.d.ts +16 -0
- package/dist/config/harperConfigEnvVars.js +162 -25
- package/dist/config/harperConfigEnvVars.js.map +1 -1
- package/dist/dataLayer/blobBackup.d.ts +49 -20
- package/dist/dataLayer/blobBackup.js +139 -50
- package/dist/dataLayer/blobBackup.js.map +1 -1
- package/dist/dataLayer/delete.js +1 -1
- package/dist/dataLayer/delete.js.map +1 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.js +51 -10
- package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
- package/dist/dataLayer/hdbInfoController.d.ts +10 -0
- package/dist/dataLayer/hdbInfoController.js +30 -1
- package/dist/dataLayer/hdbInfoController.js.map +1 -1
- package/dist/dataLayer/insert.d.ts +9 -1
- package/dist/dataLayer/insert.js +30 -0
- package/dist/dataLayer/insert.js.map +1 -1
- package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
- package/dist/dataLayer/rocksdbBackup.js +45 -8
- package/dist/dataLayer/rocksdbBackup.js.map +1 -1
- package/dist/dataLayer/schema.js +8 -0
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +8 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +52 -0
- package/dist/resources/DatabaseTransaction.d.ts +55 -3
- package/dist/resources/DatabaseTransaction.js +460 -134
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.js +22 -4
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
- package/dist/resources/PrimaryRocksDatabase.js +30 -2
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
- package/dist/resources/RecordEncoder.d.ts +20 -0
- package/dist/resources/RecordEncoder.js +94 -9
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +2 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.js +20 -11
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +20 -1
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/RocksIndexStore.d.ts +6 -1
- package/dist/resources/RocksIndexStore.js +24 -10
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.d.ts +14 -1
- package/dist/resources/RocksTransactionLogStore.js +57 -17
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.d.ts +112 -8
- package/dist/resources/Table.js +1071 -197
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/auditStore.d.ts +11 -2
- package/dist/resources/auditStore.js +183 -18
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/blob.d.ts +129 -9
- package/dist/resources/blob.js +992 -114
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/branchDatabase.d.ts +48 -0
- package/dist/resources/branchDatabase.js +892 -0
- package/dist/resources/branchDatabase.js.map +1 -0
- package/dist/resources/crdt.js +50 -12
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/databases.d.ts +150 -9
- package/dist/resources/databases.js +1247 -525
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/defineTable.d.ts +10 -2
- package/dist/resources/defineTable.js +9 -1
- package/dist/resources/defineTable.js.map +1 -1
- package/dist/resources/graphql.d.ts +1 -1
- package/dist/resources/graphql.js +50 -15
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +1 -0
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +6 -2
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/longLivedTransactions.d.ts +71 -0
- package/dist/resources/longLivedTransactions.js +358 -0
- package/dist/resources/longLivedTransactions.js.map +1 -0
- package/dist/resources/models/backendRegistry.d.ts +26 -0
- package/dist/resources/models/backendRegistry.js +60 -2
- package/dist/resources/models/backendRegistry.js.map +1 -1
- package/dist/resources/models/bootstrap.d.ts +33 -1
- package/dist/resources/models/bootstrap.js +416 -31
- package/dist/resources/models/bootstrap.js.map +1 -1
- package/dist/resources/recordLock.d.ts +80 -0
- package/dist/resources/recordLock.js +184 -0
- package/dist/resources/recordLock.js.map +1 -0
- package/dist/resources/replayLogs.d.ts +14 -1
- package/dist/resources/replayLogs.js +152 -24
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/replayLogsGuards.d.ts +94 -7
- package/dist/resources/replayLogsGuards.js +111 -7
- package/dist/resources/replayLogsGuards.js.map +1 -1
- package/dist/resources/search.js +158 -32
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transactionBroadcast.d.ts +1 -1
- package/dist/resources/transactionBroadcast.js +2 -2
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/auth.js +61 -30
- package/dist/security/auth.js.map +1 -1
- package/dist/security/authn/oidc/claims.d.ts +22 -0
- package/dist/security/authn/oidc/claims.js +71 -0
- package/dist/security/authn/oidc/claims.js.map +1 -0
- package/dist/security/authn/oidc/identityToken.d.ts +27 -0
- package/dist/security/authn/oidc/identityToken.js +111 -0
- package/dist/security/authn/oidc/identityToken.js.map +1 -0
- package/dist/security/authn/oidc/jwks.d.ts +25 -0
- package/dist/security/authn/oidc/jwks.js +261 -0
- package/dist/security/authn/oidc/jwks.js.map +1 -0
- package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
- package/dist/security/authn/oidc/providers/generic.js +34 -0
- package/dist/security/authn/oidc/providers/generic.js.map +1 -0
- package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
- package/dist/security/authn/oidc/providers/githubActions.js +129 -0
- package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
- package/dist/security/authn/oidc/providers/index.d.ts +37 -0
- package/dist/security/authn/oidc/providers/index.js +24 -0
- package/dist/security/authn/oidc/providers/index.js.map +1 -0
- package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
- package/dist/security/authn/oidc/tokenExchange.js +306 -0
- package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
- package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
- package/dist/security/authn/oidc/types.d.ts +38 -0
- package/dist/security/authn/oidc/types.js +6 -0
- package/dist/security/authn/oidc/types.js.map +1 -0
- package/dist/security/certificateVerification/index.js +40 -11
- package/dist/security/certificateVerification/index.js.map +1 -1
- package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
- package/dist/security/certificateVerification/trustedIssuers.js +79 -0
- package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
- package/dist/security/certificateVerification/types.d.ts +1 -0
- package/dist/security/credentialProvenance.d.ts +35 -0
- package/dist/security/credentialProvenance.js +51 -0
- package/dist/security/credentialProvenance.js.map +1 -0
- package/dist/security/credentialRejection.d.ts +4 -0
- package/dist/security/credentialRejection.js +24 -0
- package/dist/security/credentialRejection.js.map +1 -0
- package/dist/security/deferredAuthentication.d.ts +36 -0
- package/dist/security/deferredAuthentication.js +70 -0
- package/dist/security/deferredAuthentication.js.map +1 -0
- package/dist/security/impersonation.d.ts +21 -0
- package/dist/security/impersonation.js +108 -9
- package/dist/security/impersonation.js.map +1 -1
- package/dist/security/jsLoader.d.ts +6 -0
- package/dist/security/jsLoader.js +75 -15
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +301 -71
- package/dist/security/keys.js.map +1 -1
- package/dist/security/operationScope.d.ts +21 -0
- package/dist/security/operationScope.js +36 -0
- package/dist/security/operationScope.js.map +1 -0
- package/dist/security/permissionsTranslator.js +21 -0
- package/dist/security/permissionsTranslator.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +19 -1
- package/dist/security/tokenAuthentication.js +191 -10
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/security/user.js +4 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
- package/dist/server/DurableSubscriptionsSession.js +67 -10
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +73 -0
- package/dist/server/REST.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -0
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +11 -0
- package/dist/server/http.js +88 -15
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +8 -1
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/jobs/jobProcess.js +6 -2
- package/dist/server/jobs/jobProcess.js.map +1 -1
- package/dist/server/jobs/jobs.js +4 -1
- package/dist/server/jobs/jobs.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +26 -4
- package/dist/server/liveSubscriptionAuth.js +105 -39
- package/dist/server/liveSubscriptionAuth.js.map +1 -1
- package/dist/server/loadRootComponents.js +49 -10
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/mqtt.d.ts +2 -0
- package/dist/server/mqtt.js +165 -30
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/nodeName.d.ts +2 -0
- package/dist/server/nodeName.js +107 -23
- package/dist/server/nodeName.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +25 -0
- package/dist/server/serverHelpers/Headers.js +139 -1
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.d.ts +9 -0
- package/dist/server/serverHelpers/contentTypes.js +32 -25
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
- package/dist/server/serverHelpers/deployValidationState.js +9 -19
- package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
- package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
- package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
- package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
- package/dist/server/serverHelpers/registeredOperations.js +74 -21
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
- package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
- package/dist/server/serverHelpers/requestSanitization.js +20 -0
- package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +6 -3
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
- package/dist/server/serverHelpers/serverUtilities.js +90 -20
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
- package/dist/server/serverHelpers/uwsServer.js +19 -1
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/static.js +24 -28
- package/dist/server/static.js.map +1 -1
- package/dist/server/storageReclamation.d.ts +5 -0
- package/dist/server/storageReclamation.js +16 -0
- package/dist/server/storageReclamation.js.map +1 -1
- package/dist/server/threads/itc.d.ts +7 -2
- package/dist/server/threads/itc.js +5 -1
- package/dist/server/threads/itc.js.map +1 -1
- package/dist/server/threads/manageThreads.d.ts +26 -4
- package/dist/server/threads/manageThreads.js +517 -72
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.js +89 -26
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadHeapMemory.d.ts +2 -0
- package/dist/server/threads/threadHeapMemory.js +31 -0
- package/dist/server/threads/threadHeapMemory.js.map +1 -0
- package/dist/server/threads/threadServer.js +46 -15
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/sqlEngine/config.d.ts +1 -3
- package/dist/sqlEngine/config.js +19 -16
- package/dist/sqlEngine/config.js.map +1 -1
- package/dist/sqlTranslator/index.d.ts +1 -1
- package/dist/sqlTranslator/index.js +30 -7
- package/dist/sqlTranslator/index.js.map +1 -1
- package/dist/upgrade/directives/5-3-0.d.ts +7 -0
- package/dist/upgrade/directives/5-3-0.js +148 -0
- package/dist/upgrade/directives/5-3-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/OperationFunctionCaller.js +2 -1
- package/dist/utility/OperationFunctionCaller.js.map +1 -1
- package/dist/utility/common_utils.d.ts +16 -0
- package/dist/utility/common_utils.js +32 -6
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/componentNames.d.ts +8 -0
- package/dist/utility/componentNames.js +12 -1
- package/dist/utility/componentNames.js.map +1 -1
- package/dist/utility/environment/environmentManager.js +3 -6
- package/dist/utility/environment/environmentManager.js.map +1 -1
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/environment/systemInformation.js +1 -0
- package/dist/utility/environment/systemInformation.js.map +1 -1
- package/dist/utility/errors/commonErrors.d.ts +2 -0
- package/dist/utility/errors/commonErrors.js +2 -0
- package/dist/utility/errors/commonErrors.js.map +1 -1
- package/dist/utility/errors/hdbError.d.ts +17 -0
- package/dist/utility/errors/hdbError.js +30 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/globalSchema.d.ts +18 -0
- package/dist/utility/hdbTerms.d.ts +16 -0
- package/dist/utility/hdbTerms.js +18 -2
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +2 -0
- package/dist/utility/logging/harper_logger.js +91 -14
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logRotator.js +76 -49
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/nodeIdentity.d.ts +9 -0
- package/dist/utility/nodeIdentity.js +58 -0
- package/dist/utility/nodeIdentity.js.map +1 -0
- package/dist/utility/npmUtilities.js +9 -7
- package/dist/utility/npmUtilities.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +3 -1
- package/dist/utility/operationPermissions.js +16 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +10 -7
- package/dist/utility/operation_authorization.js +212 -41
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/watchPath.d.ts +29 -0
- package/dist/utility/watchPath.js +68 -0
- package/dist/utility/watchPath.js.map +1 -0
- package/dist/utility/watcherFallback.d.ts +86 -0
- package/dist/utility/watcherFallback.js +278 -1
- package/dist/utility/watcherFallback.js.map +1 -1
- package/dist/validation/configValidator.d.ts +12 -0
- package/dist/validation/configValidator.js +199 -72
- package/dist/validation/configValidator.js.map +1 -1
- package/dist/validation/installValidator.js +12 -0
- package/dist/validation/installValidator.js.map +1 -1
- package/dist/validation/validationWrapper.d.ts +11 -0
- package/dist/validation/validationWrapper.js +16 -3
- package/dist/validation/validationWrapper.js.map +1 -1
- package/index.ts +2 -0
- package/json/systemSchema.json +52 -0
- package/npm-shrinkwrap.json +235 -233
- package/package.json +8 -7
- package/resources/DESIGN.md +114 -52
- package/resources/DatabaseTransaction.ts +522 -130
- package/resources/LMDBTransaction.ts +26 -4
- package/resources/PrimaryRocksDatabase.ts +29 -3
- package/resources/RecordEncoder.ts +103 -8
- package/resources/RequestTarget.ts +2 -0
- package/resources/Resource.ts +17 -9
- package/resources/ResourceInterface.ts +23 -0
- package/resources/RocksIndexStore.ts +30 -10
- package/resources/RocksTransactionLogStore.ts +78 -21
- package/resources/Table.ts +1107 -164
- package/resources/auditStore.ts +187 -21
- package/resources/blob.ts +1029 -110
- package/resources/branchDatabase.ts +941 -0
- package/resources/crdt.ts +76 -12
- package/resources/databases.ts +1354 -522
- package/resources/defineTable.ts +18 -2
- package/resources/graphql.ts +70 -16
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +6 -2
- package/resources/longLivedTransactions.ts +360 -0
- package/resources/models/backendRegistry.ts +84 -2
- package/resources/models/bootstrap.ts +473 -28
- package/resources/recordLock.ts +253 -0
- package/resources/replayLogs.ts +147 -26
- package/resources/replayLogsGuards.ts +171 -8
- package/resources/search.ts +154 -32
- package/resources/transactionBroadcast.ts +3 -3
- package/security/auth.ts +68 -29
- package/security/authn/oidc/claims.ts +72 -0
- package/security/authn/oidc/identityToken.ts +129 -0
- package/security/authn/oidc/jwks.ts +260 -0
- package/security/authn/oidc/providers/generic.ts +40 -0
- package/security/authn/oidc/providers/githubActions.ts +137 -0
- package/security/authn/oidc/providers/index.ts +52 -0
- package/security/authn/oidc/tokenExchange.ts +300 -0
- package/security/authn/oidc/trustPolicyOperations.ts +343 -0
- package/security/authn/oidc/types.ts +41 -0
- package/security/certificateVerification/index.ts +54 -13
- package/security/certificateVerification/trustedIssuers.ts +76 -0
- package/security/certificateVerification/types.ts +1 -0
- package/security/credentialProvenance.ts +47 -0
- package/security/credentialRejection.ts +22 -0
- package/security/deferredAuthentication.ts +71 -0
- package/security/impersonation.ts +117 -12
- package/security/jsLoader.ts +81 -18
- package/security/keys.ts +298 -72
- package/security/operationScope.ts +33 -0
- package/security/permissionsTranslator.js +23 -0
- package/security/tokenAuthentication.ts +233 -12
- package/security/user.ts +4 -3
- package/server/DESIGN.md +183 -16
- package/server/DurableSubscriptionsSession.ts +71 -11
- package/server/REST.ts +79 -1
- package/server/graphqlQuerying.ts +4 -0
- package/server/http.ts +99 -18
- package/server/itc/serverHandlers.js +8 -1
- package/server/jobs/jobProcess.ts +8 -2
- package/server/jobs/jobs.ts +4 -1
- package/server/liveSubscriptionAuth.ts +129 -46
- package/server/loadRootComponents.js +50 -8
- package/server/mqtt.ts +179 -38
- package/server/nodeName.ts +103 -21
- package/server/serverHelpers/Headers.ts +131 -0
- package/server/serverHelpers/contentTypes.ts +29 -21
- package/server/serverHelpers/deployValidationState.ts +24 -13
- package/server/serverHelpers/operationAuthorizationState.ts +34 -3
- package/server/serverHelpers/registeredOperations.ts +79 -22
- package/server/serverHelpers/requestSanitization.ts +15 -0
- package/server/serverHelpers/serverHandlers.js +6 -3
- package/server/serverHelpers/serverUtilities.ts +104 -31
- package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
- package/server/serverHelpers/uwsServer.ts +17 -2
- package/server/static.ts +23 -29
- package/server/storageReclamation.ts +13 -0
- package/server/threads/itc.js +11 -1
- package/server/threads/manageThreads.js +526 -63
- package/server/threads/socketRouter.ts +81 -26
- package/server/threads/threadHeapMemory.ts +26 -0
- package/server/threads/threadServer.js +43 -15
- package/sqlTranslator/index.ts +31 -8
- package/studio/web/assets/{Chat-Br06zdMA.js → Chat-BnCBegQz.js} +1 -1
- package/studio/web/assets/{FloatingChat-BWImX5fA.js → FloatingChat-CoDW1ySS.js} +3 -3
- package/studio/web/assets/{apiToken-DN0nmDsq.js → apiToken-Bwk5BLXW.js} +1 -1
- package/studio/web/assets/{applications-kSxVoyeU.js → applications-DHxGi7JH.js} +1 -1
- package/studio/web/assets/{cssMode-C1JeufH5.js → cssMode-s0cWI-_M.js} +1 -1
- package/studio/web/assets/{editor-qoo9CrGO.js → editor-DNcRHK54.js} +1 -1
- package/studio/web/assets/{html-Dt4IIy04.js → html-Bdssedlg.js} +1 -1
- package/studio/web/assets/{htmlMode-DXgKKr4C.js → htmlMode-CoDlJ3fw.js} +1 -1
- package/studio/web/assets/{index-6onkYFOG.js → index-D6sxmFLR.js} +5 -5
- package/studio/web/assets/{index.lazy-BrCFnpNJ.js → index.lazy-tmU5BS8s.js} +1 -1
- package/studio/web/assets/{javascript-DNCQGUBc.js → javascript-B8meVSTH.js} +1 -1
- package/studio/web/assets/{jsonMode-CR6HWruP.js → jsonMode-DpIPd35T.js} +1 -1
- package/studio/web/assets/{languageServices-BM4fI4rS.js → languageServices-C_5FMJzQ.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-DSa1ttcD.js → lspLanguageFeatures-BIzNBzPK.js} +1 -1
- package/studio/web/assets/{notifications-BHXLnh6x.js → notifications-CQf18QKb.js} +1 -1
- package/studio/web/assets/{notifications-CUmtIA6z.js → notifications-CvZivSbh.js} +1 -1
- package/studio/web/assets/{profile-8BeFSF3j.js → profile-DdOwtntb.js} +1 -1
- package/studio/web/assets/{regions-C8qR0HhD.js → regions-n69fwagr.js} +1 -1
- package/studio/web/assets/{register-B4n5i0SD.js → register-PfWTCXWB.js} +2 -2
- package/studio/web/assets/{setComponentFile-g0_B0lgX.js → setComponentFile-Bg6O7X0S.js} +1 -1
- package/studio/web/assets/{setup-B0CTj_Q5.js → setup-CUx_aUDl.js} +2 -2
- package/studio/web/assets/{status-C6Yib7-K.js → status-D7BVKqX9.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-Dgta7bVi.js → toggleHighContrast-DBSyXzMr.js} +1 -1
- package/studio/web/assets/{tsMode-CH_jHvU-.js → tsMode-BByKCjBS.js} +1 -1
- package/studio/web/assets/{typescript-Co9LCXd5.js → typescript-DDLnLpw9.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-wKC8NsC_.js → useEntityRestURL-31CHGaHk.js} +1 -1
- package/studio/web/assets/{workers-CWeLxCXA.js → workers-pR3jRY9D.js} +1 -1
- package/studio/web/assets/{xml-BadC-0Rk.js → xml-2iRnMhQO.js} +1 -1
- package/studio/web/assets/{yaml-BiUfxPbC.js → yaml-Bf92gJpd.js} +1 -1
- package/studio/web/index.html +1 -1
- package/upgrade/directives/5-3-0.ts +132 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/OperationFunctionCaller.ts +2 -1
- package/utility/common_utils.ts +30 -5
- package/utility/componentNames.ts +12 -0
- package/utility/environment/environmentManager.ts +3 -7
- package/utility/environment/systemInformation.ts +7 -0
- package/utility/errors/commonErrors.ts +4 -0
- package/utility/errors/hdbError.ts +29 -0
- package/utility/hdbTerms.ts +17 -0
- package/utility/logging/harper_logger.ts +87 -14
- package/utility/logging/logRotator.ts +72 -45
- package/utility/nodeIdentity.ts +45 -0
- package/utility/npmUtilities.ts +10 -8
- package/utility/operationPermissions.ts +18 -1
- package/utility/operation_authorization.ts +231 -42
- package/utility/watchPath.ts +63 -0
- package/utility/watcherFallback.ts +270 -0
- package/validation/configValidator.ts +209 -72
- package/validation/installValidator.ts +15 -0
- package/validation/validationWrapper.ts +18 -4
package/dist/resources/Table.js
CHANGED
|
@@ -41,7 +41,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
41
41
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
42
42
|
};
|
|
43
43
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
44
|
-
exports.EVICTED = exports.INVALIDATED = void 0;
|
|
44
|
+
exports.EVICTED = exports.INVALIDATED = exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT = exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT = void 0;
|
|
45
|
+
exports.acquireUpdateAttributesLock = acquireUpdateAttributesLock;
|
|
46
|
+
exports.releaseUpdateAttributesLock = releaseUpdateAttributesLock;
|
|
47
|
+
exports.withUpdateAttributesLock = withUpdateAttributesLock;
|
|
45
48
|
exports.ignoreAlreadyDropped = ignoreAlreadyDropped;
|
|
46
49
|
exports.freezeRecord = freezeRecord;
|
|
47
50
|
exports.frozenRecordView = frozenRecordView;
|
|
@@ -50,6 +53,7 @@ exports.coerceType = coerceType;
|
|
|
50
53
|
const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
|
|
51
54
|
const node_vm_1 = require("node:vm");
|
|
52
55
|
const node_crypto_1 = require("node:crypto");
|
|
56
|
+
const node_perf_hooks_1 = require("node:perf_hooks");
|
|
53
57
|
const commonUtility_ts_1 = require("../utility/lmdb/commonUtility.js");
|
|
54
58
|
const nodeIdMapping_ts_1 = require("./nodeIdMapping.js");
|
|
55
59
|
const lodash_1 = __importDefault(require("lodash"));
|
|
@@ -58,6 +62,7 @@ const lmdbProcessRows_js_1 = __importDefault(require("../dataLayer/harperBridge/
|
|
|
58
62
|
const Resource_ts_1 = require("./Resource.js");
|
|
59
63
|
const when_ts_1 = require("../utility/when.js");
|
|
60
64
|
const DatabaseTransaction_ts_1 = require("./DatabaseTransaction.js");
|
|
65
|
+
const recordLock_ts_1 = require("./recordLock.js");
|
|
61
66
|
const envMngr = __importStar(require("../utility/environment/environmentManager.js"));
|
|
62
67
|
const transactionBroadcast_ts_1 = require("./transactionBroadcast.js");
|
|
63
68
|
const hdbError_ts_1 = require("../utility/errors/hdbError.js");
|
|
@@ -93,7 +98,10 @@ const { validateAttribute } = lmdbProcessRows_js_1.default;
|
|
|
93
98
|
const NULL_WITH_TIMESTAMP = new Uint8Array(9);
|
|
94
99
|
NULL_WITH_TIMESTAMP[8] = 0xc0; // null
|
|
95
100
|
const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification
|
|
101
|
+
const MAX_DATE_TIMESTAMP = 8.64e15;
|
|
96
102
|
const RECORD_PRUNING_INTERVAL = 60000; // one minute
|
|
103
|
+
const MAX_CONCURRENT_HISTORY_REMOVALS = 10;
|
|
104
|
+
const MAX_CONCURRENT_LMDB_HISTORY_REMOVALS = 1000;
|
|
97
105
|
// RocksDB-only: number of eviction/tombstone removals coalesced into a single transaction commit.
|
|
98
106
|
// Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already
|
|
99
107
|
// coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path.
|
|
@@ -102,9 +110,75 @@ const EVICTION_BATCH_SIZE = 100;
|
|
|
102
110
|
// letting an unbounded number of open transactions (and their snapshots) accumulate.
|
|
103
111
|
const MAX_INFLIGHT_EVICTION_BATCHES = 4;
|
|
104
112
|
const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
|
|
113
|
+
// Guardrails for `Prefer: count=exact`: once the requested page has been collected, counting the rest
|
|
114
|
+
// of the match set is bounded by BOTH a row cap and a wall-clock budget, so a paginated read can't turn
|
|
115
|
+
// into an unbounded scan. Exceeding either reports an unknown total (Content-Range `.../*`) rather than
|
|
116
|
+
// truncating the page. These bound the count tail, not the page itself; a genuinely expensive query
|
|
117
|
+
// (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure.
|
|
118
|
+
const MAX_EXACT_COUNT_SCAN = 1_000_000;
|
|
119
|
+
const MAX_EXACT_COUNT_MS = 1_000;
|
|
120
|
+
// Largest page a `Prefer: count=` request will materialize. A request whose limit exceeds this (or is
|
|
121
|
+
// not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal
|
|
122
|
+
// streaming path with no count, so a count request can't be coerced into buffering an unbounded page.
|
|
123
|
+
const MAX_COUNT_PAGE = 10_000;
|
|
124
|
+
// How often the exact-count drain yields to the macrotask queue (must be a power of two for the bit-mask
|
|
125
|
+
// check). Keeps a large scan from monopolizing the event loop without adding a yield per row.
|
|
126
|
+
const COUNT_YIELD_INTERVAL = 2_048;
|
|
105
127
|
envMngr.initSync();
|
|
106
128
|
const LMDB_PREFETCH_WRITES = envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
|
|
107
129
|
const LOCK_TIMEOUT = 10000;
|
|
130
|
+
// This bounds schema-lock acquisition; LOCK_TIMEOUT bounds in-flight record writes during a drop.
|
|
131
|
+
exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT = 10000;
|
|
132
|
+
const UPDATE_ATTRIBUTES_LOCK = 'update-attributes';
|
|
133
|
+
// Contention is otherwise only visible once it becomes a timeout (harper#2251).
|
|
134
|
+
exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT = 1000;
|
|
135
|
+
// raw ASCII bytes are ordered-binary's encoding of the string, so this addresses the same native
|
|
136
|
+
// lock as string-keyed tryLock/unlock calls
|
|
137
|
+
const updateAttributesLockKey = Buffer.from(UPDATE_ATTRIBUTES_LOCK);
|
|
138
|
+
const lockWait = new Int32Array(new SharedArrayBuffer(4));
|
|
139
|
+
/** The wait blocks the event loop, so the locked section must stay synchronous. */
|
|
140
|
+
function acquireUpdateAttributesLock(rootStore, scopeDescription, timeout = exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT) {
|
|
141
|
+
if (rootStore.tryLock(updateAttributesLockKey))
|
|
142
|
+
return;
|
|
143
|
+
const startTime = node_perf_hooks_1.performance.now();
|
|
144
|
+
let waitTime = 1;
|
|
145
|
+
while (!rootStore.tryLock(updateAttributesLockKey)) {
|
|
146
|
+
const elapsed = node_perf_hooks_1.performance.now() - startTime;
|
|
147
|
+
if (elapsed >= timeout) {
|
|
148
|
+
throw new hdbError_ts_1.UpdateAttributesLockTimeoutError(`Timed out after ${Math.round(elapsed)}ms waiting for the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription}; the lock holder did not release it before the deadline, so this schema/attribute update cannot proceed`);
|
|
149
|
+
}
|
|
150
|
+
if (elapsed >= 2) {
|
|
151
|
+
Atomics.wait(lockWait, 0, 0, Math.min(waitTime, timeout - elapsed));
|
|
152
|
+
if (waitTime < 16)
|
|
153
|
+
waitTime *= 2;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const waited = node_perf_hooks_1.performance.now() - startTime;
|
|
157
|
+
// The caller cannot register its release until we return, so a throw here would leak the lock
|
|
158
|
+
// with no `finally` able to reach it.
|
|
159
|
+
if (waited >= exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT)
|
|
160
|
+
try {
|
|
161
|
+
logger_ts_1.logger.warn?.(`Acquired the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription} after waiting ${Math.round(waited)}ms; this worker's event loop was blocked for that wait, and a holder that runs past ${exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT}ms fails the update outright`);
|
|
162
|
+
}
|
|
163
|
+
catch { }
|
|
164
|
+
}
|
|
165
|
+
function releaseUpdateAttributesLock(rootStore) {
|
|
166
|
+
rootStore.unlock(updateAttributesLockKey);
|
|
167
|
+
}
|
|
168
|
+
function withUpdateAttributesLock(rootStore, scopeDescription, callback) {
|
|
169
|
+
acquireUpdateAttributesLock(rootStore, scopeDescription);
|
|
170
|
+
try {
|
|
171
|
+
const result = callback();
|
|
172
|
+
if (typeof result?.then === 'function') {
|
|
173
|
+
Promise.resolve(result).catch((error) => logger_ts_1.logger.error?.(`Async update-attributes callback rejected after its lock was released (${scopeDescription})`, error));
|
|
174
|
+
throw new TypeError(`withUpdateAttributesLock callback must be synchronous (${scopeDescription}); asynchronous work may continue after the lock is released`);
|
|
175
|
+
}
|
|
176
|
+
return result;
|
|
177
|
+
}
|
|
178
|
+
finally {
|
|
179
|
+
releaseUpdateAttributesLock(rootStore);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
108
182
|
// Tolerate a redundant column family drop. Drops are broadcast to every worker
|
|
109
183
|
// thread and each holds its own handle to the same underlying family, so a
|
|
110
184
|
// concurrent worker may already have dropped it; the storage engine reports
|
|
@@ -335,8 +409,24 @@ function detectCyclicEnumerable(start) {
|
|
|
335
409
|
function chainKeyForId(id) {
|
|
336
410
|
return typeof id === 'string' ? 's' + id : 'k' + (0, DatabaseTransaction_ts_1.writeKeyId)(id);
|
|
337
411
|
}
|
|
412
|
+
/** Normalizes a passed `context` argument as `transactional()` does; undefined means fall back to ambient. */
|
|
413
|
+
function contextArgument(context) {
|
|
414
|
+
if (!context || (0, DatabaseTransaction_ts_1.isReleasedTransaction)(context))
|
|
415
|
+
return undefined;
|
|
416
|
+
const resolved = context.getContext?.() || context;
|
|
417
|
+
return resolved instanceof DatabaseTransaction_ts_1.DatabaseTransaction ? { transaction: resolved } : resolved;
|
|
418
|
+
}
|
|
419
|
+
/** Distinguishes bare lock options from a record target (id, URL, {id:...}). */
|
|
420
|
+
function isPlainOptions(value) {
|
|
421
|
+
return (typeof value === 'object' &&
|
|
422
|
+
value !== null &&
|
|
423
|
+
!Array.isArray(value) &&
|
|
424
|
+
!(value instanceof URLSearchParams) &&
|
|
425
|
+
value.id === undefined);
|
|
426
|
+
}
|
|
338
427
|
function makeTable(options) {
|
|
339
|
-
|
|
428
|
+
var _a;
|
|
429
|
+
const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, isBranch, } = options;
|
|
340
430
|
let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
|
|
341
431
|
evictionMs ??= 0;
|
|
342
432
|
// Eviction without explicit expiration means expiration:0. Apply at construction so
|
|
@@ -350,6 +440,7 @@ function makeTable(options) {
|
|
|
350
440
|
properties = (0, jsonSchemaTypes_ts_1.projectAttributesToProperties)(attributes);
|
|
351
441
|
const updateRecord = (0, RecordEncoder_ts_1.recordUpdater)(primaryStore, tableId, auditStore);
|
|
352
442
|
let warnedNullSourcePut = false; // latched: one warn per table per worker (see _writeUpdate)
|
|
443
|
+
let warnedFutureSourceVersion = false; // likewise (see getFromSource)
|
|
353
444
|
let sourceLoad; // if a source has a load function (replicator), record it here
|
|
354
445
|
let hasSourceGet;
|
|
355
446
|
let primaryKeyAttribute;
|
|
@@ -383,6 +474,10 @@ function makeTable(options) {
|
|
|
383
474
|
let cleanupPriority = 0;
|
|
384
475
|
let lastCleanupInterval;
|
|
385
476
|
let cleanupTimer;
|
|
477
|
+
let recordExpirationInterval;
|
|
478
|
+
// a reclamation pass awaits a scheduled cleanup, which only settles from its timer
|
|
479
|
+
const pendingCleanupResolvers = new Set();
|
|
480
|
+
let disposed = false;
|
|
386
481
|
// true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup
|
|
387
482
|
let expirationScanScheduled = false;
|
|
388
483
|
// set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table
|
|
@@ -431,10 +526,11 @@ function makeTable(options) {
|
|
|
431
526
|
const MAX_PREFETCH_BUNDLE = 6;
|
|
432
527
|
if (audit)
|
|
433
528
|
addDeleteRemoval();
|
|
434
|
-
|
|
529
|
+
const reclamationHandler = (priority) => {
|
|
435
530
|
if (hasSourceGet)
|
|
436
531
|
return scheduleCleanup(priority);
|
|
437
|
-
}
|
|
532
|
+
};
|
|
533
|
+
(0, storageReclamation_ts_1.onStorageReclamation)(primaryStore.path, reclamationHandler);
|
|
438
534
|
class Updatable extends tracked_ts_1.GenericTrackedObject {
|
|
439
535
|
getUpdatedTime() {
|
|
440
536
|
return RecordEncoder_ts_1.entryMap.get(this.getRecord())?.version;
|
|
@@ -533,12 +629,68 @@ function makeTable(options) {
|
|
|
533
629
|
},
|
|
534
630
|
});
|
|
535
631
|
}
|
|
632
|
+
function resolveAuditHead(id, version, nodeId, refs) {
|
|
633
|
+
if (!refs?.length)
|
|
634
|
+
return { txnLogKey: version, nodeId };
|
|
635
|
+
const visited = new Set();
|
|
636
|
+
function findHead(candidateRefs) {
|
|
637
|
+
if (!candidateRefs)
|
|
638
|
+
return;
|
|
639
|
+
const pending = candidateRefs.slice().reverse();
|
|
640
|
+
while (pending.length > 0) {
|
|
641
|
+
const ref = pending.pop();
|
|
642
|
+
const identity = `${ref.nodeId ?? 0}:${ref.version}`;
|
|
643
|
+
if (visited.has(identity))
|
|
644
|
+
continue;
|
|
645
|
+
visited.add(identity);
|
|
646
|
+
const entry = auditStore.getSync(ref.version, tableId, id, ref.nodeId);
|
|
647
|
+
if (!entry)
|
|
648
|
+
continue;
|
|
649
|
+
if (entry.version === version && (nodeId == null || (entry.nodeId ?? 0) === nodeId))
|
|
650
|
+
return { txnLogKey: ref.version, nodeId: ref.nodeId };
|
|
651
|
+
const previousRefs = entry.previousAdditionalAuditRefs;
|
|
652
|
+
if (previousRefs) {
|
|
653
|
+
for (let index = previousRefs.length - 1; index >= 0; index--)
|
|
654
|
+
pending.push(previousRefs[index]);
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
const referencedHead = findHead(refs);
|
|
659
|
+
if (referencedHead)
|
|
660
|
+
return referencedHead;
|
|
661
|
+
if (version != null) {
|
|
662
|
+
const directHead = auditStore.getSync(version, tableId, id, nodeId);
|
|
663
|
+
if (directHead?.version === version && (nodeId == null || (directHead.nodeId ?? 0) === nodeId))
|
|
664
|
+
return { txnLogKey: version, nodeId };
|
|
665
|
+
}
|
|
666
|
+
return { txnLogKey: version, nodeId };
|
|
667
|
+
}
|
|
536
668
|
class TableResource extends Resource_ts_1.Resource {
|
|
537
669
|
#record; // the stored/frozen record from the database and stored in the cache (should not be modified directly)
|
|
538
670
|
#changes; // the changes to the record that have been made (should not be modified directly)
|
|
539
671
|
#version; // version of the record
|
|
540
672
|
#entry; // the entry from the database
|
|
541
673
|
#savingOperation; // operation for the record is currently being saved
|
|
674
|
+
#lockHandle; // the record lock acquired by lock() — scoped or hold
|
|
675
|
+
#lockWritable; // set by #reloadLocked to let save() stage lock-writable updates
|
|
676
|
+
/**
|
|
677
|
+
* Shared guard: if this instance is lock-writable but the handle is gone (expired or
|
|
678
|
+
* released), throw 409 before staging any write. Covers update/invalidate/relocate/delete
|
|
679
|
+
* in addition to the save() path. Every lock-writable instance carries its own handle in
|
|
680
|
+
* #lockHandle (scoped and hold alike), so we never need to search the registry here.
|
|
681
|
+
*/
|
|
682
|
+
#assertLiveHandle(id) {
|
|
683
|
+
if (!this.#lockWritable)
|
|
684
|
+
return;
|
|
685
|
+
const handle = this.#lockHandle;
|
|
686
|
+
// Off-key writes through the same resource instance are ordinary; only guard the
|
|
687
|
+
// exact key the lock was acquired for.
|
|
688
|
+
if (handle.keyId !== (0, DatabaseTransaction_ts_1.writeKeyId)(id))
|
|
689
|
+
return;
|
|
690
|
+
if (handle.expired || handle.released) {
|
|
691
|
+
throw (0, recordLock_ts_1.lockNotHeldError)(handle);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
542
694
|
// #section: static-config
|
|
543
695
|
static name = tableName; // for display/debugging purposes
|
|
544
696
|
static primaryStore = primaryStore;
|
|
@@ -635,7 +787,7 @@ function makeTable(options) {
|
|
|
635
787
|
// perform the write of an individual write event
|
|
636
788
|
const writeUpdate = async (event, context) => {
|
|
637
789
|
const value = event.value;
|
|
638
|
-
const Table = event.table ? databases_ts_1.databases[databaseName][event.table] :
|
|
790
|
+
const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : _a;
|
|
639
791
|
if (databaseName === hdbTerms_ts_1.SYSTEM_SCHEMA_NAME &&
|
|
640
792
|
(event.table === hdbTerms_ts_1.SYSTEM_TABLE_NAMES.ROLE_TABLE_NAME || event.table === hdbTerms_ts_1.SYSTEM_TABLE_NAMES.USER_TABLE_NAME)) {
|
|
641
793
|
userRoleUpdate = true;
|
|
@@ -652,6 +804,9 @@ function makeTable(options) {
|
|
|
652
804
|
ensureLoaded: false,
|
|
653
805
|
nodeId: event.nodeId,
|
|
654
806
|
viaNodeId: event.viaNodeId,
|
|
807
|
+
// the origin's record version, stored as-is so every replica holds the version the
|
|
808
|
+
// origin holds; the transaction's own timestamp stays the origin's log key
|
|
809
|
+
version: event.version,
|
|
655
810
|
// use per-event expiresAt: batched txn context only holds the first event's expiration
|
|
656
811
|
expiresAt: event.expiresAt,
|
|
657
812
|
// bulk base-copy snapshot frame: apply current-state directly, without an audit/transaction-log
|
|
@@ -690,7 +845,7 @@ function makeTable(options) {
|
|
|
690
845
|
const stageWrite = (event, context) => {
|
|
691
846
|
let chainKey;
|
|
692
847
|
try {
|
|
693
|
-
const Table = event.table ? databases_ts_1.databases[databaseName][event.table] :
|
|
848
|
+
const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : _a;
|
|
694
849
|
const id = event.id ?? (event.value ? event.value[Table?.primaryKey] : undefined);
|
|
695
850
|
if (id != null && typeof id !== 'symbol')
|
|
696
851
|
chainKey = `${event.table ?? tableName} ${chainKeyForId(id)}`;
|
|
@@ -890,7 +1045,9 @@ function makeTable(options) {
|
|
|
890
1045
|
continue;
|
|
891
1046
|
}
|
|
892
1047
|
}
|
|
893
|
-
//
|
|
1048
|
+
// A source that reports no log position of its own (no `timestamp`) has only one clock,
|
|
1049
|
+
// so its record version doubles as the apply transaction's timestamp. A replication
|
|
1050
|
+
// receiver always sets `timestamp` from the origin's log key and never reaches this.
|
|
894
1051
|
if (!event.timestamp && event.version)
|
|
895
1052
|
event.timestamp = event.version;
|
|
896
1053
|
const commitResolution = (0, transaction_ts_1.transaction)(event, () => {
|
|
@@ -991,7 +1148,7 @@ function makeTable(options) {
|
|
|
991
1148
|
/** Indicates if the events should be revalidated when they are received. By default we do this if the get
|
|
992
1149
|
* method is overriden */
|
|
993
1150
|
static get shouldRevalidateEvents() {
|
|
994
|
-
return this.prototype.get !==
|
|
1151
|
+
return this.prototype.get !== _a.prototype.get;
|
|
995
1152
|
}
|
|
996
1153
|
/**
|
|
997
1154
|
* Gets a resource instance, as defined by the Resource class, adding the table-specific handling
|
|
@@ -1027,7 +1184,7 @@ function makeTable(options) {
|
|
|
1027
1184
|
}
|
|
1028
1185
|
return loadLocalRecord(id, request, { transaction: readTxn, ensureLoaded: resourceOptions?.ensureLoaded }, sync, (entry) => {
|
|
1029
1186
|
if (entry) {
|
|
1030
|
-
|
|
1187
|
+
_a._updateResource(this, entry);
|
|
1031
1188
|
}
|
|
1032
1189
|
else
|
|
1033
1190
|
this.#record = null;
|
|
@@ -1045,7 +1202,7 @@ function makeTable(options) {
|
|
|
1045
1202
|
if (loadingFromSource) {
|
|
1046
1203
|
txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
|
|
1047
1204
|
return (0, when_ts_1.when)(loadingFromSource, (entry) => {
|
|
1048
|
-
|
|
1205
|
+
_a._updateResource(this, entry);
|
|
1049
1206
|
return this;
|
|
1050
1207
|
});
|
|
1051
1208
|
}
|
|
@@ -1265,7 +1422,7 @@ function makeTable(options) {
|
|
|
1265
1422
|
return dbisDb.getSync([Symbol.for('residency_by_id'), id]);
|
|
1266
1423
|
}
|
|
1267
1424
|
static setResidency(getResidency) {
|
|
1268
|
-
|
|
1425
|
+
_a.getResidency =
|
|
1269
1426
|
getResidency &&
|
|
1270
1427
|
((record, context) => {
|
|
1271
1428
|
try {
|
|
@@ -1278,7 +1435,7 @@ function makeTable(options) {
|
|
|
1278
1435
|
});
|
|
1279
1436
|
}
|
|
1280
1437
|
static setResidencyById(getResidencyById) {
|
|
1281
|
-
|
|
1438
|
+
_a.getResidencyById =
|
|
1282
1439
|
getResidencyById &&
|
|
1283
1440
|
((id) => {
|
|
1284
1441
|
try {
|
|
@@ -1291,8 +1448,8 @@ function makeTable(options) {
|
|
|
1291
1448
|
});
|
|
1292
1449
|
}
|
|
1293
1450
|
static getResidency(record, context) {
|
|
1294
|
-
if (
|
|
1295
|
-
return
|
|
1451
|
+
if (_a.getResidencyById) {
|
|
1452
|
+
return _a.getResidencyById(record[primaryKey]);
|
|
1296
1453
|
}
|
|
1297
1454
|
let count = replicateToCount;
|
|
1298
1455
|
if (context.replicateTo != undefined) {
|
|
@@ -1334,7 +1491,7 @@ function makeTable(options) {
|
|
|
1334
1491
|
return; // already enabled
|
|
1335
1492
|
audit = true;
|
|
1336
1493
|
addDeleteRemoval();
|
|
1337
|
-
|
|
1494
|
+
_a.audit = true;
|
|
1338
1495
|
}
|
|
1339
1496
|
/**
|
|
1340
1497
|
* Coerce the id as a string to the correct type for the primary key
|
|
@@ -1346,7 +1503,24 @@ function makeTable(options) {
|
|
|
1346
1503
|
return null;
|
|
1347
1504
|
return coerceType(id, primaryKeyAttribute);
|
|
1348
1505
|
}
|
|
1506
|
+
/**
|
|
1507
|
+
* A branch's Table classes deliberately carry the BASE's logical database name so an
|
|
1508
|
+
* application's schema and code resolve unchanged (harper#643). That makes every schema
|
|
1509
|
+
* mutation resolve against the global catalog — a `dropTable()` through a branch would delete
|
|
1510
|
+
* the live base table. Reads and writes are per-branch and unaffected; DDL is refused until a
|
|
1511
|
+
* branch owns a schema identity of its own.
|
|
1512
|
+
*/
|
|
1513
|
+
static assertSchemaMutable(operation) {
|
|
1514
|
+
if (!isBranch)
|
|
1515
|
+
return;
|
|
1516
|
+
const error = new Error(`Cannot ${operation} through a branched database: '${tableName}' resolves to the schema of base ` +
|
|
1517
|
+
`database '${databaseName}', so the change would apply to the base rather than the branch`);
|
|
1518
|
+
error.statusCode = 400;
|
|
1519
|
+
throw error;
|
|
1520
|
+
}
|
|
1349
1521
|
static async dropTable() {
|
|
1522
|
+
_a.assertSchemaMutable('drop a table');
|
|
1523
|
+
const rootStore = primaryStore.rootStore;
|
|
1350
1524
|
if (databaseName === databasePath) {
|
|
1351
1525
|
// Persist a drop tombstone on the primary catalog entry BEFORE any
|
|
1352
1526
|
// destructive work. If the process dies or a column family drop fails
|
|
@@ -1354,9 +1528,11 @@ function makeTable(options) {
|
|
|
1354
1528
|
// the next startup (or a same-name create) completes the drop via
|
|
1355
1529
|
// completeInterruptedDrop in databases.ts instead of resurrecting
|
|
1356
1530
|
// the table.
|
|
1357
|
-
const primaryCatalogKey =
|
|
1358
|
-
const
|
|
1359
|
-
|
|
1531
|
+
const primaryCatalogKey = _a.tableName + '/';
|
|
1532
|
+
const writeTombstone = () => {
|
|
1533
|
+
const primaryMeta = dbisDb.getSync(primaryCatalogKey);
|
|
1534
|
+
if (!primaryMeta || primaryMeta.dropping)
|
|
1535
|
+
return;
|
|
1360
1536
|
primaryMeta.dropping = true;
|
|
1361
1537
|
// Stamps this drop's identity so the interrupted-drop retry budget in
|
|
1362
1538
|
// databases.ts can be scoped to THIS drop rather than the table name: a
|
|
@@ -1366,11 +1542,23 @@ function makeTable(options) {
|
|
|
1366
1542
|
// the budget by generation instead makes the new drop's tombstone carry
|
|
1367
1543
|
// its own fresh key regardless of what any worker last observed.
|
|
1368
1544
|
primaryMeta.dropGeneration = (0, node_crypto_1.randomUUID)();
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1545
|
+
return dbisDb.put(primaryCatalogKey, primaryMeta);
|
|
1546
|
+
};
|
|
1547
|
+
if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
|
|
1548
|
+
// withUpdateAttributesLock's locked section cannot be held across an await, so a durable
|
|
1549
|
+
// tombstone depends on put being rebound to putSync for RocksDB primary stores (see
|
|
1550
|
+
// createOpenDBIObject). Check that BEFORE writing anything: a tombstone left behind by a
|
|
1551
|
+
// refused drop would delete the table on the next load.
|
|
1552
|
+
if (dbisDb.put !== dbisDb.putSync)
|
|
1553
|
+
throw new Error(`Cannot drop ${databaseName}.${_a.tableName}: the catalog store's put is asynchronous, so the drop tombstone cannot be made durable before the column families are dropped`);
|
|
1554
|
+
withUpdateAttributesLock(rootStore, `drop table '${databaseName}.${_a.tableName}'`, writeTombstone);
|
|
1555
|
+
}
|
|
1556
|
+
else {
|
|
1557
|
+
let tombstoneWrite;
|
|
1558
|
+
rootStore.transactionSync(() => {
|
|
1559
|
+
tombstoneWrite = writeTombstone();
|
|
1560
|
+
});
|
|
1561
|
+
if (typeof tombstoneWrite?.then === 'function')
|
|
1374
1562
|
await tombstoneWrite;
|
|
1375
1563
|
}
|
|
1376
1564
|
}
|
|
@@ -1441,28 +1629,25 @@ function makeTable(options) {
|
|
|
1441
1629
|
// same-name create completes the interrupted drop and writes fresh
|
|
1442
1630
|
// catalog rows, and clobbering those would orphan the new table.
|
|
1443
1631
|
const removeTombstonedCatalog = () => {
|
|
1444
|
-
const currentPrimary = dbisDb.getSync(
|
|
1632
|
+
const currentPrimary = dbisDb.getSync(_a.tableName + '/');
|
|
1445
1633
|
if (!currentPrimary?.dropping)
|
|
1446
1634
|
return false;
|
|
1447
1635
|
for (const attribute of attributes) {
|
|
1448
|
-
dbisDb.remove(
|
|
1636
|
+
dbisDb.remove(_a.tableName + '/' + attribute.name);
|
|
1449
1637
|
}
|
|
1450
|
-
dbisDb.remove(
|
|
1638
|
+
dbisDb.remove(_a.tableName + '/');
|
|
1451
1639
|
return true;
|
|
1452
1640
|
};
|
|
1453
|
-
const rootStore = primaryStore.rootStore;
|
|
1454
1641
|
if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
|
|
1455
1642
|
// Serialize the drops + catalog removal against a concurrent
|
|
1456
1643
|
// same-name create (and completeInterruptedDrop) under the database's
|
|
1457
1644
|
// 'update-attributes' exclusive lock - the same lock the create path
|
|
1458
|
-
// holds. It is a synchronous
|
|
1645
|
+
// holds. It is a synchronous lock wait that blocks the event loop, so
|
|
1459
1646
|
// the locked section MUST stay synchronous: drop with dropSync (as
|
|
1460
1647
|
// completeInterruptedDrop does), never an awaited drop(), or a
|
|
1461
|
-
// concurrent create's
|
|
1462
|
-
//
|
|
1463
|
-
|
|
1464
|
-
let removed = false;
|
|
1465
|
-
try {
|
|
1648
|
+
// concurrent create's wait would be stuck on a drop that the blocked
|
|
1649
|
+
// event loop can never resolve, burning its full deadline before failing.
|
|
1650
|
+
const removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
|
|
1466
1651
|
for (const attribute of attributes) {
|
|
1467
1652
|
const index = indices[attribute.name];
|
|
1468
1653
|
if (index)
|
|
@@ -1479,11 +1664,8 @@ function makeTable(options) {
|
|
|
1479
1664
|
catch (error) {
|
|
1480
1665
|
ignoreAlreadyDropped(error);
|
|
1481
1666
|
}
|
|
1482
|
-
|
|
1483
|
-
}
|
|
1484
|
-
finally {
|
|
1485
|
-
rootStore.unlock('update-attributes');
|
|
1486
|
-
}
|
|
1667
|
+
return removeTombstonedCatalog();
|
|
1668
|
+
});
|
|
1487
1669
|
if (removed)
|
|
1488
1670
|
await dbisDb.committed;
|
|
1489
1671
|
}
|
|
@@ -1540,7 +1722,7 @@ function makeTable(options) {
|
|
|
1540
1722
|
estimatedRecordRange: undefined,
|
|
1541
1723
|
};
|
|
1542
1724
|
if (this.getContext()?.includeExpensiveRecordCountEstimates) {
|
|
1543
|
-
return
|
|
1725
|
+
return _a.getRecordCount().then((recordCount) => {
|
|
1544
1726
|
description.recordCount = recordCount.recordCount;
|
|
1545
1727
|
description.estimatedRecordRange = recordCount.estimatedRange;
|
|
1546
1728
|
return description;
|
|
@@ -1824,14 +2006,71 @@ function makeTable(options) {
|
|
|
1824
2006
|
* Save any changes into this instance to the current transaction
|
|
1825
2007
|
*/
|
|
1826
2008
|
save() {
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
2009
|
+
this.#assertLiveHandle(this.getId()); // a write through a released or expired lock never lands
|
|
2010
|
+
const operation = this.#savingOperation;
|
|
2011
|
+
if ((!operation || operation.dropped) && this.#lockWritable && this.#lockHandle?.hold) {
|
|
2012
|
+
// A held lock's record stages its update here rather than at lock() time: it is often
|
|
2013
|
+
// written after the acquiring transaction has already completed, which would have
|
|
2014
|
+
// dropped an update staged then. Nothing set means nothing to stage — a held-but-untouched
|
|
2015
|
+
// id stays untouched. Scoped locks do not take this branch: #reloadLocked stages their
|
|
2016
|
+
// TransactionWrite at lock() time (exactly like update()), so #savingOperation is always
|
|
2017
|
+
// set for a live scoped lock and the ordinary path below applies.
|
|
2018
|
+
// Verify the hold is still alive: if #lockWritable is set but the handle expired or was
|
|
2019
|
+
// released between lock acquisition and this save(), throw 409 rather than silently
|
|
2020
|
+
// committing stale data. Every lock-writable instance carries its own handle.
|
|
2021
|
+
const saveHandle = this.#lockHandle;
|
|
2022
|
+
if (saveHandle.expired || saveHandle.released) {
|
|
2023
|
+
throw (0, recordLock_ts_1.lockNotHeldError)(saveHandle);
|
|
2024
|
+
}
|
|
2025
|
+
const changes = this.#changes;
|
|
2026
|
+
if (changes && Object.keys(changes).length > 0) {
|
|
2027
|
+
this.#savingOperation = null;
|
|
2028
|
+
return (0, when_ts_1.when)(this._writeUpdate(this.getId(), changes, false), () => {
|
|
2029
|
+
const op = this.#savingOperation;
|
|
2030
|
+
if (op?.dropped) {
|
|
2031
|
+
this.#changes = undefined;
|
|
2032
|
+
return;
|
|
2033
|
+
}
|
|
2034
|
+
// Clear #savingOperation so the next sequential save() enters the lock-writable
|
|
2035
|
+
// path and creates a fresh write (otherwise a non-null #savingOperation makes
|
|
2036
|
+
// save() take the #saveOperation branch with an already-committed write, which
|
|
2037
|
+
// is a no-op, silently dropping the new change).
|
|
2038
|
+
// op.innerCommit is the real native-transaction commit Promise set on the
|
|
2039
|
+
// immediateCommit path in DatabaseTransaction.save(); await it to ensure
|
|
2040
|
+
// durability before resolving to the caller.
|
|
2041
|
+
if (op?.saved) {
|
|
2042
|
+
this.#savingOperation = null;
|
|
2043
|
+
return op?.innerCommit;
|
|
2044
|
+
}
|
|
2045
|
+
// op.saved = false means addWrite deferred the save; #saveOperation commits it
|
|
2046
|
+
// synchronously but ImmediateTransaction.save() returns undefined while the
|
|
2047
|
+
// inner rocksdb commit is still pending — return innerCommit so the caller
|
|
2048
|
+
// actually waits for durability.
|
|
2049
|
+
return (0, when_ts_1.when)(this.save(), () => op?.innerCommit);
|
|
2050
|
+
});
|
|
1830
2051
|
}
|
|
1831
|
-
|
|
2052
|
+
// No changes: nothing to stage. A dropped operation (detached at a scoped→hold
|
|
2053
|
+
// upgrade — see detachScopedUpgradeWrite) must not fall through to the ordinary
|
|
2054
|
+
// #saveOperation path below with its now-detached reference.
|
|
2055
|
+
if (!operation || operation.dropped) {
|
|
1832
2056
|
this.#savingOperation = null;
|
|
2057
|
+
return;
|
|
1833
2058
|
}
|
|
1834
2059
|
}
|
|
2060
|
+
if (this.#savingOperation) {
|
|
2061
|
+
const operation = this.#savingOperation;
|
|
2062
|
+
this.#savingOperation = null;
|
|
2063
|
+
// A write that lands via a nested immediateCommit (e.g. a second sequential save() on
|
|
2064
|
+
// the same ImmediateTransaction context, once the first has already closed it) sets
|
|
2065
|
+
// operation.innerCommit to the real native-commit promise, but the commit() sweep loop
|
|
2066
|
+
// that triggers it discards its own return value — #saveOperation()'s result can
|
|
2067
|
+
// resolve before that native commit actually settles. Chain on innerCommit (as the
|
|
2068
|
+
// lock-writable hold branch above already does) so callers awaiting save() see the
|
|
2069
|
+
// write durably land, not just the outer (possibly premature) resolution.
|
|
2070
|
+
const result = this.#saveOperation(operation);
|
|
2071
|
+
const innerCommit = operation.innerCommit;
|
|
2072
|
+
return innerCommit ? (0, when_ts_1.when)(innerCommit, () => result) : result;
|
|
2073
|
+
}
|
|
1835
2074
|
}
|
|
1836
2075
|
#saveOperation(operation) {
|
|
1837
2076
|
const transaction = txnForContext(this.getContext());
|
|
@@ -1909,6 +2148,7 @@ function makeTable(options) {
|
|
|
1909
2148
|
});
|
|
1910
2149
|
}
|
|
1911
2150
|
_writeInvalidate(id, partialRecord, options) {
|
|
2151
|
+
this.#assertLiveHandle(id);
|
|
1912
2152
|
const context = this.getContext();
|
|
1913
2153
|
checkValidId(id);
|
|
1914
2154
|
const transaction = txnForContext(this.getContext());
|
|
@@ -1917,9 +2157,12 @@ function makeTable(options) {
|
|
|
1917
2157
|
store: primaryStore,
|
|
1918
2158
|
invalidated: true,
|
|
1919
2159
|
entry: this.#entry,
|
|
2160
|
+
recordVersion: options?.version,
|
|
2161
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
|
|
1920
2162
|
commit: (txnTime, existingEntry, _retry, transaction) => {
|
|
2163
|
+
const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
1921
2164
|
write.skipped = false; // reset on each retry; cleanup happens after commit if still true
|
|
1922
|
-
if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId)
|
|
2165
|
+
if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
|
|
1923
2166
|
write.skipped = true;
|
|
1924
2167
|
return;
|
|
1925
2168
|
}
|
|
@@ -1940,7 +2183,13 @@ function makeTable(options) {
|
|
|
1940
2183
|
viaNodeId: options?.viaNodeId,
|
|
1941
2184
|
transaction,
|
|
1942
2185
|
tableToTrack: tableName,
|
|
2186
|
+
recordVersion: txnTime,
|
|
2187
|
+
additionalAuditRefs: isRocksDB && audit && txnLogKey !== txnTime
|
|
2188
|
+
? [{ version: txnLogKey, nodeId: options?.nodeId }]
|
|
2189
|
+
: undefined,
|
|
1943
2190
|
}, 'invalidate');
|
|
2191
|
+
if (write.trackRecordVersion)
|
|
2192
|
+
write.recordVersionApplied = true;
|
|
1944
2193
|
// TODO: recordDeletion?
|
|
1945
2194
|
},
|
|
1946
2195
|
};
|
|
@@ -1948,21 +2197,25 @@ function makeTable(options) {
|
|
|
1948
2197
|
transaction.addWrite(write);
|
|
1949
2198
|
}
|
|
1950
2199
|
_writeRelocate(id, options) {
|
|
2200
|
+
this.#assertLiveHandle(id);
|
|
1951
2201
|
const context = this.getContext();
|
|
1952
2202
|
checkValidId(id);
|
|
1953
2203
|
const transaction = txnForContext(this.getContext());
|
|
1954
|
-
|
|
2204
|
+
const write = {
|
|
1955
2205
|
key: id,
|
|
1956
2206
|
store: primaryStore,
|
|
1957
2207
|
invalidated: true,
|
|
1958
2208
|
entry: this.#entry,
|
|
2209
|
+
recordVersion: options?.version,
|
|
2210
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
|
|
1959
2211
|
before: this.constructor.source?.relocate && !context?.source
|
|
1960
2212
|
? this.constructor.source.relocate.bind(this.constructor.source, id, undefined, context)
|
|
1961
2213
|
: undefined,
|
|
1962
2214
|
commit: (txnTime, existingEntry, _retry, transaction) => {
|
|
1963
|
-
|
|
2215
|
+
const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
2216
|
+
if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0)
|
|
1964
2217
|
return;
|
|
1965
|
-
const residency =
|
|
2218
|
+
const residency = _a.getResidencyRecord(options.residencyId);
|
|
1966
2219
|
let metadata = 0;
|
|
1967
2220
|
let newRecord = null;
|
|
1968
2221
|
const existingRecord = existingEntry?.value;
|
|
@@ -1986,9 +2239,16 @@ function makeTable(options) {
|
|
|
1986
2239
|
viaNodeId: options?.viaNodeId,
|
|
1987
2240
|
expiresAt: options.expiresAt,
|
|
1988
2241
|
transaction,
|
|
2242
|
+
recordVersion: txnTime,
|
|
2243
|
+
additionalAuditRefs: isRocksDB && audit && txnLogKey !== txnTime
|
|
2244
|
+
? [{ version: txnLogKey, nodeId: options?.nodeId }]
|
|
2245
|
+
: undefined,
|
|
1989
2246
|
}, 'relocate', false, null);
|
|
2247
|
+
if (write.trackRecordVersion)
|
|
2248
|
+
write.recordVersionApplied = true;
|
|
1990
2249
|
},
|
|
1991
|
-
}
|
|
2250
|
+
};
|
|
2251
|
+
transaction.addWrite(write);
|
|
1992
2252
|
}
|
|
1993
2253
|
/**
|
|
1994
2254
|
* Record the relocation of an entry (when a record is moved to a different node), return true if it is now located locally
|
|
@@ -2107,10 +2367,253 @@ function makeTable(options) {
|
|
|
2107
2367
|
}
|
|
2108
2368
|
}
|
|
2109
2369
|
/**
|
|
2110
|
-
*
|
|
2370
|
+
* Static entry point: `Table.lock(id, options?, context?)` — creates an instance in the given,
|
|
2371
|
+
* ambient, or a fresh context and delegates to the instance lock(). This shadows Resource.static
|
|
2372
|
+
* lock so that both callers share the same transaction link (required for cross-instance upgrade
|
|
2373
|
+
* detection). lock() is an in-process API with no authorization hook of its own; it is not
|
|
2374
|
+
* protocol-dispatched, so no allowUpdate/allowCreate check runs on acquisition.
|
|
2375
|
+
*
|
|
2376
|
+
* Dropping the trailing `context` leaks the key: the bare `{}` fallback is an
|
|
2377
|
+
* ImmediateTransaction, which releases no record locks.
|
|
2378
|
+
*/
|
|
2379
|
+
static async lock(target, options, context) {
|
|
2380
|
+
if (!isRocksDB)
|
|
2381
|
+
throw new hdbError_ts_1.ClientError('Record locks are not supported on LMDB', 501);
|
|
2382
|
+
if (options === undefined && isPlainOptions(target)) {
|
|
2383
|
+
options = target;
|
|
2384
|
+
target = undefined;
|
|
2385
|
+
}
|
|
2386
|
+
const id = target != null ? requestTargetToId(target) : null;
|
|
2387
|
+
const resolvedContext = contextArgument(context) ?? transaction_ts_1.contextStorage.getStore() ?? {};
|
|
2388
|
+
const resource = new _a(id, resolvedContext);
|
|
2389
|
+
return resource.lock(target, options);
|
|
2390
|
+
}
|
|
2391
|
+
/**
|
|
2392
|
+
* Acquire an exclusive lock on this record (or on `target`'s) and return it ready for updates
|
|
2393
|
+
* (harper#483, Phase 0: exclusive across every worker thread of this node). The lock is held
|
|
2394
|
+
* in process memory only — no durable writes. Phase 0 contract: lock() is mutually exclusive
|
|
2395
|
+
* with other lock() calls on the same key; plain writes (put/patch/delete/create) are never
|
|
2396
|
+
* gated or blocked. The generation expires after `lease` if it is never released.
|
|
2397
|
+
*
|
|
2398
|
+
* Transaction-scoped (default): write through the returned record (or the table's static verbs
|
|
2399
|
+
* in the same transaction), and the commit or abort releases it. `{ hold: true }`: the lock
|
|
2400
|
+
* outlives the transaction; write through the returned record and release with `unlock()`, or
|
|
2401
|
+
* let the lease expire.
|
|
2402
|
+
*/
|
|
2403
|
+
// async so option/id validation rejects rather than throwing past a caller's `.catch()`; the
|
|
2404
|
+
// body still runs to completion synchronously, which is what keeps concurrent lock() calls
|
|
2405
|
+
// on one key coalescing instead of racing to tryLock.
|
|
2406
|
+
async lock(target, options) {
|
|
2407
|
+
if (!isRocksDB)
|
|
2408
|
+
throw new hdbError_ts_1.ClientError('Record locks are not supported on LMDB', 501);
|
|
2409
|
+
if (options === undefined && isPlainOptions(target)) {
|
|
2410
|
+
options = target;
|
|
2411
|
+
target = undefined;
|
|
2412
|
+
}
|
|
2413
|
+
const id = target != null ? requestTargetToId(target) : this.getId();
|
|
2414
|
+
checkValidId(id);
|
|
2415
|
+
const resolved = (0, recordLock_ts_1.resolveLockOptions)(options);
|
|
2416
|
+
const context = this.getContext();
|
|
2417
|
+
const link = txnForContext(context);
|
|
2418
|
+
const keyId = (0, DatabaseTransaction_ts_1.writeKeyId)(id);
|
|
2419
|
+
const held = this.#lockHandle;
|
|
2420
|
+
if (held && !held.released && !held.expired && held.keyId === keyId) {
|
|
2421
|
+
// Re-entrant: upgrade to hold if requested, then preserve staged changes.
|
|
2422
|
+
if (resolved.hold && !held.hold) {
|
|
2423
|
+
held.upgradeToHold(resolved.lease);
|
|
2424
|
+
// The scoped phase eagerly staged a TransactionWrite (see #reloadLocked); hold
|
|
2425
|
+
// staging is deferred and explicit-save-only, so an unsaved scoped write left
|
|
2426
|
+
// dangling here would otherwise auto-commit at the transaction sweep and clobber
|
|
2427
|
+
// whatever the hold write lands. detachScopedUpgradeWrite marks it .dropped so a
|
|
2428
|
+
// later save() on this instance falls through to the hold branch instead of the
|
|
2429
|
+
// dead #savingOperation reference.
|
|
2430
|
+
detachScopedUpgradeWrite(link, keyId, held);
|
|
2431
|
+
}
|
|
2432
|
+
return Promise.resolve(this.#reloadLocked(id, undefined, true));
|
|
2433
|
+
}
|
|
2434
|
+
const scoped = link.recordLockFor(primaryStore, keyId);
|
|
2435
|
+
if (scoped && !scoped.released && !scoped.expired) {
|
|
2436
|
+
if (resolved.hold && !scoped.hold) {
|
|
2437
|
+
// Upgrade scoped → hold: flip the existing handle object to hold mode so every
|
|
2438
|
+
// instance that already references this handle stays valid. Retiring and creating a
|
|
2439
|
+
// new handle would invalidate those other references (their save() would then throw
|
|
2440
|
+
// 409 against a released handle). The native key stays locked throughout.
|
|
2441
|
+
scoped.upgradeToHold(resolved.lease);
|
|
2442
|
+
detachScopedUpgradeWrite(link, keyId, scoped);
|
|
2443
|
+
return Promise.resolve(this.#reloadLocked(id, scoped, true));
|
|
2444
|
+
}
|
|
2445
|
+
// Already held with the same type: re-entrant return. Preserve any staged changes.
|
|
2446
|
+
return Promise.resolve(this.#reloadLocked(id, scoped, true));
|
|
2447
|
+
}
|
|
2448
|
+
const key = (0, recordLock_ts_1.lockAttemptKey)(tableId, id);
|
|
2449
|
+
// Coalesce concurrent lock() calls for the same key inside one link so they don't
|
|
2450
|
+
// self-block: Promise.all([T.lock(id), T.lock(id)]) would otherwise have both calls
|
|
2451
|
+
// reach tryLock before either registers, making the second park against the first.
|
|
2452
|
+
const pending = link.pendingLockFor(primaryStore, keyId);
|
|
2453
|
+
if (pending) {
|
|
2454
|
+
// Wait for the in-flight acquisition, then take the re-entrant path as if
|
|
2455
|
+
// recordLockFor had found it. If the first attempt timed out, re-enter so
|
|
2456
|
+
// the second caller gets its own timeout.
|
|
2457
|
+
// The follower waits on the leader's acquisition, but only for its own timeout.
|
|
2458
|
+
let followerTimer;
|
|
2459
|
+
const followerTimedOut = Symbol('follower timeout');
|
|
2460
|
+
const followerStart = Date.now();
|
|
2461
|
+
const followerDeadline = new Promise((_, reject) => {
|
|
2462
|
+
followerTimer = setTimeout(() => reject(followerTimedOut), resolved.timeout).unref();
|
|
2463
|
+
});
|
|
2464
|
+
// Try again on this caller's own terms with the budget it has left.
|
|
2465
|
+
const retryOnRemainingBudget = () => {
|
|
2466
|
+
// The enclosing transaction ended while we were parked. A retry re-resolves the
|
|
2467
|
+
// context, which no longer points at this link, so the handle it acquired would be
|
|
2468
|
+
// registered on a fresh transaction that no commit or abort ever releases — the
|
|
2469
|
+
// same abandonment the leader's own post-acquisition guard below rejects.
|
|
2470
|
+
if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.CLOSED && !link.saveCommits)
|
|
2471
|
+
throw new hdbError_ts_1.ServerError('Transaction was closed while waiting for a record lock', 500);
|
|
2472
|
+
const remaining = resolved.timeout - (Date.now() - followerStart);
|
|
2473
|
+
if (remaining <= 0)
|
|
2474
|
+
throw new hdbError_ts_1.ClientError(`Record is locked and was not released in time`, 423);
|
|
2475
|
+
return this.lock(target, { ...resolved, timeout: remaining });
|
|
2476
|
+
};
|
|
2477
|
+
return Promise.race([pending, followerDeadline]).then(() => {
|
|
2478
|
+
clearTimeout(followerTimer);
|
|
2479
|
+
const acquired = link.recordLockFor(primaryStore, keyId);
|
|
2480
|
+
if (acquired && !acquired.released && !acquired.expired) {
|
|
2481
|
+
if (resolved.hold && !acquired.hold) {
|
|
2482
|
+
detachScopedUpgradeWrite(link, keyId, acquired);
|
|
2483
|
+
acquired.upgradeToHold(resolved.lease);
|
|
2484
|
+
}
|
|
2485
|
+
return this.#reloadLocked(id, acquired, true);
|
|
2486
|
+
}
|
|
2487
|
+
return retryOnRemainingBudget();
|
|
2488
|
+
}, (error) => {
|
|
2489
|
+
clearTimeout(followerTimer);
|
|
2490
|
+
if (error === followerTimedOut)
|
|
2491
|
+
throw new hdbError_ts_1.ClientError(`Record is locked and was not released in time`, 423);
|
|
2492
|
+
return retryOnRemainingBudget();
|
|
2493
|
+
});
|
|
2494
|
+
}
|
|
2495
|
+
const pendingPromise = (0, recordLock_ts_1.acquireRecordKey)(link, primaryStore, key, keyId, resolved.timeout, resolved.lease, resolved.hold);
|
|
2496
|
+
link.registerPendingLock(primaryStore, keyId, pendingPromise);
|
|
2497
|
+
return pendingPromise.then((handle) => {
|
|
2498
|
+
link.unregisterPendingLock(primaryStore, keyId);
|
|
2499
|
+
if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.CLOSED && !link.saveCommits) {
|
|
2500
|
+
// The transaction was aborted while this call waited; nothing would ever release the handle.
|
|
2501
|
+
handle.release();
|
|
2502
|
+
throw new hdbError_ts_1.ServerError('Transaction was closed while waiting for a record lock', 500);
|
|
2503
|
+
}
|
|
2504
|
+
link.registerRecordLock(handle);
|
|
2505
|
+
if (link.saveCommits && context?.timestamp)
|
|
2506
|
+
handle.noteCandidateFloor(context.timestamp);
|
|
2507
|
+
if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.OPEN && !link.saveCommits) {
|
|
2508
|
+
// Explicit transaction() (not ImmediateTransaction): pin the clock to
|
|
2509
|
+
// acquiredAt when no writes have been staged yet. When writes already
|
|
2510
|
+
// exist, leave the clock alone (ordering is best-effort; write held records
|
|
2511
|
+
// in their own transaction for the guarantee). ImmediateTransaction is
|
|
2512
|
+
// excluded (saveCommits=true) — its clock is never pinned in lock();
|
|
2513
|
+
// each save() stamps from the handle's committed version floor instead.
|
|
2514
|
+
if (link.writes.length === 0 && !link.timestamp) {
|
|
2515
|
+
link.timestamp = handle.acquiredAt;
|
|
2516
|
+
}
|
|
2517
|
+
if (!resolved.hold && link.transaction) {
|
|
2518
|
+
// Scoped lock: the read snapshot may predate the lock; drop it so the
|
|
2519
|
+
// scope reads what it locked. Hold locks use acquiredAt directly and
|
|
2520
|
+
// do not update the read snapshot.
|
|
2521
|
+
// The timestamp guard matches DatabaseTransaction's own setTimestamp calls: a
|
|
2522
|
+
// deferred update() write leaves the clock at 0, which rocksdb-js rejects.
|
|
2523
|
+
if (link.writes.length === 0 && link.readTxnsUsed <= 1) {
|
|
2524
|
+
link.releaseReadTxn();
|
|
2525
|
+
link.snapshotFree = true;
|
|
2526
|
+
}
|
|
2527
|
+
else if (link.timestamp)
|
|
2528
|
+
link.transaction.setTimestamp(link.timestamp);
|
|
2529
|
+
}
|
|
2530
|
+
}
|
|
2531
|
+
// ImmediateTransaction: no clock pinning in lock(); save() stamps each write
|
|
2532
|
+
// from the committed handle floor for both scoped and hold handles.
|
|
2533
|
+
return this.#reloadLocked(id, handle);
|
|
2534
|
+
}, (err) => {
|
|
2535
|
+
link.unregisterPendingLock(primaryStore, keyId);
|
|
2536
|
+
throw err;
|
|
2537
|
+
});
|
|
2538
|
+
}
|
|
2539
|
+
#reloadLocked(id, holdHandle, preserveChanges = false) {
|
|
2540
|
+
// For freshness, read the committed entry (snapshot-free) so a hold lock sees concurrent
|
|
2541
|
+
// committed writes rather than a stale snapshot. A write earlier in THIS explicit
|
|
2542
|
+
// transaction has not landed in that committed entry yet (harper#1968: Harper defers an
|
|
2543
|
+
// explicit transaction's writes until the writing call actually runs them), so pull the
|
|
2544
|
+
// current value the same way a chained write picks up its basis (priorStagedWrite): the
|
|
2545
|
+
// record comes from the prior staged write, the rest of the entry (version, audit chain,
|
|
2546
|
+
// blob metadata) stays the pre-transaction one.
|
|
2547
|
+
const link = txnForContext(this.getContext());
|
|
2548
|
+
let entryForReload = primaryStore.getEntry(id);
|
|
2549
|
+
if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.OPEN) {
|
|
2550
|
+
const keyId = (0, DatabaseTransaction_ts_1.writeKeyId)(id);
|
|
2551
|
+
const tailWrite = link.writesByKey?.get(primaryStore)?.get(keyId);
|
|
2552
|
+
const priorStaged = tailWrite && (tailWrite.stagedEntry !== undefined ? tailWrite : (0, DatabaseTransaction_ts_1.priorStagedWrite)(tailWrite));
|
|
2553
|
+
if (priorStaged?.stagedEntry !== undefined) {
|
|
2554
|
+
entryForReload = entryForReload
|
|
2555
|
+
? { ...entryForReload, value: priorStaged.stagedEntry.value }
|
|
2556
|
+
: { value: priorStaged.stagedEntry.value };
|
|
2557
|
+
if (entryForReload.value && typeof entryForReload.value === 'object') {
|
|
2558
|
+
// Register the merged entry in entryMap so getUpdatedTime() works.
|
|
2559
|
+
RecordEncoder_ts_1.entryMap.set(entryForReload.value, entryForReload);
|
|
2560
|
+
}
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
if ((0, DatabaseTransaction_ts_1.writeKeyId)(id) !== (0, DatabaseTransaction_ts_1.writeKeyId)(this.getId())) {
|
|
2564
|
+
// lock(target) where target differs from this record: return a separate instance.
|
|
2565
|
+
const fresh = new this.constructor(id, this.getContext());
|
|
2566
|
+
_a._updateResource(fresh, entryForReload);
|
|
2567
|
+
if (holdHandle != null) {
|
|
2568
|
+
fresh.#lockHandle = holdHandle;
|
|
2569
|
+
// Do not clear this.#lockHandle: the original instance keeps its own lock on its
|
|
2570
|
+
// own id; the fresh instance owns the lock on the target id independently.
|
|
2571
|
+
}
|
|
2572
|
+
fresh.#lockWritable = true;
|
|
2573
|
+
// Scoped (not hold) stages exactly like update(): create the TransactionWrite now so
|
|
2574
|
+
// save() is the ordinary #savingOperation path. Hold keeps deferred staging (the
|
|
2575
|
+
// acquiring transaction may commit before the holder ever writes).
|
|
2576
|
+
if (!fresh.#lockHandle.hold)
|
|
2577
|
+
fresh._writeUpdate(id, fresh.#changes, false);
|
|
2578
|
+
return fresh;
|
|
2579
|
+
}
|
|
2580
|
+
// Store the handle for both scoped and hold locks; undefined (re-entrant hold fast-path)
|
|
2581
|
+
// must not clear a handle already set.
|
|
2582
|
+
if (holdHandle != null)
|
|
2583
|
+
this.#lockHandle = holdHandle;
|
|
2584
|
+
_a._updateResource(this, entryForReload);
|
|
2585
|
+
// Preserve staged changes when upgrading the same instance from scoped to hold so that
|
|
2586
|
+
// set() calls made under the scoped lock survive the reload.
|
|
2587
|
+
if (!preserveChanges)
|
|
2588
|
+
this.#changes = undefined;
|
|
2589
|
+
this.#lockWritable = true;
|
|
2590
|
+
// Scoped (not hold): stage now, same as update() would. Skip if a write from an earlier
|
|
2591
|
+
// lock() cycle on this instance is still pending (re-entrant call before its save()).
|
|
2592
|
+
if (!this.#lockHandle.hold && !this.#savingOperation)
|
|
2593
|
+
this._writeUpdate(id, this.#changes, false);
|
|
2594
|
+
return this;
|
|
2595
|
+
}
|
|
2596
|
+
/**
|
|
2597
|
+
* Release the lock this instance holds. Resolves true when this call cleared the native key lock.
|
|
2598
|
+
* Works for both held (`{ hold: true }`) and transaction-scoped locks. After unlock() the
|
|
2599
|
+
* instance is no longer lock-writable; writes through it require a fresh lock.
|
|
2111
2600
|
*/
|
|
2112
|
-
|
|
2113
|
-
|
|
2601
|
+
unlock() {
|
|
2602
|
+
// Always clear the local lock-writable state so subsequent writes on this instance are
|
|
2603
|
+
// ungated, regardless of whether the handle was already released.
|
|
2604
|
+
const handle = this.#lockHandle;
|
|
2605
|
+
this.#lockHandle = undefined;
|
|
2606
|
+
this.#lockWritable = false;
|
|
2607
|
+
if (!handle || handle.released)
|
|
2608
|
+
return Promise.resolve(false);
|
|
2609
|
+
const link = txnForContext(this.getContext());
|
|
2610
|
+
// A scoped lock staged its write at lock() time; released before commit, that write must not
|
|
2611
|
+
// run into the released-handle guard at the sweep.
|
|
2612
|
+
if (this.#savingOperation && !this.#savingOperation.saved && this.#savingOperation.lockHandle === handle)
|
|
2613
|
+
this.#savingOperation = null;
|
|
2614
|
+
detachScopedUpgradeWrite(link, (0, DatabaseTransaction_ts_1.writeKeyId)(this.getId()), handle);
|
|
2615
|
+
link.unregisterRecordLock(handle);
|
|
2616
|
+
return Promise.resolve(handle.release());
|
|
2114
2617
|
}
|
|
2115
2618
|
static operation(operation, context) {
|
|
2116
2619
|
operation.table ||= tableName;
|
|
@@ -2226,8 +2729,10 @@ function makeTable(options) {
|
|
|
2226
2729
|
// a notification that a write has already occurred in the canonical data source, we need to update our
|
|
2227
2730
|
// local copy
|
|
2228
2731
|
_writeUpdate(id, recordUpdate, fullUpdate, options) {
|
|
2732
|
+
this.#assertLiveHandle(id);
|
|
2229
2733
|
const context = this.getContext();
|
|
2230
2734
|
const transaction = txnForContext(context);
|
|
2735
|
+
const replaying = transaction.isReplay === true;
|
|
2231
2736
|
checkValidId(id);
|
|
2232
2737
|
if (fullUpdate && recordUpdate == null && options?.isNotification) {
|
|
2233
2738
|
// A source/replication-applied put must carry the record; these applies skip record
|
|
@@ -2270,6 +2775,13 @@ function makeTable(options) {
|
|
|
2270
2775
|
nodeName: context?.nodeName,
|
|
2271
2776
|
fullUpdate,
|
|
2272
2777
|
deferSave: true,
|
|
2778
|
+
// the origin's record version on an applied write; absent for a locally-originated one
|
|
2779
|
+
recordVersion: options?.version,
|
|
2780
|
+
// Include the lock handle (if any) so the expired-handle guard in
|
|
2781
|
+
// DatabaseTransaction.save() can throw 409 when the lease has lapsed.
|
|
2782
|
+
// Only attach the hold handle when it covers exactly this key; off-key writes
|
|
2783
|
+
// are ordinary and must not carry an unrelated hold's handle.
|
|
2784
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
|
|
2273
2785
|
validate: (txnTime, committedBy = transaction) => {
|
|
2274
2786
|
if (!recordUpdate)
|
|
2275
2787
|
recordUpdate = this.#changes;
|
|
@@ -2394,6 +2906,7 @@ function makeTable(options) {
|
|
|
2394
2906
|
this.#savingOperation = null;
|
|
2395
2907
|
write.stagedIn = undefined; // nothing may pin this write's transaction past its commit
|
|
2396
2908
|
let omitLocalRecord = false;
|
|
2909
|
+
const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
2397
2910
|
// we use optimistic locking to only commit if the existing record state still holds true.
|
|
2398
2911
|
// this is superior to using an async transaction since it doesn't require JS execution
|
|
2399
2912
|
// during the write transaction.
|
|
@@ -2454,8 +2967,8 @@ function makeTable(options) {
|
|
|
2454
2967
|
// best-effort keyed lookup in the capped block below — see #1148. precedesExistingVersion(...)
|
|
2455
2968
|
// === 0 is the identity tie: same version AND same node (the local node is id 0, so an undefined
|
|
2456
2969
|
// options?.nodeId resolves to the same 0 the ref stored).
|
|
2457
|
-
if (existingEntry.additionalAuditRefs?.some((ref) => ref.version ===
|
|
2458
|
-
precedesExistingVersion(txnTime, { version: txnTime, localTime:
|
|
2970
|
+
if (existingEntry.additionalAuditRefs?.some((ref) => ref.version === txnLogKey &&
|
|
2971
|
+
precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: ref.nodeId }, options?.nodeId) === 0)) {
|
|
2459
2972
|
write.skipped = true;
|
|
2460
2973
|
return; // out-of-order write already folded into this record
|
|
2461
2974
|
}
|
|
@@ -2483,10 +2996,10 @@ function makeTable(options) {
|
|
|
2483
2996
|
if (!oldestRetainedAuditTimeResolved) {
|
|
2484
2997
|
oldestRetainedAuditTimeResolved = true;
|
|
2485
2998
|
// getRange yields ascending by audit-log key, so the first entry is the oldest retained.
|
|
2486
|
-
// Mirror replicationConnection's retention check and the cleanup key basis (
|
|
2487
|
-
//
|
|
2999
|
+
// Mirror replicationConnection's retention check and the cleanup key basis (`txnLogKey`).
|
|
3000
|
+
// Fall back to the nominal time-based purge floor when the log is empty/unavailable.
|
|
2488
3001
|
for (const entry of auditStore.getRange({ start: 1, log: options?.nodeId })) {
|
|
2489
|
-
oldestRetainedAuditTime = entry.
|
|
3002
|
+
oldestRetainedAuditTime = entry.txnLogKey;
|
|
2490
3003
|
break;
|
|
2491
3004
|
}
|
|
2492
3005
|
oldestRetainedAuditTime ??= Date.now() - auditStore_ts_1.auditRetention;
|
|
@@ -2499,43 +3012,67 @@ function makeTable(options) {
|
|
|
2499
3012
|
// depth-cap block. This is the same keyed lookup that block performs, hoisted ahead of the walk.
|
|
2500
3013
|
// It is what catches transitive/proxied re-deliveries: they arrive buried below the record head
|
|
2501
3014
|
// (so replication's head-tie fast-skip can't see them) yet are exact duplicates. Keyed by nodeId,
|
|
2502
|
-
// so it is correct across multiple source nodes.
|
|
2503
|
-
//
|
|
2504
|
-
//
|
|
3015
|
+
// so it is correct across multiple source nodes. The lookup key is this write's LOG key, not its
|
|
3016
|
+
// record version — a replication apply commits under the origin's log key while storing the
|
|
3017
|
+
// origin's version, and only the log key addresses the entry (harper#2412).
|
|
3018
|
+
// RocksDB-only: LMDB audit entries are keyed by local audit time, so this lookup doesn't apply
|
|
3019
|
+
// there (LMDB keeps the exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
|
|
2505
3020
|
// simply falls through to the walk, so this never changes correctness; the additionalAuditRefs
|
|
2506
3021
|
// check above remains the read-your-writes guard. Never when this write staged in a prior
|
|
2507
3022
|
// failed attempt: that attempt already appended this write's own audit entry, so the lookup
|
|
2508
3023
|
// would find it and skip the write as "already applied" when the record was never committed.
|
|
2509
3024
|
// A recommit of the same transaction survived that skip only because the old write batch
|
|
2510
3025
|
// still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
|
|
2511
|
-
if (isRocksDB && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(
|
|
2512
|
-
const priorAudit = auditStore.get(
|
|
3026
|
+
if (isRocksDB && !replaying && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnLogKey)) {
|
|
3027
|
+
const priorAudit = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
|
|
2513
3028
|
if (priorAudit &&
|
|
2514
|
-
priorAudit.
|
|
2515
|
-
precedesExistingVersion(txnTime, { version: txnTime, localTime:
|
|
3029
|
+
priorAudit.txnLogKey === txnLogKey &&
|
|
3030
|
+
precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: priorAudit.nodeId }, options?.nodeId) === 0) {
|
|
2516
3031
|
write.skipped = true;
|
|
2517
3032
|
return; // duplicate already applied; avoid the resequencing walk
|
|
2518
3033
|
}
|
|
2519
3034
|
}
|
|
2520
3035
|
// incremental CRDT updates are only available with audit logging on
|
|
2521
|
-
|
|
3036
|
+
const initialAuditHead = isRocksDB
|
|
3037
|
+
? resolveAuditHead(id, existingEntry.version, existingEntry.nodeId, existingEntry.additionalAuditRefs)
|
|
3038
|
+
: { txnLogKey: existingEntry.localTime, nodeId: existingEntry.nodeId };
|
|
3039
|
+
let localTime = initialAuditHead.txnLogKey;
|
|
2522
3040
|
let auditedVersion = existingEntry.version;
|
|
2523
3041
|
logger_ts_1.logger.debug?.('Applying CRDT update to record with id: ', id, 'txn time', new Date(txnTime), 'applying later update from:', new Date(auditedVersion), 'local recorded time', new Date(localTime));
|
|
2524
|
-
let nodeId =
|
|
3042
|
+
let nodeId = initialAuditHead.nodeId;
|
|
2525
3043
|
const succeedingUpdates = []; // record the "future" updates, as we need to apply the updates in reverse order
|
|
2526
3044
|
const auditRefsToVisit = existingEntry.additionalAuditRefs
|
|
2527
3045
|
? existingEntry.additionalAuditRefs.map((ref) => ({ localTime: ref.version, nodeId: ref.nodeId }))
|
|
2528
3046
|
: [];
|
|
2529
|
-
//
|
|
3047
|
+
// Out-of-order merges retain every existing branch head; per-origin log keys are not globally ordered.
|
|
2530
3048
|
if (existingEntry.additionalAuditRefs) {
|
|
2531
3049
|
for (const ref of existingEntry.additionalAuditRefs) {
|
|
2532
|
-
|
|
2533
|
-
additionalAuditRefs.push(ref);
|
|
2534
|
-
}
|
|
3050
|
+
additionalAuditRefs.push(ref);
|
|
2535
3051
|
}
|
|
2536
3052
|
}
|
|
2537
3053
|
let addedAuditRef = false;
|
|
2538
3054
|
let nextRef;
|
|
3055
|
+
const visitedAuditRefs = new Set();
|
|
3056
|
+
const queuePreviousAuditRefs = (auditRecord) => {
|
|
3057
|
+
const previousRefs = auditRecord.previousAdditionalAuditRefs;
|
|
3058
|
+
if (previousRefs) {
|
|
3059
|
+
for (const ref of previousRefs) {
|
|
3060
|
+
auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
|
|
3061
|
+
logger_ts_1.logger.debug?.('Adding audit ref from audit record to visit queue', {
|
|
3062
|
+
version: ref.version,
|
|
3063
|
+
nodeId: ref.nodeId,
|
|
3064
|
+
});
|
|
3065
|
+
}
|
|
3066
|
+
}
|
|
3067
|
+
};
|
|
3068
|
+
const advanceToPreviousAudit = (auditRecord) => {
|
|
3069
|
+
const previousRefs = auditRecord.previousAdditionalAuditRefs;
|
|
3070
|
+
const previousHead = isRocksDB && previousRefs?.length
|
|
3071
|
+
? resolveAuditHead(id, auditRecord.previousVersion, auditRecord.previousNodeId, previousRefs)
|
|
3072
|
+
: { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
|
|
3073
|
+
localTime = previousHead.txnLogKey;
|
|
3074
|
+
nodeId = previousHead.nodeId;
|
|
3075
|
+
};
|
|
2539
3076
|
let walkSteps = 0;
|
|
2540
3077
|
let auditWalkCapped = false;
|
|
2541
3078
|
// Early-out residual: as we walk the chain newest-first, fold each succeeding patch into a
|
|
@@ -2554,17 +3091,21 @@ function makeTable(options) {
|
|
|
2554
3091
|
// appended this write's own audit entry, so the lookup would match it while the record was
|
|
2555
3092
|
// never committed (see the up-front keyed dedup above).
|
|
2556
3093
|
const isReDeliveredDuplicate = () => {
|
|
2557
|
-
if (stagedOwnAuditEntry)
|
|
3094
|
+
if (replaying || stagedOwnAuditEntry)
|
|
2558
3095
|
return false;
|
|
2559
|
-
if (!dedupVersionCouldBeRetained(
|
|
2560
|
-
return false; // pre-retention
|
|
2561
|
-
const duplicate = auditStore.get(
|
|
3096
|
+
if (!dedupVersionCouldBeRetained(txnLogKey))
|
|
3097
|
+
return false; // pre-retention log key — skip the end-of-log scan (best-effort; see above)
|
|
3098
|
+
const duplicate = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
|
|
2562
3099
|
return (duplicate &&
|
|
2563
|
-
duplicate.
|
|
2564
|
-
precedesExistingVersion(txnTime, { version: txnTime, localTime:
|
|
3100
|
+
duplicate.txnLogKey === txnLogKey &&
|
|
3101
|
+
precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: duplicate.nodeId }, options?.nodeId) === 0);
|
|
2565
3102
|
};
|
|
2566
3103
|
do {
|
|
2567
3104
|
while (localTime > txnTime || (auditedVersion >= txnTime && localTime > 0)) {
|
|
3105
|
+
const auditIdentity = `${nodeId ?? 0}:${localTime}`;
|
|
3106
|
+
if (visitedAuditRefs.has(auditIdentity))
|
|
3107
|
+
break;
|
|
3108
|
+
visitedAuditRefs.add(auditIdentity);
|
|
2568
3109
|
// Bound the walk only for RocksDB, where the OOM was observed (issue #1114): each step
|
|
2569
3110
|
// is a transaction-log range scan + msgpackr decode, and the per-node logs can be huge.
|
|
2570
3111
|
// LMDB audit entries are keyed by local audit time (not version), so the duplicate
|
|
@@ -2576,19 +3117,39 @@ function makeTable(options) {
|
|
|
2576
3117
|
const auditRecord = auditStore.get(localTime, tableId, id, nodeId);
|
|
2577
3118
|
if (!auditRecord)
|
|
2578
3119
|
break;
|
|
3120
|
+
queuePreviousAuditRefs(auditRecord);
|
|
3121
|
+
if (isRocksDB &&
|
|
3122
|
+
!replaying &&
|
|
3123
|
+
!stagedOwnAuditEntry &&
|
|
3124
|
+
localTime === txnLogKey &&
|
|
3125
|
+
precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: auditRecord.nodeId }, options?.nodeId) === 0) {
|
|
3126
|
+
write.skipped = true;
|
|
3127
|
+
return;
|
|
3128
|
+
}
|
|
2579
3129
|
auditedVersion = auditRecord.version;
|
|
2580
3130
|
if (auditedVersion >= txnTime) {
|
|
2581
3131
|
if (auditedVersion === txnTime) {
|
|
2582
3132
|
precedesExisting = precedesExistingVersion(txnTime, { version: auditedVersion, localTime: localTime, key: id, nodeId: auditRecord.nodeId }, options?.nodeId);
|
|
2583
3133
|
if (precedesExisting === 0) {
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
3134
|
+
if (isRocksDB && localTime !== txnLogKey) {
|
|
3135
|
+
// Same origin and record version, but a distinct write. Its per-origin log key
|
|
3136
|
+
// orders the otherwise non-unique record clock without comparing keys across origins.
|
|
3137
|
+
precedesExisting = txnLogKey > localTime ? 1 : -1;
|
|
3138
|
+
}
|
|
3139
|
+
else if (replaying || stagedOwnAuditEntry) {
|
|
3140
|
+
// The log entry being replayed (or staged by this write's failed attempt) is
|
|
3141
|
+
// the write itself, not proof that its primary-store mutation committed.
|
|
3142
|
+
precedesExisting = 1;
|
|
3143
|
+
}
|
|
3144
|
+
else {
|
|
3145
|
+
logger_ts_1.logger.debug?.('The transaction time and log key match the existing write, treating as duplicate', id);
|
|
3146
|
+
write.skipped = true;
|
|
3147
|
+
return;
|
|
3148
|
+
}
|
|
2587
3149
|
}
|
|
2588
3150
|
if (precedesExisting > 0) {
|
|
2589
3151
|
// if the existing version is older, we can skip this update
|
|
2590
|
-
|
|
2591
|
-
nodeId = auditRecord.previousNodeId;
|
|
3152
|
+
advanceToPreviousAudit(auditRecord);
|
|
2592
3153
|
continue;
|
|
2593
3154
|
}
|
|
2594
3155
|
}
|
|
@@ -2623,23 +3184,16 @@ function makeTable(options) {
|
|
|
2623
3184
|
}
|
|
2624
3185
|
if (!addedAuditRef && isRocksDB) {
|
|
2625
3186
|
addedAuditRef = true;
|
|
2626
|
-
// Add a reference to this older audit record if we had out-of-order writes
|
|
2627
|
-
|
|
3187
|
+
// Add a reference to this older audit record if we had out-of-order writes. The stored
|
|
3188
|
+
// value is a LOG key, not a record version: every consumer follows it straight into
|
|
3189
|
+
// `auditStore.get` (see the `auditRefsToVisit` mapping above and below), and on an
|
|
3190
|
+
// applied write those two clocks differ.
|
|
3191
|
+
additionalAuditRefs.push({ version: txnLogKey, nodeId: options?.nodeId });
|
|
2628
3192
|
logger_ts_1.logger.debug?.('Adding additional audit ref for out-of-order write', {
|
|
2629
|
-
|
|
3193
|
+
txnLogKey,
|
|
2630
3194
|
nodeId: options?.nodeId,
|
|
2631
3195
|
});
|
|
2632
3196
|
}
|
|
2633
|
-
// Collect any additional audit refs from this audit record to traverse other branches
|
|
2634
|
-
if (auditRecord.previousAdditionalAuditRefs) {
|
|
2635
|
-
for (const ref of auditRecord.previousAdditionalAuditRefs) {
|
|
2636
|
-
auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
|
|
2637
|
-
logger_ts_1.logger.debug?.('Adding audit ref from audit record to visit queue', {
|
|
2638
|
-
version: ref.version,
|
|
2639
|
-
nodeId: ref.nodeId,
|
|
2640
|
-
});
|
|
2641
|
-
}
|
|
2642
|
-
}
|
|
2643
3197
|
// Every field of this write is overwritten by newer writes, and there is no alternate
|
|
2644
3198
|
// audit branch left to scan, so it is fully superseded — the same outcome as walking to
|
|
2645
3199
|
// the end and taking the `writeCommit(false)` escape below, reached without paying the rest
|
|
@@ -2657,8 +3211,7 @@ function makeTable(options) {
|
|
|
2657
3211
|
}
|
|
2658
3212
|
return writeCommit(false);
|
|
2659
3213
|
}
|
|
2660
|
-
|
|
2661
|
-
nodeId = auditRecord.previousNodeId;
|
|
3214
|
+
advanceToPreviousAudit(auditRecord);
|
|
2662
3215
|
}
|
|
2663
3216
|
// Check if we need to scan additional audit refs from this record
|
|
2664
3217
|
if (auditWalkCapped)
|
|
@@ -2753,14 +3306,14 @@ function makeTable(options) {
|
|
|
2753
3306
|
throw new Error('Can not assign a record to a record, check for circular references');
|
|
2754
3307
|
if (residencyId == undefined) {
|
|
2755
3308
|
if (entry?.residencyId)
|
|
2756
|
-
context.previousResidency =
|
|
2757
|
-
const residency = residencyFromFunction(
|
|
3309
|
+
context.previousResidency = _a.getResidencyRecord(entry.residencyId);
|
|
3310
|
+
const residency = residencyFromFunction(_a.getResidency(recordToStore, context));
|
|
2758
3311
|
if (residency) {
|
|
2759
3312
|
if (!residency.includes(server.hostname)) {
|
|
2760
3313
|
// if we aren't in the residency list, specify that our local record should be omitted or be partial
|
|
2761
3314
|
auditRecordToStore ??= recordToStore;
|
|
2762
3315
|
omitLocalRecord = true;
|
|
2763
|
-
if (
|
|
3316
|
+
if (_a.getResidencyById) {
|
|
2764
3317
|
// complete omission of the record that doesn't belong here
|
|
2765
3318
|
recordToStore = undefined;
|
|
2766
3319
|
}
|
|
@@ -2830,7 +3383,17 @@ function makeTable(options) {
|
|
|
2830
3383
|
}
|
|
2831
3384
|
})());
|
|
2832
3385
|
updateIndices(id, existingRecord, recordToStore, transaction && { transaction });
|
|
3386
|
+
// Preserve an addressable audit head when the record and log clocks diverge.
|
|
3387
|
+
if (isRocksDB && audit && !isCopyApply && txnLogKey !== txnTime) {
|
|
3388
|
+
const headIndex = additionalAuditRefs.findIndex((ref) => ref.version === txnLogKey && (ref.nodeId ?? 0) === (options?.nodeId ?? 0));
|
|
3389
|
+
if (headIndex > 0)
|
|
3390
|
+
additionalAuditRefs.unshift(additionalAuditRefs.splice(headIndex, 1)[0]);
|
|
3391
|
+
else if (headIndex < 0)
|
|
3392
|
+
additionalAuditRefs.unshift({ version: txnLogKey, nodeId: options?.nodeId });
|
|
3393
|
+
}
|
|
2833
3394
|
writeCommit(true);
|
|
3395
|
+
if (write.trackRecordVersion)
|
|
3396
|
+
write.recordVersionApplied = true;
|
|
2834
3397
|
if (expiresAt >= 0) {
|
|
2835
3398
|
scheduleCleanup(); // arm for replicated writes too, not just local-context writes
|
|
2836
3399
|
// A runtime per-record expiresAt on a table with no table-level expiration/eviction, no expiresAt
|
|
@@ -2857,6 +3420,8 @@ function makeTable(options) {
|
|
|
2857
3420
|
user: context?.user,
|
|
2858
3421
|
residencyId,
|
|
2859
3422
|
expiresAt,
|
|
3423
|
+
recordVersion: txnTime,
|
|
3424
|
+
recordNodeId: precedesExisting < 0 ? existingEntry?.nodeId : options?.nodeId,
|
|
2860
3425
|
nodeId: options?.nodeId,
|
|
2861
3426
|
viaNodeId: options?.viaNodeId,
|
|
2862
3427
|
originatingOperation: context?.originatingOperation,
|
|
@@ -2894,7 +3459,7 @@ function makeTable(options) {
|
|
|
2894
3459
|
// calls the backend, and a tracked-instance mutation (update(id,{}); row.source=…;
|
|
2895
3460
|
// save()) that sets the source via accessors after update() won't re-embed. A
|
|
2896
3461
|
// resource-layer re-embed is the proper fix; tracked as a follow-up.
|
|
2897
|
-
const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(recordUpdate, context, options,
|
|
3462
|
+
const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(recordUpdate, context, options, _a.embedAttributes, _a.userEmbedders);
|
|
2898
3463
|
const proceed = () => {
|
|
2899
3464
|
// On a source/replication apply (`isNotification`), the record's already-saved blobs were
|
|
2900
3465
|
// received out-of-band for THIS write, so track them for skip/abort cleanup (harper-pro#406).
|
|
@@ -2950,6 +3515,7 @@ function makeTable(options) {
|
|
|
2950
3515
|
return Boolean(this.#record);
|
|
2951
3516
|
}
|
|
2952
3517
|
_writeDelete(id, options) {
|
|
3518
|
+
this.#assertLiveHandle(id);
|
|
2953
3519
|
const context = this.getContext();
|
|
2954
3520
|
const transaction = txnForContext(context);
|
|
2955
3521
|
checkValidId(id);
|
|
@@ -2960,6 +3526,8 @@ function makeTable(options) {
|
|
|
2960
3526
|
entry,
|
|
2961
3527
|
chainsStagedState: true,
|
|
2962
3528
|
nodeName: context?.nodeName,
|
|
3529
|
+
recordVersion: options?.version,
|
|
3530
|
+
lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
|
|
2963
3531
|
before: this.constructor.source?.delete && !context?.source
|
|
2964
3532
|
? this.constructor.source.delete.bind(this.constructor.source, id, undefined, context)
|
|
2965
3533
|
: undefined,
|
|
@@ -2971,10 +3539,11 @@ function makeTable(options) {
|
|
|
2971
3539
|
const priorStagedOp = (0, DatabaseTransaction_ts_1.priorStagedWrite)(write);
|
|
2972
3540
|
const priorStaged = priorStagedOp?.stagedEntry;
|
|
2973
3541
|
const existingRecord = priorStaged ? priorStaged.value : existingEntry?.value;
|
|
3542
|
+
const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
|
|
2974
3543
|
if (retry) {
|
|
2975
3544
|
if (context && existingEntry?.version > (context.lastModified || 0))
|
|
2976
3545
|
context.lastModified = existingEntry.version;
|
|
2977
|
-
|
|
3546
|
+
_a._updateResource(this, existingEntry);
|
|
2978
3547
|
}
|
|
2979
3548
|
// a strictly newer record exists locally, so this delete loses. An earlier write in this
|
|
2980
3549
|
// transaction can never trip this guard — it shares this transaction's timestamp and
|
|
@@ -2992,6 +3561,10 @@ function makeTable(options) {
|
|
|
2992
3561
|
viaNodeId: options?.viaNodeId,
|
|
2993
3562
|
transaction,
|
|
2994
3563
|
tableToTrack: tableName,
|
|
3564
|
+
recordVersion: txnTime,
|
|
3565
|
+
additionalAuditRefs: isRocksDB && audit && txnLogKey !== txnTime
|
|
3566
|
+
? [{ version: txnLogKey, nodeId: options?.nodeId }]
|
|
3567
|
+
: undefined,
|
|
2995
3568
|
}, 'delete');
|
|
2996
3569
|
if (!audit || isRocksDB)
|
|
2997
3570
|
scheduleCleanup();
|
|
@@ -3001,6 +3574,8 @@ function makeTable(options) {
|
|
|
3001
3574
|
(0, RecordEncoder_ts_1.removeEntry)(primaryStore, existingEntry, isRocksDB && transaction ? { transaction } : undefined);
|
|
3002
3575
|
}
|
|
3003
3576
|
write.stagedEntry = { value: undefined }; // the key holds no record for the rest of this transaction
|
|
3577
|
+
if (write.trackRecordVersion)
|
|
3578
|
+
write.recordVersionApplied = true;
|
|
3004
3579
|
// the removal supersedes the nearest record an earlier write in this transaction stored
|
|
3005
3580
|
// (older ones were already marked by their staged successors), so its saved blobs are
|
|
3006
3581
|
// cleaned up post-commit unless its audit entry references them
|
|
@@ -3161,7 +3736,6 @@ function makeTable(options) {
|
|
|
3161
3736
|
// objects. Entries are small and shallow; the clone is cheap next to the query.
|
|
3162
3737
|
conditions = cloneConditions(conditions);
|
|
3163
3738
|
let orderAlignedCondition;
|
|
3164
|
-
let syntheticOrderCondition;
|
|
3165
3739
|
const filtered = {};
|
|
3166
3740
|
function prepareConditions(conditions, operator) {
|
|
3167
3741
|
// some validation:
|
|
@@ -3260,7 +3834,7 @@ function makeTable(options) {
|
|
|
3260
3834
|
// Note, that we do allow users to disable condition re-ordering, in case they have knowledge of a preferred
|
|
3261
3835
|
// order for their query.
|
|
3262
3836
|
if (conditions.length > 1 && operator !== 'or')
|
|
3263
|
-
return sortBy(conditions, (0, search_ts_1.estimateCondition)(
|
|
3837
|
+
return sortBy(conditions, (0, search_ts_1.estimateCondition)(_a));
|
|
3264
3838
|
else
|
|
3265
3839
|
return conditions;
|
|
3266
3840
|
}
|
|
@@ -3301,7 +3875,7 @@ function makeTable(options) {
|
|
|
3301
3875
|
// if it is indexed, we add a pseudo-condition to align with the natural sort order of the index.
|
|
3302
3876
|
// the primary key has no secondary index, but the primary store is itself keyed in
|
|
3303
3877
|
// primary-key order, so scanning it is already aligned with the sort
|
|
3304
|
-
orderAlignedCondition =
|
|
3878
|
+
orderAlignedCondition = { ...sort, comparator: 'sort' };
|
|
3305
3879
|
conditions.push(orderAlignedCondition);
|
|
3306
3880
|
}
|
|
3307
3881
|
else if (conditions.length === 0 && !target.allowFullScan)
|
|
@@ -3328,15 +3902,17 @@ function makeTable(options) {
|
|
|
3328
3902
|
}
|
|
3329
3903
|
}
|
|
3330
3904
|
else {
|
|
3331
|
-
// if we had to add an aligned condition that isn't first, we remove it and do ordering later
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
if (syntheticIndex >= 0)
|
|
3335
|
-
conditions.splice(syntheticIndex, 1);
|
|
3905
|
+
// if we had to add an aligned condition that isn't first, we remove it and do ordering later
|
|
3906
|
+
if (orderAlignedCondition)
|
|
3907
|
+
conditions.splice(conditions.indexOf(orderAlignedCondition), 1);
|
|
3336
3908
|
postOrdering = sort;
|
|
3337
3909
|
}
|
|
3338
3910
|
}
|
|
3339
3911
|
const select = target.select;
|
|
3912
|
+
// Whether the caller supplied real filter conditions — read from the raw request, NOT the
|
|
3913
|
+
// planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and
|
|
3914
|
+
// injected full-scan condition). Used to pick the count-estimate source below.
|
|
3915
|
+
const hasUserConditions = Array.isArray(target.conditions) && target.conditions.length > 0;
|
|
3340
3916
|
if (conditions.length === 0) {
|
|
3341
3917
|
conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }];
|
|
3342
3918
|
}
|
|
@@ -3373,7 +3949,7 @@ function makeTable(options) {
|
|
|
3373
3949
|
const recordAccess = boundRowFilter || typeof target.vectorFilter === 'function'
|
|
3374
3950
|
? { rowFilter: boundRowFilter, vectorFilter: target.vectorFilter }
|
|
3375
3951
|
: undefined;
|
|
3376
|
-
const entries = (0, search_ts_1.executeConditions)(conditions, operator,
|
|
3952
|
+
const entries = (0, search_ts_1.executeConditions)(conditions, operator, _a, readTxn, target, context, (results, filters) => transformToEntries(results, select, context, readTxn, filters), filtered, recordAccess);
|
|
3377
3953
|
const ensure_loaded = target.ensureLoaded !== false;
|
|
3378
3954
|
// The guards inside executeConditions evaluate the
|
|
3379
3955
|
// LOCAL record, but on a caching table transformEntryForSelect may then revalidate an
|
|
@@ -3390,11 +3966,126 @@ function makeTable(options) {
|
|
|
3390
3966
|
// since those never run the ensureLoaded-gated freshness check this transform
|
|
3391
3967
|
// otherwise applies unconditionally to every read.
|
|
3392
3968
|
const includeExpired = target.includeExpired === true;
|
|
3393
|
-
const transformToRecord =
|
|
3394
|
-
let results =
|
|
3969
|
+
const transformToRecord = _a.transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded, true, boundRowFilter, includeExpired, postOrdering);
|
|
3970
|
+
let results = _a.transformToOrderedSelect(entries, select, postOrdering, context, readTxn, transformToRecord);
|
|
3971
|
+
const offset = target.offset || 0;
|
|
3972
|
+
const end = target.limit !== undefined ? offset + target.limit : undefined;
|
|
3973
|
+
// `Prefer: count=` (REST pagination): materialize the requested page and attach a total record
|
|
3974
|
+
// count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once,
|
|
3975
|
+
// windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/
|
|
3976
|
+
// table estimate. Opt-in only — the default streaming path below is untouched.
|
|
3977
|
+
//
|
|
3978
|
+
// Requires a bounded page AND window. Counting is a pagination feature; both the limit and the
|
|
3979
|
+
// offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the
|
|
3980
|
+
// window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/
|
|
3981
|
+
// oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity),
|
|
3982
|
+
// limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the
|
|
3983
|
+
// normal streaming path with no count. This bounds the offset too: without it a huge offset would
|
|
3984
|
+
// postpone the exact guardrail (which only engages past the page) until that offset was scanned.
|
|
3985
|
+
const pageLimit = target.limit;
|
|
3986
|
+
if (target.count &&
|
|
3987
|
+
Number.isInteger(pageLimit) &&
|
|
3988
|
+
pageLimit >= 0 &&
|
|
3989
|
+
pageLimit <= MAX_COUNT_PAGE &&
|
|
3990
|
+
Number.isInteger(offset) &&
|
|
3991
|
+
offset >= 0 &&
|
|
3992
|
+
offset + pageLimit <= MAX_EXACT_COUNT_SCAN) {
|
|
3993
|
+
const wantExact = target.count === 'exact';
|
|
3994
|
+
const pageEnd = offset + pageLimit;
|
|
3995
|
+
const countStart = node_perf_hooks_1.performance.now();
|
|
3996
|
+
// A custom-index (vector/HNSW) traversal returns a bounded, approximate candidate set whose size is
|
|
3997
|
+
// chosen from `minResults` (offset + limit), so `scanned` over it tracks the requested page size, not
|
|
3998
|
+
// the true match count — the same query at limit(5) vs limit(200) would otherwise advertise two
|
|
3999
|
+
// different `count=exact` totals. Any query whose execution touches a custom index is affected: a
|
|
4000
|
+
// custom-index sort (its aligned pseudo-condition lands in `conditions`), a custom-index threshold
|
|
4001
|
+
// filter (an HNSW `lt`/`le` is the same minResults-widened traversal as a sort), or an opaque vector
|
|
4002
|
+
// filter. Report the total as unavailable for those rather than advertising it as count=exact
|
|
4003
|
+
// (mirroring how the estimated branch below bails to null for an opaque row/vector filter). A vector
|
|
4004
|
+
// sort applied as in-memory post-ordering leaves no custom-index condition here and stays exact.
|
|
4005
|
+
const touchesCustomIndex = (conds) => conds.some((c) => {
|
|
4006
|
+
if (!c)
|
|
4007
|
+
return false;
|
|
4008
|
+
if (c.conditions)
|
|
4009
|
+
return touchesCustomIndex(c.conditions);
|
|
4010
|
+
const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]);
|
|
4011
|
+
return typeof attr === 'string' && Boolean(indices[attr]?.customIndex);
|
|
4012
|
+
});
|
|
4013
|
+
const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions);
|
|
4014
|
+
return (async () => {
|
|
4015
|
+
const page = [];
|
|
4016
|
+
let scanned = 0;
|
|
4017
|
+
let exact = true;
|
|
4018
|
+
try {
|
|
4019
|
+
for await (const record of results) {
|
|
4020
|
+
if (scanned >= offset && scanned < pageEnd)
|
|
4021
|
+
page.push(record);
|
|
4022
|
+
scanned++;
|
|
4023
|
+
// A store whose async iterator settles synchronously (the common indexed-scan case) would
|
|
4024
|
+
// otherwise let this drain spin as one uninterrupted microtask run, blocking the event loop
|
|
4025
|
+
// for the whole count. Yield to the macrotask queue periodically so concurrent requests and
|
|
4026
|
+
// I/O still make progress during a large exact scan.
|
|
4027
|
+
if ((scanned & (COUNT_YIELD_INTERVAL - 1)) === 0)
|
|
4028
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
4029
|
+
// The page window [offset, pageEnd) is always collected in full first — the guardrail
|
|
4030
|
+
// only ever abandons the running TOTAL, never truncates the page body.
|
|
4031
|
+
if (scanned >= pageEnd) {
|
|
4032
|
+
// `estimated` needs nothing past the page; an approximate (vector) exact total is going to
|
|
4033
|
+
// be reported unavailable anyway, so don't drain its tail for a number we won't publish.
|
|
4034
|
+
if (!wantExact || approximateResultSet)
|
|
4035
|
+
break;
|
|
4036
|
+
// `exact` keeps counting the tail, bounded by a row cap AND a time budget so a
|
|
4037
|
+
// large match set can't turn a bounded page fetch into an unbounded scan.
|
|
4038
|
+
if (scanned > MAX_EXACT_COUNT_SCAN || node_perf_hooks_1.performance.now() - countStart > MAX_EXACT_COUNT_MS) {
|
|
4039
|
+
exact = false;
|
|
4040
|
+
break;
|
|
4041
|
+
}
|
|
4042
|
+
}
|
|
4043
|
+
}
|
|
4044
|
+
}
|
|
4045
|
+
finally {
|
|
4046
|
+
// We own the iteration here (no results.onDone consumer), so release the read
|
|
4047
|
+
// transaction unconditionally — including when the drain throws — or the snapshot leaks.
|
|
4048
|
+
txn.doneReadTxn();
|
|
4049
|
+
}
|
|
4050
|
+
let total;
|
|
4051
|
+
if (wantExact) {
|
|
4052
|
+
// `scanned` is only an authoritative total when the iteration was exhaustive and deterministic;
|
|
4053
|
+
// an approximate (vector/HNSW) result set is neither, so report the total as unavailable.
|
|
4054
|
+
total = exact && !approximateResultSet ? scanned : null;
|
|
4055
|
+
}
|
|
4056
|
+
else if (boundRowFilter || typeof target.vectorFilter === 'function') {
|
|
4057
|
+
// An opaque row/vector filter shapes the result but isn't reflected in the index/condition
|
|
4058
|
+
// estimate; guessing would both mislead and disclose cardinality the filter hides.
|
|
4059
|
+
total = null;
|
|
4060
|
+
}
|
|
4061
|
+
else if (!hasUserConditions) {
|
|
4062
|
+
total = (0, search_ts_1.estimatedEntryCount)(primaryStore);
|
|
4063
|
+
}
|
|
4064
|
+
else {
|
|
4065
|
+
// Estimate from the real conditions only — drop the planner's synthetic `sort`
|
|
4066
|
+
// pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality.
|
|
4067
|
+
const est = (0, search_ts_1.estimateCondition)(_a)({
|
|
4068
|
+
conditions: conditions.filter((c) => c.comparator !== 'sort'),
|
|
4069
|
+
operator: operator ? String(operator).toLowerCase() : 'and',
|
|
4070
|
+
});
|
|
4071
|
+
total = isFinite(est) ? Math.round(est) : null;
|
|
4072
|
+
}
|
|
4073
|
+
// For an estimate, never report a total below the last row actually returned — keeps the
|
|
4074
|
+
// Content-Range valid (start-end/total) when an estimate undershoots a non-empty page.
|
|
4075
|
+
// Exact totals are authoritative (and an empty page past the end must not be clamped up).
|
|
4076
|
+
if (!wantExact && total != null && page.length > 0 && total < offset + page.length) {
|
|
4077
|
+
total = offset + page.length;
|
|
4078
|
+
}
|
|
4079
|
+
page.recordCount = total;
|
|
4080
|
+
page.recordCountExact = wantExact && exact && !approximateResultSet;
|
|
4081
|
+
page.selectApplied = true;
|
|
4082
|
+
page.getColumns = getColumns;
|
|
4083
|
+
return page;
|
|
4084
|
+
})();
|
|
4085
|
+
}
|
|
3395
4086
|
// apply any offset/limit after all the sorting and filtering
|
|
3396
4087
|
if (target.offset || target.limit !== undefined)
|
|
3397
|
-
results = results.slice(
|
|
4088
|
+
results = results.slice(offset, end);
|
|
3398
4089
|
results.onDone = () => {
|
|
3399
4090
|
results.onDone = null; // ensure that it isn't called twice
|
|
3400
4091
|
txn.doneReadTxn();
|
|
@@ -3709,12 +4400,12 @@ function makeTable(options) {
|
|
|
3709
4400
|
if (resolver.directReturn)
|
|
3710
4401
|
return callback(value, attribute_name);
|
|
3711
4402
|
if (value && typeof value === 'object') {
|
|
3712
|
-
const targetTable = resolver.definition?.tableClass ||
|
|
4403
|
+
const targetTable = resolver.definition?.tableClass || _a;
|
|
3713
4404
|
if (!transformCache)
|
|
3714
4405
|
transformCache = {};
|
|
3715
4406
|
// Use the target table's own read transaction; each table's readTxn is
|
|
3716
4407
|
// scoped to its RocksDB column family and cannot read another table's store.
|
|
3717
|
-
const targetReadTxn = targetTable ===
|
|
4408
|
+
const targetReadTxn = targetTable === _a ? readTxn : targetTable._readTxnForContext(context);
|
|
3718
4409
|
const transform = transformCache[attribute_name] ||
|
|
3719
4410
|
(transformCache[attribute_name] = targetTable.transformEntryForSelect(
|
|
3720
4411
|
// if it is a simple string, there is no select for the next level,
|
|
@@ -3767,7 +4458,7 @@ function makeTable(options) {
|
|
|
3767
4458
|
else {
|
|
3768
4459
|
value = record[attribute_name];
|
|
3769
4460
|
if (value && typeof value === 'object' && attribute_name !== attribute) {
|
|
3770
|
-
const subTransform =
|
|
4461
|
+
const subTransform = _a.transformEntryForSelect(attribute.select || attribute, context, readTxn, null);
|
|
3771
4462
|
// Plain JSON nested values: arrays project per-element so that
|
|
3772
4463
|
// `select: [{ name: 'addresses', select: ['city'] }]` returns
|
|
3773
4464
|
// `addresses: [{ city }, { city }]` rather than a single object.
|
|
@@ -3846,6 +4537,10 @@ function makeTable(options) {
|
|
|
3846
4537
|
if (!auditStore)
|
|
3847
4538
|
throw new Error('Can not subscribe to a table without an audit log');
|
|
3848
4539
|
if (!audit) {
|
|
4540
|
+
// Turning auditing on is a schema write, and a branch's Table classes carry the base's
|
|
4541
|
+
// logical name: without this a subscribe through a branched application would enable
|
|
4542
|
+
// auditing on the live base table for every other consumer, with no DDL call involved.
|
|
4543
|
+
_a.assertSchemaMutable('enable auditing for a subscription');
|
|
3849
4544
|
(0, databases_ts_1.table)({ table: tableName, database: databaseName, schemaDefined, attributes, audit: true });
|
|
3850
4545
|
}
|
|
3851
4546
|
const getFullRecord = !request.rawEvents;
|
|
@@ -3901,7 +4596,7 @@ function makeTable(options) {
|
|
|
3901
4596
|
return evaluateFilter(rowFilter, event.value, 'rowFilter');
|
|
3902
4597
|
}
|
|
3903
4598
|
: null;
|
|
3904
|
-
const subscription = (0, transactionBroadcast_ts_1.addSubscription)(
|
|
4599
|
+
const subscription = (0, transactionBroadcast_ts_1.addSubscription)(_a, thisId, function (id, auditRecord, txnLogKey, beginTxn) {
|
|
3905
4600
|
if (dropDuringReplay)
|
|
3906
4601
|
return;
|
|
3907
4602
|
try {
|
|
@@ -3928,8 +4623,7 @@ function makeTable(options) {
|
|
|
3928
4623
|
// been written, so are fresh in memory.
|
|
3929
4624
|
const entry = primaryStore.getEntry(id);
|
|
3930
4625
|
if (entry) {
|
|
3931
|
-
|
|
3932
|
-
if (entry.version !== (auditRecord.recordVersion ?? auditRecord.version))
|
|
4626
|
+
if (entry.version !== auditRecord.version)
|
|
3933
4627
|
return; // out of order event, with old update, don't send anything
|
|
3934
4628
|
value = entry.value;
|
|
3935
4629
|
type = entry.metadataFlags & exports.INVALIDATED ? 'invalidate' : value ? 'put' : 'delete';
|
|
@@ -3940,7 +4634,7 @@ function makeTable(options) {
|
|
|
3940
4634
|
}
|
|
3941
4635
|
const event = {
|
|
3942
4636
|
id,
|
|
3943
|
-
localTime,
|
|
4637
|
+
localTime: txnLogKey,
|
|
3944
4638
|
value,
|
|
3945
4639
|
version: auditRecord.version,
|
|
3946
4640
|
type,
|
|
@@ -4009,10 +4703,10 @@ function makeTable(options) {
|
|
|
4009
4703
|
continue;
|
|
4010
4704
|
const id = auditRecord.recordId;
|
|
4011
4705
|
if (thisId == null || isDescendantId(thisId, id)) {
|
|
4012
|
-
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.
|
|
4706
|
+
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
|
|
4013
4707
|
if (!send({
|
|
4014
4708
|
id,
|
|
4015
|
-
localTime: auditRecord.
|
|
4709
|
+
localTime: auditRecord.txnLogKey,
|
|
4016
4710
|
value,
|
|
4017
4711
|
version: auditRecord.version,
|
|
4018
4712
|
type: auditRecord.type,
|
|
@@ -4025,7 +4719,7 @@ function makeTable(options) {
|
|
|
4025
4719
|
return;
|
|
4026
4720
|
}
|
|
4027
4721
|
}
|
|
4028
|
-
subscription.startTime = auditRecord.
|
|
4722
|
+
subscription.startTime = auditRecord.txnLogKey; // update so we don't double send
|
|
4029
4723
|
}
|
|
4030
4724
|
}
|
|
4031
4725
|
finally {
|
|
@@ -4059,10 +4753,10 @@ function makeTable(options) {
|
|
|
4059
4753
|
logger_ts_1.logger.warn?.(`previousCount backfill on ${tableName} stopped after inspecting ${MAX_PREVIOUS_COUNT_SCAN} in-scope audit records without collecting ${request.previousCount} accepted event(s); returning ${history.length} instead`);
|
|
4060
4754
|
break;
|
|
4061
4755
|
}
|
|
4062
|
-
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.
|
|
4756
|
+
const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
|
|
4063
4757
|
const historyEntry = {
|
|
4064
4758
|
id,
|
|
4065
|
-
localTime: auditRecord.
|
|
4759
|
+
localTime: auditRecord.txnLogKey,
|
|
4066
4760
|
value,
|
|
4067
4761
|
version: auditRecord.version,
|
|
4068
4762
|
type: auditRecord.type,
|
|
@@ -4079,7 +4773,7 @@ function makeTable(options) {
|
|
|
4079
4773
|
}
|
|
4080
4774
|
}
|
|
4081
4775
|
catch (error) {
|
|
4082
|
-
logger_ts_1.logger.error?.('Error getting history entry', auditRecord.
|
|
4776
|
+
logger_ts_1.logger.error?.('Error getting history entry', auditRecord.txnLogKey, error);
|
|
4083
4777
|
}
|
|
4084
4778
|
}
|
|
4085
4779
|
for (let i = history.length; i > 0;) {
|
|
@@ -4167,6 +4861,12 @@ function makeTable(options) {
|
|
|
4167
4861
|
logger_ts_1.logger.trace?.('re-retrieved record', localTime, this.#entry?.localTime);
|
|
4168
4862
|
localTime = entry?.localTime;
|
|
4169
4863
|
}
|
|
4864
|
+
let nodeId = entry?.nodeId;
|
|
4865
|
+
if (isRocksDB && entry) {
|
|
4866
|
+
const head = resolveAuditHead(thisId, entry.version, nodeId, entry.additionalAuditRefs);
|
|
4867
|
+
localTime = head.txnLogKey;
|
|
4868
|
+
nodeId = head.nodeId;
|
|
4869
|
+
}
|
|
4170
4870
|
logger_ts_1.logger.trace?.('Subscription from', startTime, 'from', thisId, localTime);
|
|
4171
4871
|
if (startTime < localTime) {
|
|
4172
4872
|
// start time specified, get the audit history for this record. Set startTime up
|
|
@@ -4177,7 +4877,6 @@ function makeTable(options) {
|
|
|
4177
4877
|
const history = [];
|
|
4178
4878
|
let inspected = 0;
|
|
4179
4879
|
let nextTime = localTime;
|
|
4180
|
-
let nodeId = entry?.nodeId;
|
|
4181
4880
|
do {
|
|
4182
4881
|
if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
|
|
4183
4882
|
recordsSinceYield = 0;
|
|
@@ -4208,8 +4907,11 @@ function makeTable(options) {
|
|
|
4208
4907
|
else if (!isActive())
|
|
4209
4908
|
return;
|
|
4210
4909
|
}
|
|
4211
|
-
|
|
4212
|
-
|
|
4910
|
+
const previousHead = isRocksDB
|
|
4911
|
+
? resolveAuditHead(thisId, auditRecord.previousVersion, auditRecord.previousNodeId, auditRecord.previousAdditionalAuditRefs)
|
|
4912
|
+
: { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
|
|
4913
|
+
nextTime = previousHead.txnLogKey;
|
|
4914
|
+
nodeId = previousHead.nodeId;
|
|
4213
4915
|
}
|
|
4214
4916
|
else
|
|
4215
4917
|
break;
|
|
@@ -4404,6 +5106,7 @@ function makeTable(options) {
|
|
|
4404
5106
|
store: primaryStore,
|
|
4405
5107
|
entry: this.#entry,
|
|
4406
5108
|
nodeName: context?.nodeName,
|
|
5109
|
+
recordVersion: options?.version,
|
|
4407
5110
|
validate: () => {
|
|
4408
5111
|
if (!context?.source) {
|
|
4409
5112
|
transaction.checkOverloaded();
|
|
@@ -4617,6 +5320,7 @@ function makeTable(options) {
|
|
|
4617
5320
|
return this.#version;
|
|
4618
5321
|
}
|
|
4619
5322
|
static async addAttributes(attributesToAdd) {
|
|
5323
|
+
_a.assertSchemaMutable('add attributes');
|
|
4620
5324
|
const new_attributes = attributes.slice(0);
|
|
4621
5325
|
for (const attribute of attributesToAdd) {
|
|
4622
5326
|
if (!attribute.name)
|
|
@@ -4632,9 +5336,10 @@ function makeTable(options) {
|
|
|
4632
5336
|
schemaDefined,
|
|
4633
5337
|
attributes: new_attributes,
|
|
4634
5338
|
});
|
|
4635
|
-
return
|
|
5339
|
+
return _a.indexingOperation;
|
|
4636
5340
|
}
|
|
4637
5341
|
static async removeAttributes(names) {
|
|
5342
|
+
_a.assertSchemaMutable('remove attributes');
|
|
4638
5343
|
const new_attributes = attributes.filter((attribute) => !names.includes(attribute.name));
|
|
4639
5344
|
(0, databases_ts_1.table)({
|
|
4640
5345
|
table: tableName,
|
|
@@ -4642,7 +5347,7 @@ function makeTable(options) {
|
|
|
4642
5347
|
schemaDefined,
|
|
4643
5348
|
attributes: new_attributes,
|
|
4644
5349
|
});
|
|
4645
|
-
return
|
|
5350
|
+
return _a.indexingOperation;
|
|
4646
5351
|
}
|
|
4647
5352
|
/**
|
|
4648
5353
|
* Get the size of the table in bytes (based on amount of pages stored in the database)
|
|
@@ -4655,6 +5360,10 @@ function makeTable(options) {
|
|
|
4655
5360
|
const stats = primaryStore.getStats();
|
|
4656
5361
|
return (stats.treeBranchPageCount + stats.treeLeafPageCount + stats.overflowPages) * stats.pageSize;
|
|
4657
5362
|
}
|
|
5363
|
+
/** Sizes of this table's durable record-structure dictionaries. */
|
|
5364
|
+
static getStructureCounts() {
|
|
5365
|
+
return primaryStore.encoder?.getStructureCounts?.();
|
|
5366
|
+
}
|
|
4658
5367
|
static getAuditSize() {
|
|
4659
5368
|
const stats = auditStore?.getStats();
|
|
4660
5369
|
return (stats &&
|
|
@@ -4673,7 +5382,7 @@ function makeTable(options) {
|
|
|
4673
5382
|
// iterate through the metadata entries to exclude their count and exclude the deletion counts
|
|
4674
5383
|
const exactCount = options?.exactCount;
|
|
4675
5384
|
const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms
|
|
4676
|
-
const start = performance.now();
|
|
5385
|
+
const start = node_perf_hooks_1.performance.now();
|
|
4677
5386
|
// `entryCount` (the exact key count) is only needed once the scan blows the time budget --
|
|
4678
5387
|
// to decide whether to estimate and as the extrapolation base. On RocksDB it is a full
|
|
4679
5388
|
// key-only scan, so we defer it: tables that finish within budget (the common case) and
|
|
@@ -4690,7 +5399,7 @@ function makeTable(options) {
|
|
|
4690
5399
|
recordCount++;
|
|
4691
5400
|
entriesScanned++;
|
|
4692
5401
|
await rest();
|
|
4693
|
-
if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) {
|
|
5402
|
+
if (!exactCount && !completeForExact && node_perf_hooks_1.performance.now() - start > TIME_LIMIT) {
|
|
4694
5403
|
if (!counted) {
|
|
4695
5404
|
counted = true;
|
|
4696
5405
|
entryCount = isRocksDB
|
|
@@ -4814,7 +5523,7 @@ function makeTable(options) {
|
|
|
4814
5523
|
const computed = attribute.computed;
|
|
4815
5524
|
// Register the default embedder unless an author override is set. Sits outside
|
|
4816
5525
|
// the resolver chain below so `@embed` fields still flow through auto-HNSW indexing.
|
|
4817
|
-
if (attribute.embed && !
|
|
5526
|
+
if (attribute.embed && !_a.userSetEmbedders.has(attribute.name)) {
|
|
4818
5527
|
this.userEmbedders[attribute.name] = (0, embedHook_ts_1.createDefaultEmbedder)(attribute.embed);
|
|
4819
5528
|
}
|
|
4820
5529
|
if (relationship) {
|
|
@@ -4880,7 +5589,7 @@ function makeTable(options) {
|
|
|
4880
5589
|
const options = { transaction: txnForContext(context).getReadTxn() };
|
|
4881
5590
|
const results = normalizedIds.map((id) => {
|
|
4882
5591
|
const value = store[method](id, options);
|
|
4883
|
-
if (
|
|
5592
|
+
if (_a.loadAsInstance === false)
|
|
4884
5593
|
freezeRecord(returnEntry ? value?.value : value);
|
|
4885
5594
|
return value;
|
|
4886
5595
|
});
|
|
@@ -4889,7 +5598,7 @@ function makeTable(options) {
|
|
|
4889
5598
|
const value = definition.tableClass.primaryStore[returnEntry ? 'getEntry' : 'getSync'](ids, {
|
|
4890
5599
|
transaction: txnForContext(context).getReadTxn(),
|
|
4891
5600
|
});
|
|
4892
|
-
if (
|
|
5601
|
+
if (_a.loadAsInstance === false)
|
|
4893
5602
|
freezeRecord(returnEntry ? value?.value : value);
|
|
4894
5603
|
return value;
|
|
4895
5604
|
};
|
|
@@ -5044,30 +5753,85 @@ function makeTable(options) {
|
|
|
5044
5753
|
this.userSetEmbedders.add(attribute_name);
|
|
5045
5754
|
}
|
|
5046
5755
|
static async deleteHistory(endTime = 0, cleanupDeletedRecords = false) {
|
|
5047
|
-
|
|
5756
|
+
const maxConcurrentRemovals = isRocksDB ? MAX_CONCURRENT_HISTORY_REMOVALS : MAX_CONCURRENT_LMDB_HISTORY_REMOVALS;
|
|
5757
|
+
const inFlightRemovals = new Set();
|
|
5758
|
+
const removalSlotWaiters = [];
|
|
5759
|
+
let removalsAttempted = 0;
|
|
5760
|
+
let removalsSucceeded = 0;
|
|
5761
|
+
let firstRemovalError;
|
|
5762
|
+
function startRemoval(remove, errorMessage, onSuccess) {
|
|
5763
|
+
removalsAttempted++;
|
|
5764
|
+
const removal = new Promise((resolve) => resolve(remove()))
|
|
5765
|
+
.then(() => {
|
|
5766
|
+
removalsSucceeded++;
|
|
5767
|
+
onSuccess?.();
|
|
5768
|
+
}, (error) => {
|
|
5769
|
+
// capture before logging: a throwing logger must not cost us the error we may rethrow
|
|
5770
|
+
if (firstRemovalError === undefined)
|
|
5771
|
+
firstRemovalError = error;
|
|
5772
|
+
harper_logger_ts_1.default.warn(errorMessage, error);
|
|
5773
|
+
})
|
|
5774
|
+
.catch(() => undefined)
|
|
5775
|
+
.finally(() => {
|
|
5776
|
+
inFlightRemovals.delete(removal);
|
|
5777
|
+
removalSlotWaiters.shift()?.();
|
|
5778
|
+
});
|
|
5779
|
+
inFlightRemovals.add(removal);
|
|
5780
|
+
}
|
|
5781
|
+
function queueRemoval(remove, errorMessage, onSuccess) {
|
|
5782
|
+
if (inFlightRemovals.size >= maxConcurrentRemovals) {
|
|
5783
|
+
return new Promise((resolve) => {
|
|
5784
|
+
removalSlotWaiters.push(resolve);
|
|
5785
|
+
}).then(() => startRemoval(remove, errorMessage, onSuccess));
|
|
5786
|
+
}
|
|
5787
|
+
startRemoval(remove, errorMessage, onSuccess);
|
|
5788
|
+
}
|
|
5789
|
+
const drainRemovals = () => Promise.all(inFlightRemovals);
|
|
5048
5790
|
let entriesDeleted = 0;
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5056
|
-
|
|
5057
|
-
|
|
5791
|
+
try {
|
|
5792
|
+
for (const auditRecord of auditStore.getRange({
|
|
5793
|
+
start: 1, // must not be zero; see getHistory below for why
|
|
5794
|
+
end: endTime,
|
|
5795
|
+
})) {
|
|
5796
|
+
await rest(); // yield to other async operations
|
|
5797
|
+
if (auditRecord.tableId !== tableId)
|
|
5798
|
+
continue;
|
|
5799
|
+
const backpressure = queueRemoval(() => (0, auditStore_ts_1.removeAuditEntry)(auditStore, auditRecord), 'Error removing audit entry during deleteHistory', () => {
|
|
5800
|
+
entriesDeleted++;
|
|
5801
|
+
});
|
|
5802
|
+
if (backpressure)
|
|
5803
|
+
await backpressure;
|
|
5804
|
+
}
|
|
5805
|
+
}
|
|
5806
|
+
finally {
|
|
5807
|
+
await drainRemovals();
|
|
5058
5808
|
}
|
|
5059
5809
|
if (cleanupDeletedRecords) {
|
|
5060
5810
|
// this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't
|
|
5061
5811
|
// ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records
|
|
5062
|
-
|
|
5063
|
-
const {
|
|
5064
|
-
|
|
5065
|
-
|
|
5066
|
-
|
|
5812
|
+
try {
|
|
5813
|
+
for (const entry of primaryStore.getRange({ start: 0, versions: true })) {
|
|
5814
|
+
const { key, value, localTime, version } = entry;
|
|
5815
|
+
await rest(); // yield to other async operations
|
|
5816
|
+
const auditTime = isRocksDB && version != null
|
|
5817
|
+
? resolveAuditHead(key, version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
|
|
5818
|
+
: localTime;
|
|
5819
|
+
if (value === null && version != null && auditTime < endTime) {
|
|
5820
|
+
const backpressure = queueRemoval(() => primaryStore.remove(key, version), 'Error removing deleted record during deleteHistory');
|
|
5821
|
+
if (backpressure)
|
|
5822
|
+
await backpressure;
|
|
5823
|
+
}
|
|
5067
5824
|
}
|
|
5068
5825
|
}
|
|
5826
|
+
finally {
|
|
5827
|
+
await drainRemovals();
|
|
5828
|
+
}
|
|
5829
|
+
}
|
|
5830
|
+
if (removalsAttempted > 0 && removalsSucceeded === 0) {
|
|
5831
|
+
// zero progress must not report the same success as "nothing was eligible" (see DESIGN.md);
|
|
5832
|
+
// partial failures stay best-effort, logged and excluded from the returned count
|
|
5833
|
+
throw firstRemovalError ?? new Error('Every removal attempted during deleteHistory failed');
|
|
5069
5834
|
}
|
|
5070
|
-
await completion;
|
|
5071
5835
|
return entriesDeleted;
|
|
5072
5836
|
}
|
|
5073
5837
|
static async *getHistory(startTime = 0, endTime = Infinity) {
|
|
@@ -5080,10 +5844,11 @@ function makeTable(options) {
|
|
|
5080
5844
|
continue;
|
|
5081
5845
|
yield {
|
|
5082
5846
|
id: auditRecord.recordId,
|
|
5083
|
-
|
|
5847
|
+
// Compatibility-facing LMDB history has always reported/grouped by record version.
|
|
5848
|
+
localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
|
|
5084
5849
|
version: auditRecord.version,
|
|
5085
5850
|
type: auditRecord.type,
|
|
5086
|
-
value: auditRecord.getValue(primaryStore, true, auditRecord.
|
|
5851
|
+
value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
|
|
5087
5852
|
user: auditRecord.user,
|
|
5088
5853
|
operation: auditRecord.originatingOperation,
|
|
5089
5854
|
};
|
|
@@ -5096,7 +5861,9 @@ function makeTable(options) {
|
|
|
5096
5861
|
const entry = primaryStore.getEntry(id);
|
|
5097
5862
|
if (!entry)
|
|
5098
5863
|
return history;
|
|
5099
|
-
let nextVersion =
|
|
5864
|
+
let nextVersion = isRocksDB
|
|
5865
|
+
? resolveAuditHead(id, entry.version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
|
|
5866
|
+
: entry.localTime;
|
|
5100
5867
|
if (!nextVersion)
|
|
5101
5868
|
throw new Error('The entry does not have a local audit time');
|
|
5102
5869
|
const count = 0;
|
|
@@ -5110,17 +5877,20 @@ function makeTable(options) {
|
|
|
5110
5877
|
if (auditRecord.tableId === tableId && (0, ordered_binary_1.compareKeys)(auditRecord.recordId, id) === 0) {
|
|
5111
5878
|
history.splice(insertionPoint, 0, {
|
|
5112
5879
|
id: auditRecord.recordId,
|
|
5113
|
-
localTime: auditRecord.version,
|
|
5880
|
+
localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
|
|
5114
5881
|
version: auditRecord.version,
|
|
5115
5882
|
type: auditRecord.type,
|
|
5116
|
-
// reconstruct each entry's record image as of its own
|
|
5883
|
+
// reconstruct each entry's record image as of its own log position, not the audit
|
|
5117
5884
|
// window boundary (nextVersion), matching getHistory (issue #1330)
|
|
5118
|
-
value: auditRecord.getValue(primaryStore, true, auditRecord.
|
|
5885
|
+
value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
|
|
5119
5886
|
user: auditRecord.user,
|
|
5120
5887
|
operation: auditRecord.originatingOperation,
|
|
5121
5888
|
});
|
|
5122
|
-
|
|
5123
|
-
|
|
5889
|
+
const previousVersion = isRocksDB
|
|
5890
|
+
? resolveAuditHead(id, auditRecord.previousVersion, auditRecord.previousNodeId, auditRecord.previousAdditionalAuditRefs).txnLogKey
|
|
5891
|
+
: auditRecord.previousVersion;
|
|
5892
|
+
if (previousVersion > highestPreviousVersion && previousVersion < start) {
|
|
5893
|
+
highestPreviousVersion = previousVersion;
|
|
5124
5894
|
}
|
|
5125
5895
|
}
|
|
5126
5896
|
}
|
|
@@ -5139,13 +5909,20 @@ function makeTable(options) {
|
|
|
5139
5909
|
}
|
|
5140
5910
|
return Promise.all(promises);
|
|
5141
5911
|
}
|
|
5912
|
+
/** Release everything makeTable() registered process-wide; the class must not be used afterwards. */
|
|
5142
5913
|
static cleanup() {
|
|
5914
|
+
disposed = true;
|
|
5915
|
+
clearTimeout(cleanupTimer);
|
|
5916
|
+
settlePendingCleanup();
|
|
5917
|
+
clearInterval(recordExpirationInterval);
|
|
5143
5918
|
deleteCallbackHandle?.remove();
|
|
5919
|
+
(0, storageReclamation_ts_1.removeStorageReclamationHandler)(primaryStore.path, reclamationHandler);
|
|
5144
5920
|
}
|
|
5145
5921
|
static _readTxnForContext(context) {
|
|
5146
5922
|
return txnForContext(context).getReadTxn();
|
|
5147
5923
|
}
|
|
5148
5924
|
}
|
|
5925
|
+
_a = TableResource;
|
|
5149
5926
|
const throttledCallToSource = (0, throttle_ts_1.throttle)(async (source, id, sourceContext, existingEntry) => {
|
|
5150
5927
|
// call the data source if it exists and will fulfill our request for data
|
|
5151
5928
|
if (source && source.get && (!source.get.reliesOnPrototype || source.prototype.get)) {
|
|
@@ -5159,11 +5936,17 @@ function makeTable(options) {
|
|
|
5159
5936
|
}, () => {
|
|
5160
5937
|
throw new hdbError_ts_1.ServerError('Service unavailable, exceeded request queue limit for resolving cache record', 503);
|
|
5161
5938
|
});
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5939
|
+
try {
|
|
5940
|
+
TableResource.updatedAttributes(); // on creation, update accessors as well
|
|
5941
|
+
if (expirationMs)
|
|
5942
|
+
TableResource.setTTLExpiration(expirationMs / 1000);
|
|
5943
|
+
if (expiresAtProperty)
|
|
5944
|
+
runRecordExpirationEviction();
|
|
5945
|
+
}
|
|
5946
|
+
catch (error) {
|
|
5947
|
+
TableResource.cleanup();
|
|
5948
|
+
throw error;
|
|
5949
|
+
}
|
|
5167
5950
|
return TableResource;
|
|
5168
5951
|
function updateIndices(id, existingRecord, record, options) {
|
|
5169
5952
|
let hasChanges;
|
|
@@ -5590,6 +6373,22 @@ function makeTable(options) {
|
|
|
5590
6373
|
return transaction;
|
|
5591
6374
|
}
|
|
5592
6375
|
}
|
|
6376
|
+
/**
|
|
6377
|
+
* Detach an unsaved TransactionWrite that a scoped lock() eagerly staged (see #reloadLocked)
|
|
6378
|
+
* once its handle upgrades to hold: hold staging is deferred and explicit-save-only, so a
|
|
6379
|
+
* dangling scoped write would otherwise auto-commit at the transaction sweep and clobber
|
|
6380
|
+
* whatever the hold write lands. Marking it .dropped lets a later save() on the instance that
|
|
6381
|
+
* owns it (checked via #savingOperation === this write) fall through to the hold branch
|
|
6382
|
+
* instead of resolving a detached, dead reference.
|
|
6383
|
+
*/
|
|
6384
|
+
function detachScopedUpgradeWrite(link, keyId, handle) {
|
|
6385
|
+
for (const write of link.writes) {
|
|
6386
|
+
if (write && !write.saved && write.lockHandle === handle && (0, DatabaseTransaction_ts_1.writeKeyId)(write.key) === keyId) {
|
|
6387
|
+
write.dropped = true;
|
|
6388
|
+
link.detachWrite(write);
|
|
6389
|
+
}
|
|
6390
|
+
}
|
|
6391
|
+
}
|
|
5593
6392
|
function getAttributeValue(entry, attribute_name, context, sort) {
|
|
5594
6393
|
if (!entry) {
|
|
5595
6394
|
return;
|
|
@@ -5707,6 +6506,10 @@ function makeTable(options) {
|
|
|
5707
6506
|
async function getFromSource(source, id, existingEntry, context, target) {
|
|
5708
6507
|
const metadataFlags = existingEntry?.metadataFlags;
|
|
5709
6508
|
const existingVersion = existingEntry?.version;
|
|
6509
|
+
const existingRecord = existingEntry?.value;
|
|
6510
|
+
const inheritedTimestamp = context?.timestamp || context?.transaction?.timestamp;
|
|
6511
|
+
const sourceTimestamp = inheritedTimestamp ||
|
|
6512
|
+
(isRocksDB ? primaryStore.getMonotonicTimestamp() : (0, commonUtility_ts_1.getNextMonotonicTime)());
|
|
5710
6513
|
let whenResolved, timer;
|
|
5711
6514
|
// We start by locking the record so that there is only one resolution happening at once;
|
|
5712
6515
|
// if there is already a resolution in process, we want to use the results of that resolution
|
|
@@ -5741,10 +6544,8 @@ function makeTable(options) {
|
|
|
5741
6544
|
}
|
|
5742
6545
|
// lock acquired — this request will actually load from source
|
|
5743
6546
|
setLoadedFromSource(target, true);
|
|
5744
|
-
const existingRecord = existingEntry?.value;
|
|
5745
6547
|
// it is important to remember that this is _NOT_ part of the current transaction; nothing is changing
|
|
5746
|
-
// with the canonical data, we are simply fulfilling our local copy of the canonical data
|
|
5747
|
-
// want a timestamp later than the current transaction
|
|
6548
|
+
// with the canonical data, we are simply fulfilling our local copy of the canonical data.
|
|
5748
6549
|
// we create a new context for the source, we want to determine the timestamp and don't want to
|
|
5749
6550
|
// attribute this to the current user
|
|
5750
6551
|
const sourceContext = {
|
|
@@ -5779,15 +6580,36 @@ function makeTable(options) {
|
|
|
5779
6580
|
// belt to that suspenders, at the cost of a bounded wait on a merely slow source
|
|
5780
6581
|
// before the drain's fail-closed timeout below.
|
|
5781
6582
|
const commitPromise = (0, transaction_ts_1.transaction)(sourceContext, async (_txn) => {
|
|
5782
|
-
const start = performance.now();
|
|
5783
|
-
let updatedRecord;
|
|
6583
|
+
const start = node_perf_hooks_1.performance.now();
|
|
6584
|
+
let updatedRecord, assignCreatedTime, sourceVersion;
|
|
5784
6585
|
let hasChanges, invalidated;
|
|
5785
6586
|
try {
|
|
5786
6587
|
updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry);
|
|
5787
6588
|
invalidated = metadataFlags & exports.INVALIDATED;
|
|
5788
|
-
|
|
5789
|
-
|
|
5790
|
-
|
|
6589
|
+
const reportedVersion = sourceContext.lastModified;
|
|
6590
|
+
const validReportedVersion = typeof reportedVersion === 'number' &&
|
|
6591
|
+
Number.isFinite(reportedVersion) &&
|
|
6592
|
+
reportedVersion > 0 &&
|
|
6593
|
+
reportedVersion <= MAX_DATE_TIMESTAMP;
|
|
6594
|
+
if (validReportedVersion) {
|
|
6595
|
+
// A record version is also this node's ordering token (precedesExistingVersion), so a
|
|
6596
|
+
// source-reported version ahead of local time would make every subsequent local write look
|
|
6597
|
+
// out-of-order and be discarded until wall-clock caught up — freezing the row. Honor what
|
|
6598
|
+
// the source reports, but never beyond now.
|
|
6599
|
+
const versionCeiling = Math.max(sourceTimestamp, Date.now());
|
|
6600
|
+
sourceVersion = Math.min(reportedVersion, versionCeiling);
|
|
6601
|
+
if (sourceVersion !== reportedVersion) {
|
|
6602
|
+
logger_ts_1.logger.trace?.(`Capping future source version for ${tableName} id ${id}: ${reportedVersion} -> ${sourceVersion}`);
|
|
6603
|
+
if (!warnedFutureSourceVersion) {
|
|
6604
|
+
warnedFutureSourceVersion = true;
|
|
6605
|
+
logger_ts_1.logger.warn?.(`The source for ${tableName} reported a lastModified ahead of local time (${new Date(reportedVersion).toISOString()}) for id ${id}; capping cached record versions at local time`);
|
|
6606
|
+
}
|
|
6607
|
+
}
|
|
6608
|
+
}
|
|
6609
|
+
else
|
|
6610
|
+
sourceVersion = sourceTimestamp;
|
|
6611
|
+
hasChanges = invalidated || (validReportedVersion && reportedVersion > existingVersion) || !existingRecord;
|
|
6612
|
+
const resolveDuration = node_perf_hooks_1.performance.now() - start;
|
|
5791
6613
|
(0, write_ts_1.recordAction)(resolveDuration, 'cache-resolution', tableName, null, 'success');
|
|
5792
6614
|
if (responseHeaders)
|
|
5793
6615
|
(0, Headers_ts_1.appendHeader)(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
|
|
@@ -5802,7 +6624,7 @@ function makeTable(options) {
|
|
|
5802
6624
|
if (status === 304) {
|
|
5803
6625
|
// revalidation of our current cached record
|
|
5804
6626
|
updatedRecord = existingRecord;
|
|
5805
|
-
|
|
6627
|
+
sourceVersion = existingVersion;
|
|
5806
6628
|
}
|
|
5807
6629
|
else if (!CACHEABLE_STATUS_CODES.has(status)) {
|
|
5808
6630
|
// non-cacheable status - propagate to client without caching
|
|
@@ -5880,10 +6702,14 @@ function makeTable(options) {
|
|
|
5880
6702
|
if (primaryKey && updatedRecord[primaryKey] !== id)
|
|
5881
6703
|
updatedRecord[primaryKey] = id;
|
|
5882
6704
|
}
|
|
6705
|
+
assignCreatedTime = createdTimeProperty && updatedRecord?.[createdTimeProperty.name] == null;
|
|
5883
6706
|
resolved = true;
|
|
6707
|
+
const resolvedVersion = isRocksDB && updatedRecord && existingVersion != null
|
|
6708
|
+
? Math.max(sourceVersion, existingVersion)
|
|
6709
|
+
: sourceVersion;
|
|
5884
6710
|
const resolvedEntry = {
|
|
5885
6711
|
key: id,
|
|
5886
|
-
version,
|
|
6712
|
+
version: resolvedVersion,
|
|
5887
6713
|
value: updatedRecord,
|
|
5888
6714
|
expiresAt: sourceContext.expiresAt,
|
|
5889
6715
|
metadataFlags: 0,
|
|
@@ -5922,7 +6748,7 @@ function makeTable(options) {
|
|
|
5922
6748
|
}
|
|
5923
6749
|
else
|
|
5924
6750
|
reject(error);
|
|
5925
|
-
const resolveDuration = performance.now() - start;
|
|
6751
|
+
const resolveDuration = node_perf_hooks_1.performance.now() - start;
|
|
5926
6752
|
(0, write_ts_1.recordAction)(resolveDuration, 'cache-resolution', tableName, null, 'fail');
|
|
5927
6753
|
if (responseHeaders)
|
|
5928
6754
|
(0, Headers_ts_1.appendHeader)(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
|
|
@@ -5941,16 +6767,28 @@ function makeTable(options) {
|
|
|
5941
6767
|
const sourceWrite = {
|
|
5942
6768
|
key: id,
|
|
5943
6769
|
store: primaryStore,
|
|
5944
|
-
entry:
|
|
6770
|
+
entry: undefined,
|
|
5945
6771
|
nodeName: 'source',
|
|
5946
|
-
commit: (
|
|
6772
|
+
commit: (_txnTime, existingEntry, _retry, transaction) => {
|
|
5947
6773
|
sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true
|
|
5948
|
-
|
|
5949
|
-
|
|
6774
|
+
const racedVersion = existingEntry?.version;
|
|
6775
|
+
// A first fill may replace a record that raced it only when its candidate version strictly
|
|
6776
|
+
// orders after that record. The comparison has to be replica-independent, so a tie leaves the
|
|
6777
|
+
// raced record in place: precedesExistingVersion() would break the tie with *this* node's
|
|
6778
|
+
// name, and a fill from a shared source has no node identity of its own, so two replicas
|
|
6779
|
+
// resolving the same tie could keep different values at the same version.
|
|
6780
|
+
const replacesRacedRecord = racedVersion == null || sourceVersion > racedVersion;
|
|
6781
|
+
if (racedVersion !== existingVersion &&
|
|
6782
|
+
// Revalidations retain exact-CAS semantics; first fills use deterministic ordering.
|
|
6783
|
+
(existingVersion != null || !updatedRecord || !replacesRacedRecord)) {
|
|
6784
|
+
logger_ts_1.logger.trace?.(`Discarding resolved record from source with id: ${id}, source version: ${sourceVersion}, current version: ${racedVersion}`);
|
|
5950
6785
|
sourceWrite.skipped = true;
|
|
5951
6786
|
return;
|
|
5952
6787
|
}
|
|
5953
|
-
|
|
6788
|
+
const currentRecord = existingEntry?.value;
|
|
6789
|
+
const recordVersion = isRocksDB && racedVersion != null ? Math.max(sourceVersion, racedVersion) : sourceVersion;
|
|
6790
|
+
const txnLogKey = isRocksDB ? transaction?.getTimestamp?.() : recordVersion;
|
|
6791
|
+
updateIndices(id, currentRecord, updatedRecord, transaction && { transaction });
|
|
5954
6792
|
if (updatedRecord) {
|
|
5955
6793
|
if (existingEntry) {
|
|
5956
6794
|
context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
|
|
@@ -5961,23 +6799,23 @@ function makeTable(options) {
|
|
|
5961
6799
|
if (updatedTimeProperty) {
|
|
5962
6800
|
updatedRecord[updatedTimeProperty.name] =
|
|
5963
6801
|
updatedTimeProperty.type === 'Date'
|
|
5964
|
-
? new Date(
|
|
6802
|
+
? new Date(recordVersion)
|
|
5965
6803
|
: updatedTimeProperty.type === 'String'
|
|
5966
|
-
? new Date(
|
|
5967
|
-
:
|
|
6804
|
+
? new Date(recordVersion).toISOString()
|
|
6805
|
+
: recordVersion;
|
|
5968
6806
|
}
|
|
5969
|
-
if (
|
|
5970
|
-
const existingCreatedTime =
|
|
6807
|
+
if (assignCreatedTime) {
|
|
6808
|
+
const existingCreatedTime = currentRecord?.[createdTimeProperty.name];
|
|
5971
6809
|
if (existingCreatedTime != null) {
|
|
5972
6810
|
updatedRecord[createdTimeProperty.name] = existingCreatedTime;
|
|
5973
6811
|
}
|
|
5974
6812
|
else {
|
|
5975
6813
|
updatedRecord[createdTimeProperty.name] =
|
|
5976
6814
|
createdTimeProperty.type === 'Date'
|
|
5977
|
-
? new Date(
|
|
6815
|
+
? new Date(recordVersion)
|
|
5978
6816
|
: createdTimeProperty.type === 'String'
|
|
5979
|
-
? new Date(
|
|
5980
|
-
:
|
|
6817
|
+
? new Date(recordVersion).toISOString()
|
|
6818
|
+
: recordVersion;
|
|
5981
6819
|
}
|
|
5982
6820
|
}
|
|
5983
6821
|
const residency = residencyFromFunction(TableResource.getResidency(updatedRecord, context));
|
|
@@ -6010,23 +6848,33 @@ function makeTable(options) {
|
|
|
6010
6848
|
}
|
|
6011
6849
|
residencyId = getResidencyId(residency);
|
|
6012
6850
|
}
|
|
6013
|
-
logger_ts_1.logger.trace?.(`Writing resolved record from source with id: ${id}, timestamp: ${new Date(
|
|
6851
|
+
logger_ts_1.logger.trace?.(`Writing resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`);
|
|
6014
6852
|
// TODO: We are doing a double check for ifVersion that should probably be cleaned out
|
|
6015
|
-
|
|
6853
|
+
const writeAudit = (audit && (hasChanges || omitLocalRecord)) || null;
|
|
6854
|
+
updateRecord(id, updatedRecord, existingEntry, recordVersion, omitLocalRecord ? exports.INVALIDATED : 0, writeAudit, {
|
|
6016
6855
|
user: sourceContext?.user,
|
|
6017
6856
|
expiresAt: sourceContext.expiresAt,
|
|
6018
6857
|
residencyId,
|
|
6019
6858
|
transaction,
|
|
6020
6859
|
tableToTrack: tableName,
|
|
6860
|
+
additionalAuditRefs: writeAudit && txnLogKey !== recordVersion ? [{ version: txnLogKey, nodeId: 0 }] : undefined,
|
|
6021
6861
|
}, 'put', Boolean(invalidated), auditRecord);
|
|
6022
6862
|
// arm the eviction scanner, mirroring the .put() path
|
|
6023
6863
|
if (sourceContext.expiresAt)
|
|
6024
6864
|
scheduleCleanup();
|
|
6025
6865
|
}
|
|
6026
6866
|
else if (existingEntry) {
|
|
6027
|
-
logger_ts_1.logger.trace?.(`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(
|
|
6867
|
+
logger_ts_1.logger.trace?.(`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`);
|
|
6028
6868
|
if (audit || trackDeletes) {
|
|
6029
|
-
updateRecord(id, null, existingEntry,
|
|
6869
|
+
updateRecord(id, null, existingEntry, recordVersion, 0, (audit && hasChanges) || null, {
|
|
6870
|
+
user: sourceContext?.user,
|
|
6871
|
+
transaction,
|
|
6872
|
+
tableToTrack: tableName,
|
|
6873
|
+
recordVersion,
|
|
6874
|
+
additionalAuditRefs: audit && hasChanges && txnLogKey !== recordVersion
|
|
6875
|
+
? [{ version: txnLogKey, nodeId: 0 }]
|
|
6876
|
+
: undefined,
|
|
6877
|
+
}, 'delete', Boolean(invalidated));
|
|
6030
6878
|
}
|
|
6031
6879
|
else {
|
|
6032
6880
|
(0, RecordEncoder_ts_1.removeEntry)(primaryStore, existingEntry, existingVersion);
|
|
@@ -6186,7 +7034,15 @@ function makeTable(options) {
|
|
|
6186
7034
|
},
|
|
6187
7035
|
};
|
|
6188
7036
|
}
|
|
7037
|
+
function settlePendingCleanup() {
|
|
7038
|
+
for (const resolve of pendingCleanupResolvers)
|
|
7039
|
+
resolve();
|
|
7040
|
+
pendingCleanupResolvers.clear();
|
|
7041
|
+
}
|
|
6189
7042
|
function scheduleCleanup(priority) {
|
|
7043
|
+
// a reclamation run may still hold this class's handler after cleanup(); a promise here would never settle
|
|
7044
|
+
if (disposed)
|
|
7045
|
+
return;
|
|
6190
7046
|
let runImmediately = false;
|
|
6191
7047
|
if (priority) {
|
|
6192
7048
|
// run immediately if there is a big increase in priority
|
|
@@ -6202,9 +7058,19 @@ function makeTable(options) {
|
|
|
6202
7058
|
// run on the last thread so we aren't overloading lower-numbered threads
|
|
6203
7059
|
if (cleanupTimer)
|
|
6204
7060
|
clearTimeout(cleanupTimer);
|
|
6205
|
-
if (!cleanupInterval)
|
|
7061
|
+
if (!cleanupInterval) {
|
|
7062
|
+
// no replacement pass is being scheduled, so nothing is left to settle a superseded one
|
|
7063
|
+
settlePendingCleanup();
|
|
6206
7064
|
return;
|
|
7065
|
+
}
|
|
7066
|
+
// This pass adopts the awaiters of the pass whose timer it just cleared: they settle when
|
|
7067
|
+
// this pass's scan completes, so a reclamation run is never told the storage was reclaimed
|
|
7068
|
+
// before any scan ran. It has to run now, though — that run blocks its whole path on the
|
|
7069
|
+
// promise, and the replacement's own slot can be a full interval out.
|
|
7070
|
+
if (pendingCleanupResolvers.size > 0)
|
|
7071
|
+
runImmediately = true;
|
|
6207
7072
|
return new Promise((resolve) => {
|
|
7073
|
+
pendingCleanupResolvers.add(resolve);
|
|
6208
7074
|
const startOfYear = new Date();
|
|
6209
7075
|
startOfYear.setMonth(0);
|
|
6210
7076
|
startOfYear.setDate(1);
|
|
@@ -6217,6 +7083,8 @@ function makeTable(options) {
|
|
|
6217
7083
|
? Date.now()
|
|
6218
7084
|
: Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime();
|
|
6219
7085
|
const startNextTimer = (nextScheduled) => {
|
|
7086
|
+
if (disposed)
|
|
7087
|
+
return;
|
|
6220
7088
|
logger_ts_1.logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`);
|
|
6221
7089
|
// noinspection JSVoidFunctionReturnValueUsed
|
|
6222
7090
|
cleanupTimer = setTimeout(() => (lastEvictionCompletion = lastEvictionCompletion.then(async () => {
|
|
@@ -6225,8 +7093,11 @@ function makeTable(options) {
|
|
|
6225
7093
|
const rootStore = primaryStore.rootStore;
|
|
6226
7094
|
if (rootStore.status !== 'open') {
|
|
6227
7095
|
clearTimeout(cleanupTimer);
|
|
7096
|
+
settlePendingCleanup();
|
|
6228
7097
|
return;
|
|
6229
7098
|
}
|
|
7099
|
+
// snapshot: an awaiter that arrives during this scan belongs to the pass that supersedes it
|
|
7100
|
+
const settling = [...pendingCleanupResolvers];
|
|
6230
7101
|
const MAX_CLEANUP_CONCURRENCY = 50;
|
|
6231
7102
|
const outstandingCleanupOperations = new Array(MAX_CLEANUP_CONCURRENCY);
|
|
6232
7103
|
let cleanupIndex = 0;
|
|
@@ -6306,7 +7177,10 @@ function makeTable(options) {
|
|
|
6306
7177
|
catch (error) {
|
|
6307
7178
|
logger_ts_1.logger.warn?.(`Error in cleanup scan for ${tableName}:`, error);
|
|
6308
7179
|
}
|
|
6309
|
-
|
|
7180
|
+
for (const settle of settling) {
|
|
7181
|
+
pendingCleanupResolvers.delete(settle);
|
|
7182
|
+
settle();
|
|
7183
|
+
}
|
|
6310
7184
|
cleanupPriority = 0; // reset the priority
|
|
6311
7185
|
})), Math.min(nextScheduled - Date.now(), hdbTerms_ts_1.MAX_SET_TIMEOUT_MS) // make sure it can fit in 32-bit signed number
|
|
6312
7186
|
).unref(); // don't let this prevent closing the thread
|
|
@@ -6317,17 +7191,17 @@ function makeTable(options) {
|
|
|
6317
7191
|
}
|
|
6318
7192
|
function addDeleteRemoval() {
|
|
6319
7193
|
deleteCallbackHandle = auditStore?.addDeleteRemovalCallback(tableId, primaryStore, (id, version) => {
|
|
6320
|
-
primaryStore.remove(id, version);
|
|
7194
|
+
return primaryStore.remove(id, version);
|
|
6321
7195
|
});
|
|
6322
7196
|
}
|
|
6323
7197
|
function runRecordExpirationEviction() {
|
|
6324
7198
|
// Periodically evict expired records, searching for records who expiresAt timestamp is before now
|
|
6325
7199
|
if ((0, manageThreads_js_1.getWorkerIndex)() === 0) {
|
|
6326
7200
|
// we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting
|
|
6327
|
-
setInterval(async () => {
|
|
7201
|
+
recordExpirationInterval = setInterval(async () => {
|
|
6328
7202
|
// go through each database and table and then search for expired entries
|
|
6329
7203
|
// find any entries that are set to expire before now
|
|
6330
|
-
if (runningRecordExpiration)
|
|
7204
|
+
if (disposed || runningRecordExpiration)
|
|
6331
7205
|
return;
|
|
6332
7206
|
runningRecordExpiration = true;
|
|
6333
7207
|
try {
|