@harperfast/harper 5.2.13 → 5.3.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/mcpTools.ts +1 -1
- package/agent/session.ts +25 -14
- package/bin/cliOperations.ts +58 -9
- package/bin/copyDb.ts +282 -59
- package/bin/deploySetup.ts +16 -5
- package/bin/harper.ts +1 -1
- package/bin/help.ts +4 -1
- package/bin/lite.ts +4 -1
- package/bin/restart.ts +120 -4
- package/bin/run.ts +6 -11
- package/bin/upgrade.js +7 -3
- package/bin/workloadIdentity.ts +119 -0
- package/components/Application.ts +3436 -242
- package/components/ApplicationScope.ts +8 -0
- package/components/EntryHandler.ts +59 -39
- package/components/OptionsWatcher.ts +440 -98
- package/components/RuntimeModuleTracker.ts +38 -7
- package/components/Scope.ts +56 -15
- package/components/awaitRestart.ts +84 -0
- package/components/componentLoader.ts +381 -32
- package/components/componentPreparationLock.ts +16 -5
- package/components/deploymentOperations.ts +4 -1
- package/components/deploymentRecorder.ts +9 -2
- package/components/mcp/adapters/harperHttp.ts +4 -0
- package/components/mcp/listChanged.ts +4 -0
- package/components/mcp/toolRegistry.ts +2 -0
- package/components/mcp/tools/operations.ts +9 -0
- package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
- package/components/operations.js +537 -113
- package/components/operationsValidation.js +98 -3
- package/components/packageComponent.ts +25 -1
- package/components/requestRestart.ts +11 -0
- package/components/status/ComponentStatusRegistry.ts +59 -0
- package/config/RootConfigWatcher.ts +240 -40
- package/config/configReadRetry.ts +62 -0
- package/config/configUtils.ts +357 -48
- package/config/harperConfigEnvVars.ts +170 -27
- package/config/parseConfigFile.ts +34 -0
- package/config/readConfigFileSync.ts +44 -0
- package/config/watcherArming.ts +59 -0
- package/config-root.schema.json +33 -0
- package/dataLayer/blobBackup.ts +160 -50
- package/dataLayer/delete.ts +6 -1
- package/dataLayer/harperBridge/ResourceBridge.ts +80 -10
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dataLayer/hdbInfoController.ts +34 -1
- package/dataLayer/insert.ts +44 -1
- package/dataLayer/rocksdbBackup.ts +53 -10
- package/dataLayer/schema.ts +11 -1
- package/dataLayer/schemaDescribe.ts +8 -1
- package/dist/agent/mcpTools.js +1 -1
- package/dist/agent/mcpTools.js.map +1 -1
- package/dist/agent/session.d.ts +22 -0
- package/dist/agent/session.js +26 -15
- package/dist/agent/session.js.map +1 -1
- package/dist/bin/cliOperations.js +61 -9
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/copyDb.d.ts +12 -1
- package/dist/bin/copyDb.js +248 -60
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/bin/deploySetup.d.ts +2 -0
- package/dist/bin/deploySetup.js +11 -3
- package/dist/bin/deploySetup.js.map +1 -1
- package/dist/bin/harper.js +1 -1
- package/dist/bin/harper.js.map +1 -1
- package/dist/bin/help.js +4 -1
- package/dist/bin/help.js.map +1 -1
- package/dist/bin/lite.js +4 -1
- package/dist/bin/lite.js.map +1 -1
- package/dist/bin/restart.js +91 -6
- package/dist/bin/restart.js.map +1 -1
- package/dist/bin/run.js +4 -10
- package/dist/bin/run.js.map +1 -1
- package/dist/bin/upgrade.js +4 -3
- package/dist/bin/upgrade.js.map +1 -1
- package/dist/bin/workloadIdentity.d.ts +18 -0
- package/dist/bin/workloadIdentity.js +100 -0
- package/dist/bin/workloadIdentity.js.map +1 -0
- package/dist/components/Application.d.ts +235 -17
- package/dist/components/Application.js +3038 -238
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +8 -0
- package/dist/components/ApplicationScope.js +7 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/EntryHandler.js +26 -10
- package/dist/components/EntryHandler.js.map +1 -1
- package/dist/components/OptionsWatcher.d.ts +4 -0
- package/dist/components/OptionsWatcher.js +440 -99
- package/dist/components/OptionsWatcher.js.map +1 -1
- package/dist/components/RuntimeModuleTracker.js +40 -6
- package/dist/components/RuntimeModuleTracker.js.map +1 -1
- package/dist/components/Scope.js +52 -13
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/awaitRestart.d.ts +33 -0
- package/dist/components/awaitRestart.js +61 -0
- package/dist/components/awaitRestart.js.map +1 -0
- package/dist/components/componentLoader.d.ts +38 -1
- package/dist/components/componentLoader.js +313 -24
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentPreparationLock.d.ts +5 -0
- package/dist/components/componentPreparationLock.js +14 -6
- package/dist/components/componentPreparationLock.js.map +1 -1
- package/dist/components/deploymentOperations.js +4 -1
- package/dist/components/deploymentOperations.js.map +1 -1
- package/dist/components/deploymentRecorder.d.ts +4 -2
- package/dist/components/deploymentRecorder.js +1 -0
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.js +4 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/listChanged.js +5 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +1 -0
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +5 -0
- package/dist/components/mcp/tools/operations.js +9 -0
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/operations.d.ts +28 -0
- package/dist/components/operations.js +476 -113
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.js +97 -3
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/packageComponent.js +24 -0
- package/dist/components/packageComponent.js.map +1 -1
- package/dist/components/requestRestart.d.ts +1 -0
- package/dist/components/requestRestart.js +7 -0
- package/dist/components/requestRestart.js.map +1 -1
- package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
- package/dist/components/status/ComponentStatusRegistry.js +63 -0
- package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
- package/dist/config/RootConfigWatcher.d.ts +9 -1
- package/dist/config/RootConfigWatcher.js +228 -37
- package/dist/config/RootConfigWatcher.js.map +1 -1
- package/dist/config/configReadRetry.d.ts +8 -0
- package/dist/config/configReadRetry.js +62 -0
- package/dist/config/configReadRetry.js.map +1 -0
- package/dist/config/configUtils.d.ts +11 -2
- package/dist/config/configUtils.js +304 -47
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/harperConfigEnvVars.d.ts +16 -0
- package/dist/config/harperConfigEnvVars.js +162 -25
- package/dist/config/harperConfigEnvVars.js.map +1 -1
- package/dist/config/parseConfigFile.d.ts +4 -0
- package/dist/config/parseConfigFile.js +35 -0
- package/dist/config/parseConfigFile.js.map +1 -0
- package/dist/config/readConfigFileSync.d.ts +1 -0
- package/dist/config/readConfigFileSync.js +47 -0
- package/dist/config/readConfigFileSync.js.map +1 -0
- package/dist/config/watcherArming.d.ts +15 -0
- package/dist/config/watcherArming.js +59 -0
- package/dist/config/watcherArming.js.map +1 -0
- package/dist/dataLayer/blobBackup.d.ts +49 -20
- package/dist/dataLayer/blobBackup.js +139 -50
- package/dist/dataLayer/blobBackup.js.map +1 -1
- package/dist/dataLayer/delete.js +1 -1
- package/dist/dataLayer/delete.js.map +1 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.js +72 -12
- package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
- package/dist/dataLayer/hdbInfoController.d.ts +10 -0
- package/dist/dataLayer/hdbInfoController.js +30 -1
- package/dist/dataLayer/hdbInfoController.js.map +1 -1
- package/dist/dataLayer/insert.d.ts +9 -1
- package/dist/dataLayer/insert.js +30 -0
- package/dist/dataLayer/insert.js.map +1 -1
- package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
- package/dist/dataLayer/rocksdbBackup.js +45 -8
- package/dist/dataLayer/rocksdbBackup.js.map +1 -1
- package/dist/dataLayer/schema.js +8 -0
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +8 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +55 -0
- package/dist/resources/DatabaseTransaction.d.ts +77 -0
- package/dist/resources/DatabaseTransaction.js +586 -54
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +2 -1
- package/dist/resources/LMDBTransaction.js +43 -6
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
- package/dist/resources/PrimaryRocksDatabase.js +49 -5
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
- package/dist/resources/RecordEncoder.d.ts +20 -0
- package/dist/resources/RecordEncoder.js +110 -9
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +2 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.js +117 -24
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +28 -1
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/RocksIndexStore.d.ts +6 -1
- package/dist/resources/RocksIndexStore.js +24 -9
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.d.ts +24 -1
- package/dist/resources/RocksTransactionLogStore.js +160 -49
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.d.ts +165 -10
- package/dist/resources/Table.js +1805 -303
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/write.js +10 -3
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +181 -2
- package/dist/resources/auditStore.js +640 -29
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/blob.d.ts +129 -9
- package/dist/resources/blob.js +992 -114
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/branchDatabase.d.ts +48 -0
- package/dist/resources/branchDatabase.js +892 -0
- package/dist/resources/branchDatabase.js.map +1 -0
- package/dist/resources/crdt.js +50 -12
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/dataLoader.js +3 -4
- package/dist/resources/dataLoader.js.map +1 -1
- package/dist/resources/databases.d.ts +157 -14
- package/dist/resources/databases.js +1462 -325
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/defineTable.d.ts +10 -2
- package/dist/resources/defineTable.js +9 -1
- package/dist/resources/defineTable.js.map +1 -1
- package/dist/resources/derivedIndexRegistry.d.ts +5 -0
- package/dist/resources/derivedIndexRegistry.js +68 -0
- package/dist/resources/derivedIndexRegistry.js.map +1 -0
- package/dist/resources/derivedIndexRuntime.d.ts +215 -0
- package/dist/resources/derivedIndexRuntime.js +2027 -0
- package/dist/resources/derivedIndexRuntime.js.map +1 -0
- package/dist/resources/graphql.d.ts +1 -1
- package/dist/resources/graphql.js +52 -16
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +103 -9
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +854 -41
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
- package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
- package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
- package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
- package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
- package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
- package/dist/resources/longLivedTransactions.d.ts +71 -0
- package/dist/resources/longLivedTransactions.js +358 -0
- package/dist/resources/longLivedTransactions.js.map +1 -0
- package/dist/resources/models/backendRegistry.d.ts +26 -0
- package/dist/resources/models/backendRegistry.js +60 -2
- package/dist/resources/models/backendRegistry.js.map +1 -1
- package/dist/resources/models/bootstrap.d.ts +33 -1
- package/dist/resources/models/bootstrap.js +416 -31
- package/dist/resources/models/bootstrap.js.map +1 -1
- package/dist/resources/nodeIdMapping.d.ts +5 -0
- package/dist/resources/nodeIdMapping.js +49 -0
- package/dist/resources/nodeIdMapping.js.map +1 -1
- package/dist/resources/recordLock.d.ts +123 -0
- package/dist/resources/recordLock.js +315 -0
- package/dist/resources/recordLock.js.map +1 -0
- package/dist/resources/recordLockCoordinator.d.ts +557 -0
- package/dist/resources/recordLockCoordinator.js +2565 -0
- package/dist/resources/recordLockCoordinator.js.map +1 -0
- package/dist/resources/replayLogs.d.ts +14 -1
- package/dist/resources/replayLogs.js +157 -24
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/replayLogsGuards.d.ts +94 -7
- package/dist/resources/replayLogsGuards.js +111 -7
- package/dist/resources/replayLogsGuards.js.map +1 -1
- package/dist/resources/replicatedApplyFailure.d.ts +16 -0
- package/dist/resources/replicatedApplyFailure.js +63 -0
- package/dist/resources/replicatedApplyFailure.js.map +1 -0
- package/dist/resources/scheduler/scheduler.js +3 -3
- package/dist/resources/scheduler/scheduler.js.map +1 -1
- package/dist/resources/search.d.ts +10 -4
- package/dist/resources/search.js +283 -37
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/tracked.d.ts +5 -1
- package/dist/resources/tracked.js +74 -23
- package/dist/resources/tracked.js.map +1 -1
- package/dist/resources/transactionBroadcast.d.ts +1 -1
- package/dist/resources/transactionBroadcast.js +2 -2
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/auth.js +61 -30
- package/dist/security/auth.js.map +1 -1
- package/dist/security/authn/oidc/claims.d.ts +22 -0
- package/dist/security/authn/oidc/claims.js +71 -0
- package/dist/security/authn/oidc/claims.js.map +1 -0
- package/dist/security/authn/oidc/identityToken.d.ts +27 -0
- package/dist/security/authn/oidc/identityToken.js +111 -0
- package/dist/security/authn/oidc/identityToken.js.map +1 -0
- package/dist/security/authn/oidc/jwks.d.ts +25 -0
- package/dist/security/authn/oidc/jwks.js +261 -0
- package/dist/security/authn/oidc/jwks.js.map +1 -0
- package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
- package/dist/security/authn/oidc/providers/generic.js +34 -0
- package/dist/security/authn/oidc/providers/generic.js.map +1 -0
- package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
- package/dist/security/authn/oidc/providers/githubActions.js +129 -0
- package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
- package/dist/security/authn/oidc/providers/index.d.ts +37 -0
- package/dist/security/authn/oidc/providers/index.js +24 -0
- package/dist/security/authn/oidc/providers/index.js.map +1 -0
- package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
- package/dist/security/authn/oidc/tokenExchange.js +306 -0
- package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
- package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
- package/dist/security/authn/oidc/types.d.ts +38 -0
- package/dist/security/authn/oidc/types.js +6 -0
- package/dist/security/authn/oidc/types.js.map +1 -0
- package/dist/security/certificateVerification/index.js +40 -11
- package/dist/security/certificateVerification/index.js.map +1 -1
- package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
- package/dist/security/certificateVerification/trustedIssuers.js +79 -0
- package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
- package/dist/security/certificateVerification/types.d.ts +1 -0
- package/dist/security/credentialProvenance.d.ts +35 -0
- package/dist/security/credentialProvenance.js +51 -0
- package/dist/security/credentialProvenance.js.map +1 -0
- package/dist/security/credentialRejection.d.ts +4 -0
- package/dist/security/credentialRejection.js +24 -0
- package/dist/security/credentialRejection.js.map +1 -0
- package/dist/security/deferredAuthentication.d.ts +36 -0
- package/dist/security/deferredAuthentication.js +70 -0
- package/dist/security/deferredAuthentication.js.map +1 -0
- package/dist/security/impersonation.d.ts +21 -0
- package/dist/security/impersonation.js +108 -9
- package/dist/security/impersonation.js.map +1 -1
- package/dist/security/jsLoader.d.ts +6 -0
- package/dist/security/jsLoader.js +77 -15
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +301 -71
- package/dist/security/keys.js.map +1 -1
- package/dist/security/operationScope.d.ts +21 -0
- package/dist/security/operationScope.js +36 -0
- package/dist/security/operationScope.js.map +1 -0
- package/dist/security/permissionsTranslator.js +21 -0
- package/dist/security/permissionsTranslator.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +19 -1
- package/dist/security/tokenAuthentication.js +191 -10
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/security/user.js +4 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
- package/dist/server/DurableSubscriptionsSession.js +67 -10
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +106 -2
- package/dist/server/REST.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -0
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +16 -1
- package/dist/server/http.js +122 -17
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +8 -1
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/jobs/jobProcess.js +6 -2
- package/dist/server/jobs/jobProcess.js.map +1 -1
- package/dist/server/jobs/jobs.js +4 -1
- package/dist/server/jobs/jobs.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +26 -4
- package/dist/server/liveSubscriptionAuth.js +105 -39
- package/dist/server/liveSubscriptionAuth.js.map +1 -1
- package/dist/server/loadRootComponents.js +49 -10
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/mqtt.d.ts +2 -0
- package/dist/server/mqtt.js +165 -30
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/nodeName.d.ts +2 -0
- package/dist/server/nodeName.js +107 -23
- package/dist/server/nodeName.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +27 -0
- package/dist/server/serverHelpers/Headers.js +145 -1
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
- package/dist/server/serverHelpers/Request.d.ts +5 -10
- package/dist/server/serverHelpers/Request.js +38 -136
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.d.ts +11 -0
- package/dist/server/serverHelpers/contentTypes.js +221 -40
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
- package/dist/server/serverHelpers/deployValidationState.js +9 -19
- package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
- package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
- package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
- package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
- package/dist/server/serverHelpers/registeredOperations.js +74 -21
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
- package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
- package/dist/server/serverHelpers/requestSanitization.js +20 -0
- package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +6 -3
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
- package/dist/server/serverHelpers/serverUtilities.js +178 -29
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
- package/dist/server/serverHelpers/uwsServer.js +19 -1
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/static.js +24 -28
- package/dist/server/static.js.map +1 -1
- package/dist/server/storageReclamation.d.ts +5 -0
- package/dist/server/storageReclamation.js +17 -1
- package/dist/server/storageReclamation.js.map +1 -1
- package/dist/server/threads/isolatedApplications.d.ts +47 -0
- package/dist/server/threads/isolatedApplications.js +171 -0
- package/dist/server/threads/isolatedApplications.js.map +1 -0
- package/dist/server/threads/itc.d.ts +7 -2
- package/dist/server/threads/itc.js +5 -1
- package/dist/server/threads/itc.js.map +1 -1
- package/dist/server/threads/logRotationTransport.d.ts +1 -0
- package/dist/server/threads/logRotationTransport.js +33 -0
- package/dist/server/threads/logRotationTransport.js.map +1 -0
- package/dist/server/threads/manageThreads.d.ts +83 -3
- package/dist/server/threads/manageThreads.js +769 -75
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.d.ts +1 -0
- package/dist/server/threads/socketRouter.js +277 -31
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadHeapMemory.d.ts +2 -0
- package/dist/server/threads/threadHeapMemory.js +31 -0
- package/dist/server/threads/threadHeapMemory.js.map +1 -0
- package/dist/server/threads/threadServer.js +76 -22
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/sqlEngine/config.d.ts +1 -3
- package/dist/sqlEngine/config.js +19 -16
- package/dist/sqlEngine/config.js.map +1 -1
- package/dist/sqlTranslator/index.d.ts +1 -1
- package/dist/sqlTranslator/index.js +30 -7
- package/dist/sqlTranslator/index.js.map +1 -1
- package/dist/upgrade/directives/5-3-0.d.ts +7 -0
- package/dist/upgrade/directives/5-3-0.js +148 -0
- package/dist/upgrade/directives/5-3-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/OperationFunctionCaller.js +2 -1
- package/dist/utility/OperationFunctionCaller.js.map +1 -1
- package/dist/utility/common_utils.d.ts +16 -0
- package/dist/utility/common_utils.js +32 -6
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/componentNames.d.ts +8 -0
- package/dist/utility/componentNames.js +12 -1
- package/dist/utility/componentNames.js.map +1 -1
- package/dist/utility/environment/environmentManager.js +3 -6
- package/dist/utility/environment/environmentManager.js.map +1 -1
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/environment/systemInformation.js +1 -0
- package/dist/utility/environment/systemInformation.js.map +1 -1
- package/dist/utility/errors/commonErrors.d.ts +2 -0
- package/dist/utility/errors/commonErrors.js +2 -0
- package/dist/utility/errors/commonErrors.js.map +1 -1
- package/dist/utility/errors/hdbError.d.ts +33 -0
- package/dist/utility/errors/hdbError.js +58 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/globalSchema.d.ts +18 -0
- package/dist/utility/hdbTerms.d.ts +18 -0
- package/dist/utility/hdbTerms.js +20 -2
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +2 -0
- package/dist/utility/logging/harper_logger.js +286 -30
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
- package/dist/utility/logging/logGenerationCoordinator.js +184 -0
- package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
- package/dist/utility/logging/logRotation.d.ts +46 -0
- package/dist/utility/logging/logRotation.js +365 -0
- package/dist/utility/logging/logRotation.js.map +1 -0
- package/dist/utility/logging/logRotator.d.ts +1 -1
- package/dist/utility/logging/logRotator.js +192 -85
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/nodeIdentity.d.ts +9 -0
- package/dist/utility/nodeIdentity.js +58 -0
- package/dist/utility/nodeIdentity.js.map +1 -0
- package/dist/utility/npmUtilities.js +11 -7
- package/dist/utility/npmUtilities.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +3 -1
- package/dist/utility/operationPermissions.js +16 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +10 -7
- package/dist/utility/operation_authorization.js +212 -41
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/watchPath.d.ts +29 -0
- package/dist/utility/watchPath.js +68 -0
- package/dist/utility/watchPath.js.map +1 -0
- package/dist/utility/watcherFallback.d.ts +41 -0
- package/dist/utility/watcherFallback.js +153 -0
- package/dist/utility/watcherFallback.js.map +1 -1
- package/dist/validation/configValidator.d.ts +12 -0
- package/dist/validation/configValidator.js +205 -75
- package/dist/validation/configValidator.js.map +1 -1
- package/dist/validation/installValidator.js +12 -0
- package/dist/validation/installValidator.js.map +1 -1
- package/dist/validation/validationWrapper.d.ts +11 -0
- package/dist/validation/validationWrapper.js +16 -3
- package/dist/validation/validationWrapper.js.map +1 -1
- package/index.ts +8 -0
- package/json/systemSchema.json +55 -0
- package/npm-shrinkwrap.json +286 -194
- package/package.json +15 -7
- package/resources/DESIGN.md +219 -52
- package/resources/DatabaseTransaction.ts +657 -52
- package/resources/LMDBTransaction.ts +46 -6
- package/resources/PrimaryRocksDatabase.ts +46 -7
- package/resources/RecordEncoder.ts +124 -9
- package/resources/RequestTarget.ts +2 -0
- package/resources/Resource.ts +114 -22
- package/resources/ResourceInterface.ts +31 -0
- package/resources/RocksIndexStore.ts +30 -9
- package/resources/RocksTransactionLogStore.ts +188 -51
- package/resources/Table.ts +1960 -301
- package/resources/analytics/write.ts +10 -3
- package/resources/auditStore.ts +647 -32
- package/resources/blob.ts +1029 -110
- package/resources/branchDatabase.ts +941 -0
- package/resources/crdt.ts +76 -12
- package/resources/dataLoader.ts +3 -4
- package/resources/databases.ts +1584 -320
- package/resources/defineTable.ts +18 -2
- package/resources/derivedIndexRegistry.ts +56 -0
- package/resources/derivedIndexRuntime.ts +2292 -0
- package/resources/graphql.ts +72 -17
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +911 -48
- package/resources/indexes/hnswDerivedIndex.ts +531 -0
- package/resources/indexes/hnswPlaneBinding.ts +174 -0
- package/resources/longLivedTransactions.ts +360 -0
- package/resources/models/backendRegistry.ts +84 -2
- package/resources/models/bootstrap.ts +473 -28
- package/resources/nodeIdMapping.ts +50 -0
- package/resources/recordLock.ts +419 -0
- package/resources/recordLockCoordinator.ts +3043 -0
- package/resources/replayLogs.ts +152 -26
- package/resources/replayLogsGuards.ts +171 -8
- package/resources/replicatedApplyFailure.ts +77 -0
- package/resources/scheduler/scheduler.ts +4 -4
- package/resources/search.ts +288 -46
- package/resources/tracked.ts +73 -22
- package/resources/transactionBroadcast.ts +3 -3
- package/security/auth.ts +68 -29
- package/security/authn/oidc/claims.ts +72 -0
- package/security/authn/oidc/identityToken.ts +129 -0
- package/security/authn/oidc/jwks.ts +260 -0
- package/security/authn/oidc/providers/generic.ts +40 -0
- package/security/authn/oidc/providers/githubActions.ts +137 -0
- package/security/authn/oidc/providers/index.ts +52 -0
- package/security/authn/oidc/tokenExchange.ts +300 -0
- package/security/authn/oidc/trustPolicyOperations.ts +343 -0
- package/security/authn/oidc/types.ts +41 -0
- package/security/certificateVerification/index.ts +54 -13
- package/security/certificateVerification/trustedIssuers.ts +76 -0
- package/security/certificateVerification/types.ts +1 -0
- package/security/credentialProvenance.ts +47 -0
- package/security/credentialRejection.ts +22 -0
- package/security/deferredAuthentication.ts +71 -0
- package/security/impersonation.ts +117 -12
- package/security/jsLoader.ts +83 -18
- package/security/keys.ts +298 -72
- package/security/operationScope.ts +33 -0
- package/security/permissionsTranslator.js +23 -0
- package/security/tokenAuthentication.ts +233 -12
- package/security/user.ts +4 -3
- package/server/DESIGN.md +194 -16
- package/server/DurableSubscriptionsSession.ts +71 -11
- package/server/REST.ts +115 -4
- package/server/graphqlQuerying.ts +4 -0
- package/server/http.ts +133 -20
- package/server/itc/serverHandlers.js +8 -1
- package/server/jobs/jobProcess.ts +8 -2
- package/server/jobs/jobs.ts +4 -1
- package/server/liveSubscriptionAuth.ts +129 -46
- package/server/loadRootComponents.js +50 -8
- package/server/mqtt.ts +179 -38
- package/server/nodeName.ts +103 -21
- package/server/serverHelpers/Headers.ts +135 -0
- package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
- package/server/serverHelpers/Request.ts +33 -131
- package/server/serverHelpers/contentTypes.ts +217 -36
- package/server/serverHelpers/deployValidationState.ts +24 -13
- package/server/serverHelpers/operationAuthorizationState.ts +34 -3
- package/server/serverHelpers/registeredOperations.ts +79 -22
- package/server/serverHelpers/requestSanitization.ts +15 -0
- package/server/serverHelpers/serverHandlers.js +6 -3
- package/server/serverHelpers/serverUtilities.ts +232 -40
- package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
- package/server/serverHelpers/uwsServer.ts +17 -2
- package/server/static.ts +23 -29
- package/server/storageReclamation.ts +15 -2
- package/server/threads/isolatedApplications.ts +157 -0
- package/server/threads/itc.js +11 -1
- package/server/threads/logRotationTransport.ts +40 -0
- package/server/threads/manageThreads.js +771 -66
- package/server/threads/socketRouter.ts +291 -30
- package/server/threads/threadHeapMemory.ts +26 -0
- package/server/threads/threadServer.js +73 -22
- package/sqlTranslator/index.ts +31 -8
- package/studio/web/assets/{Chat-4RrB5134.js → Chat-D3j-1yY1.js} +1 -1
- package/studio/web/assets/{FloatingChat-omlNMDcJ.js → FloatingChat-BxJGYcfB.js} +3 -3
- package/studio/web/assets/{apiToken-Bke3wvfZ.js → apiToken-CT55oWOe.js} +1 -1
- package/studio/web/assets/{applications-DvFDYJgK.js → applications-D9Ct9_vm.js} +1 -1
- package/studio/web/assets/{cssMode-C1vRa7zh.js → cssMode-DV8H7VwA.js} +1 -1
- package/studio/web/assets/{editor-3XRWEDWX.js → editor-uatc0unt.js} +1 -1
- package/studio/web/assets/{html-CTY5tdMr.js → html-Bm6D6paN.js} +1 -1
- package/studio/web/assets/{htmlMode-CG1vSD9t.js → htmlMode-CEn7tpLG.js} +1 -1
- package/studio/web/assets/{index-Dfpeofdu.js → index-BIXW6Pu4.js} +5 -5
- package/studio/web/assets/{index.lazy-B1VOIv-t.js → index.lazy-UI7L-Vrk.js} +1 -1
- package/studio/web/assets/{javascript-CJeJzGnI.js → javascript-CJ0G3AFZ.js} +1 -1
- package/studio/web/assets/{jsonMode-CYPBwM82.js → jsonMode-DQADAYEa.js} +1 -1
- package/studio/web/assets/{languageServices-IKH4GUHl.js → languageServices-CAQJXWcI.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-BC8_gxKG.js → lspLanguageFeatures-CCQ8P5sY.js} +1 -1
- package/studio/web/assets/{notifications-BfKBpYcq.js → notifications-BbxTU6Aw.js} +1 -1
- package/studio/web/assets/{notifications-DFbArTfC.js → notifications-Cvb3P1lB.js} +1 -1
- package/studio/web/assets/{profile-Q4-T6c-S.js → profile-Yyb7gsvL.js} +1 -1
- package/studio/web/assets/{regions-CtkV0xje.js → regions-OgjGHlU5.js} +1 -1
- package/studio/web/assets/{register-1ZZuMsiA.js → register-6qwNEOY3.js} +2 -2
- package/studio/web/assets/{setComponentFile-ZWLcWv5X.js → setComponentFile-BilDMtgB.js} +1 -1
- package/studio/web/assets/{setup-DWprJyJy.js → setup-J6qJ7OIU.js} +2 -2
- package/studio/web/assets/{status--aNm8isn.js → status-0RWGcfyD.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-Cq_lt3XD.js → toggleHighContrast-BIn-vErT.js} +1 -1
- package/studio/web/assets/{tsMode-pjgytARx.js → tsMode-DgUXku4d.js} +1 -1
- package/studio/web/assets/{typescript-Cdg0mqUh.js → typescript-C9orXcsM.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-RZhaY8Rn.js → useEntityRestURL-BEoXXbUB.js} +1 -1
- package/studio/web/assets/{workers-BgoXIqQe.js → workers-JVzSDmgx.js} +1 -1
- package/studio/web/assets/{xml-BwSeDMiP.js → xml-Cq-S8S4X.js} +1 -1
- package/studio/web/assets/{yaml-DotCUG5l.js → yaml-sfoRdh1M.js} +1 -1
- package/studio/web/index.html +1 -1
- package/upgrade/directives/5-3-0.ts +132 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/OperationFunctionCaller.ts +2 -1
- package/utility/common_utils.ts +30 -5
- package/utility/componentNames.ts +12 -0
- package/utility/environment/environmentManager.ts +3 -7
- package/utility/environment/systemInformation.ts +7 -0
- package/utility/errors/commonErrors.ts +4 -0
- package/utility/errors/hdbError.ts +57 -0
- package/utility/hdbTerms.ts +19 -0
- package/utility/logging/harper_logger.ts +278 -26
- package/utility/logging/logGenerationCoordinator.ts +196 -0
- package/utility/logging/logRotation.ts +367 -0
- package/utility/logging/logRotator.ts +213 -81
- package/utility/nodeIdentity.ts +45 -0
- package/utility/npmUtilities.ts +12 -8
- package/utility/operationPermissions.ts +18 -1
- package/utility/operation_authorization.ts +231 -42
- package/utility/watchPath.ts +63 -0
- package/utility/watcherFallback.ts +148 -0
- package/validation/configValidator.ts +215 -75
- package/validation/installValidator.ts +15 -0
- package/validation/validationWrapper.ts +18 -4
|
@@ -33,12 +33,18 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.Decoder = exports.LOCAL_ONLY = exports.HAS_ADDITIONAL_AUDIT_REFS = exports.HAS_BLOBS = exports.HAS_EXPIRATION_EXTENDED_TYPE = exports.HAS_ORIGINATING_OPERATION = exports.HAS_PREVIOUS_RESIDENCY_ID = exports.HAS_CURRENT_RESIDENCY_ID = exports.REMOTE_SEQUENCE_UPDATE = exports.ACTION_64_BIT = exports.ACTION_32_BIT = exports.auditRetention = exports.AUDIT_STORE_OPTIONS = exports.transactionKeyEncoder = exports.ENTRY_DATAVIEW = void 0;
|
|
36
|
+
exports.Decoder = exports.LOCAL_ONLY = exports.HAS_ADDITIONAL_AUDIT_REFS = exports.HAS_BLOBS = exports.HAS_EXPIRATION_EXTENDED_TYPE = exports.HAS_ORIGINATING_OPERATION = exports.HAS_PREVIOUS_RESIDENCY_ID = exports.HAS_CURRENT_RESIDENCY_ID = exports.LOCK_BARRIER = exports.LOCK_RELEASE = exports.REMOTE_SEQUENCE_UPDATE = exports.ACTION_64_BIT = exports.ACTION_32_BIT = exports.auditRetention = exports.AUDIT_STORE_OPTIONS = exports.transactionKeyEncoder = exports.ENTRY_DATAVIEW = void 0;
|
|
37
37
|
exports.openAuditStore = openAuditStore;
|
|
38
|
+
exports.isAuditEntryWrite = isAuditEntryWrite;
|
|
38
39
|
exports.removeAuditEntry = removeAuditEntry;
|
|
39
40
|
exports.getLastRemoved = getLastRemoved;
|
|
41
|
+
exports.boundedAuditPruneEnd = boundedAuditPruneEnd;
|
|
42
|
+
exports.raiseAuditFloor = raiseAuditFloor;
|
|
43
|
+
exports.establishAuditFloor = establishAuditFloor;
|
|
44
|
+
exports.getAuditFloor = getAuditFloor;
|
|
40
45
|
exports.setAuditRetention = setAuditRetention;
|
|
41
46
|
exports.purgeAgedLogs = purgeAgedLogs;
|
|
47
|
+
exports.isLockControlType = isLockControlType;
|
|
42
48
|
exports.createAuditEntry = createAuditEntry;
|
|
43
49
|
exports.readAuditEntry = readAuditEntry;
|
|
44
50
|
const ordered_binary_1 = require("ordered-binary");
|
|
@@ -53,6 +59,7 @@ const crdt_ts_1 = require("./crdt.js");
|
|
|
53
59
|
const blob_ts_1 = require("./blob.js");
|
|
54
60
|
const storageReclamation_ts_1 = require("../server/storageReclamation.js");
|
|
55
61
|
const rocksdb_js_1 = require("@harperfast/rocksdb-js");
|
|
62
|
+
const lmdb_1 = require("lmdb");
|
|
56
63
|
const RocksTransactionLogStore_ts_1 = require("./RocksTransactionLogStore.js");
|
|
57
64
|
const databases_ts_1 = require("./databases.js");
|
|
58
65
|
/**
|
|
@@ -127,8 +134,40 @@ const MAX_CLEANUP_DELAY = 2 ** 31 - 1;
|
|
|
127
134
|
// mint latch keyed per (table, type) so one entry type can't silence another's producer stack
|
|
128
135
|
const warnedBodylessMints = new Set();
|
|
129
136
|
const warnedBodylessTables = new Set();
|
|
137
|
+
// legacy-format latches, mirroring the bodyless ones above: a range scan over millions of pre-#2247
|
|
138
|
+
// entries must not turn a recovered read into a log flood. Recovery latches per table (the table is
|
|
139
|
+
// known by then); an undecodable header has no table, so it latches once per process.
|
|
140
|
+
const warnedRecoveredTables = new Set();
|
|
141
|
+
let warnedUndecodableHeader = false;
|
|
142
|
+
// keyed per (table, type) like warnedBodylessMints, so one entry type cannot silence another's stack
|
|
143
|
+
const warnedPendingPreviousVersion = new Set();
|
|
130
144
|
const FLOAT_TARGET = new Float64Array(1);
|
|
131
145
|
const FLOAT_BUFFER = new Uint8Array(FLOAT_TARGET.buffer);
|
|
146
|
+
/**
|
|
147
|
+
* Key of this database's audit retention floor — the staleness horizon `getAuditFloor` reports.
|
|
148
|
+
* Its *presence* is what marks the floor trustworthy, which is why it is not the `last-removed`
|
|
149
|
+
* marker above: that one is written after its removals and by only one of the five prune paths, so a
|
|
150
|
+
* value found there cannot be told apart from one carrying the write-ahead and monotonicity
|
|
151
|
+
* guarantees `raiseAuditFloor` provides. The two coexist deliberately and answer different
|
|
152
|
+
* questions.
|
|
153
|
+
*/
|
|
154
|
+
const AUDIT_FLOOR_KEY = Symbol.for('audit-floor');
|
|
155
|
+
// The epoch `establishAuditFloor` stamped, never raised or removed. Its PRESENCE marks the floor as
|
|
156
|
+
// unverified provenance — a guess bounded by what survived, blind to history a legacy prune removed
|
|
157
|
+
// before tracking began — until a database generation (#2451) retires the mark. No comparison does:
|
|
158
|
+
// a later prune certifies only what it removed, so `floor > bootstrap` says nothing about the older
|
|
159
|
+
// gap. The value records how far the guess reached, for that repair. See `establishAuditFloor`.
|
|
160
|
+
const AUDIT_FLOOR_BOOTSTRAP_KEY = Symbol.for('audit-floor-bootstrap');
|
|
161
|
+
/**
|
|
162
|
+
* The floor's own eight bytes, deliberately NOT the FLOAT_TARGET/FLOAT_BUFFER pair the `last-removed`
|
|
163
|
+
* marker uses: decoding a floor writes into its buffer on every read, including the pre-check on each
|
|
164
|
+
* prune, while `updateLastRemoved` hands the shared buffer to an async `put` it has not yet consumed.
|
|
165
|
+
* Sharing them would let a floor read rewrite a marker still in flight.
|
|
166
|
+
*/
|
|
167
|
+
const FLOOR_TARGET = new Float64Array(1);
|
|
168
|
+
const FLOOR_BUFFER = new Uint8Array(FLOOR_TARGET.buffer);
|
|
169
|
+
/** No trustworthy floor: the highest possible floor, so every cursor compares as stale. */
|
|
170
|
+
const AUDIT_FLOOR_UNKNOWN = Infinity;
|
|
132
171
|
/** Last resort on a detached path: a failing log sink must not itself become an unhandled rejection. */
|
|
133
172
|
function warnContained(message, error) {
|
|
134
173
|
try {
|
|
@@ -160,13 +199,14 @@ function openAuditStore(rootStore) {
|
|
|
160
199
|
if (options.values === false)
|
|
161
200
|
return superGetRange(options); // getKeys shouldn't be modified
|
|
162
201
|
return superGetRange(options).map(({ key, value }) => {
|
|
163
|
-
value.key = value.
|
|
202
|
+
value.key = value.txnLogKey = key;
|
|
164
203
|
return value;
|
|
165
204
|
});
|
|
166
205
|
};
|
|
167
206
|
}
|
|
168
207
|
rootStore.auditStore = auditStore;
|
|
169
208
|
auditStore.rootStore = rootStore;
|
|
209
|
+
establishAuditFloor(auditStore);
|
|
170
210
|
auditStore.tableStores = [];
|
|
171
211
|
const deleteCallbacks = [];
|
|
172
212
|
auditStore.addDeleteRemovalCallback = function (tableId, table, callback) {
|
|
@@ -176,6 +216,8 @@ function openAuditStore(rootStore) {
|
|
|
176
216
|
return {
|
|
177
217
|
remove() {
|
|
178
218
|
delete deleteCallbacks[tableId];
|
|
219
|
+
if (auditStore.tableStores[tableId] === table)
|
|
220
|
+
delete auditStore.tableStores[tableId];
|
|
179
221
|
},
|
|
180
222
|
};
|
|
181
223
|
};
|
|
@@ -240,26 +282,40 @@ function openAuditStore(rootStore) {
|
|
|
240
282
|
let lastKey;
|
|
241
283
|
try {
|
|
242
284
|
if (isRocksAuditStore) {
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
});
|
|
285
|
+
const before = retentionCutoff(1 + passCleanupPriority * passCleanupPriority);
|
|
286
|
+
raiseAuditFloor(auditStore, before);
|
|
287
|
+
auditStore.rootStore.purgeLogs({ before });
|
|
246
288
|
}
|
|
247
289
|
else {
|
|
248
290
|
// Driven explicitly rather than with for-of: this loop suspends on the awaits below, and a
|
|
249
291
|
// close landing mid-pass closes the env under it. for-of calls next() before the body, so a
|
|
250
292
|
// check inside the body advances the cursor first — the guard has to precede every next().
|
|
293
|
+
// remove up until the audit retention time, reducing audit retention time if cleanup is higher priority
|
|
294
|
+
const end = retentionCutoff(1 + passCleanupPriority * passCleanupPriority);
|
|
295
|
+
// Probe before raising, so an idle database does not write a floor transaction on every
|
|
296
|
+
// pass forever. `end` is fixed and audit keys only move forward, so an empty probe means
|
|
297
|
+
// the loop below finds nothing either.
|
|
251
298
|
const entries = auditStore
|
|
252
299
|
.getRange({
|
|
253
300
|
start: 1, // must not be zero or it will be interpreted as null and overlap with symbols in search
|
|
254
301
|
snapshot: false,
|
|
255
|
-
end
|
|
302
|
+
end,
|
|
256
303
|
})[Symbol.iterator]();
|
|
257
304
|
try {
|
|
305
|
+
// Raised off the first eligible entry rather than a separate probe range: one cursor,
|
|
306
|
+
// so a pass over an idle database writes no floor at all, and nothing else observing
|
|
307
|
+
// this range sees an extra advance. Still strictly before any removal — see
|
|
308
|
+
// raiseAuditFloor.
|
|
309
|
+
let floorRaised = false;
|
|
258
310
|
while (!cleanupStopped && !storeClosing()) {
|
|
259
311
|
const entry = entries.next();
|
|
260
312
|
if (entry.done)
|
|
261
313
|
break;
|
|
262
314
|
const auditRecord = entry.value;
|
|
315
|
+
if (!floorRaised) {
|
|
316
|
+
raiseAuditFloor(auditStore, end);
|
|
317
|
+
floorRaised = true;
|
|
318
|
+
}
|
|
263
319
|
try {
|
|
264
320
|
// awaited so a rejection (not just a synchronous throw) is caught here instead of
|
|
265
321
|
// escaping as an unhandled rejection once a later iteration's promise replaces this one
|
|
@@ -338,7 +394,7 @@ function openAuditStore(rootStore) {
|
|
|
338
394
|
// sites already restrict Rocks to the last worker
|
|
339
395
|
if (!cleanupStopped &&
|
|
340
396
|
!storeClosing() &&
|
|
341
|
-
(!isRocksAuditStore || ((0, manageThreads_js_1.
|
|
397
|
+
(!isRocksAuditStore || ((0, manageThreads_js_1.ownsStoreMaintenance)(rootStore.path) && !pendingCleanupResolve))) {
|
|
342
398
|
scheduleAuditCleanup();
|
|
343
399
|
}
|
|
344
400
|
}
|
|
@@ -374,7 +430,7 @@ function openAuditStore(rootStore) {
|
|
|
374
430
|
pendingCleanupResolve = null;
|
|
375
431
|
return lastCleanupResolution ?? Promise.resolve();
|
|
376
432
|
};
|
|
377
|
-
if ((0, manageThreads_js_1.
|
|
433
|
+
if ((0, manageThreads_js_1.ownsStoreMaintenance)(rootStore.path)) {
|
|
378
434
|
scheduleAuditCleanup();
|
|
379
435
|
}
|
|
380
436
|
if ((0, manageThreads_js_1.getWorkerIndex)() === 0 && !timestampErrored) {
|
|
@@ -388,16 +444,42 @@ function openAuditStore(rootStore) {
|
|
|
388
444
|
}
|
|
389
445
|
return auditStore;
|
|
390
446
|
}
|
|
447
|
+
/**
|
|
448
|
+
* Whether `entry` is the very write `auditRecord` describes. Write identity is (origin node, log key),
|
|
449
|
+
* never the record version: a version is legitimately non-unique, so a version match can name a
|
|
450
|
+
* different write and authorize destroying live state (a tombstone, a still-referenced blob).
|
|
451
|
+
*
|
|
452
|
+
* On LMDB this is exact — the audit-store key IS the record's `txnLogKey`. On RocksDB a record stores
|
|
453
|
+
* its version in the compatibility word and keeps a divergent log key in `additionalAuditRefs`.
|
|
454
|
+
* Absent identity answers false. The direction is deliberate: uncertainty retains.
|
|
455
|
+
*/
|
|
456
|
+
function isAuditEntryWrite(entry, auditRecord) {
|
|
457
|
+
if (entry == null || auditRecord.txnLogKey == null)
|
|
458
|
+
return false;
|
|
459
|
+
const auditNodeId = auditRecord.nodeId ?? 0;
|
|
460
|
+
return ((entry.localTime === auditRecord.txnLogKey && (entry.nodeId ?? 0) === auditNodeId) ||
|
|
461
|
+
entry.additionalAuditRefs?.some((ref) => ref.version === auditRecord.txnLogKey && (ref.nodeId ?? 0) === auditNodeId) === true);
|
|
462
|
+
}
|
|
391
463
|
function removeAuditEntry(auditStore, auditRecord) {
|
|
464
|
+
let tombstoneRemoval;
|
|
392
465
|
if (auditRecord.type === 'delete') {
|
|
393
466
|
// if this is a delete, we remove the delete entry from the primary table
|
|
394
|
-
// at the same time so the audit table the primary table are in sync, assuming the entry
|
|
467
|
+
// at the same time so the audit table the primary table are in sync, assuming the entry is still
|
|
468
|
+
// the record state this audit record wrote
|
|
395
469
|
const tableId = auditRecord.tableId;
|
|
396
470
|
const primaryStore = auditStore.tableStores[auditRecord.tableId];
|
|
397
|
-
|
|
398
|
-
|
|
471
|
+
const tombstone = primaryStore?.getEntry(auditRecord.recordId);
|
|
472
|
+
if (isAuditEntryWrite(tombstone, auditRecord))
|
|
473
|
+
// a failed tombstone removal doesn't mean the audit entry removal failed — only
|
|
474
|
+
// auditStore.remove() below decides this function's outcome
|
|
475
|
+
tombstoneRemoval = new Promise((resolve) => {
|
|
476
|
+
resolve(auditStore.deleteCallbacks?.[tableId]?.(auditRecord.recordId, tombstone.version));
|
|
477
|
+
}).catch((error) => {
|
|
478
|
+
harperLogger.warn('Error removing deleted record while removing its audit entry', error);
|
|
479
|
+
});
|
|
399
480
|
}
|
|
400
|
-
|
|
481
|
+
const auditRemoval = auditStore.remove(auditRecord.key);
|
|
482
|
+
return tombstoneRemoval ? Promise.all([tombstoneRemoval, auditRemoval]).then(() => undefined) : auditRemoval;
|
|
401
483
|
}
|
|
402
484
|
function updateLastRemoved(auditStore, lastKey) {
|
|
403
485
|
FLOAT_TARGET[0] = lastKey;
|
|
@@ -410,10 +492,371 @@ function getLastRemoved(auditStore) {
|
|
|
410
492
|
return FLOAT_TARGET[0];
|
|
411
493
|
}
|
|
412
494
|
}
|
|
495
|
+
/**
|
|
496
|
+
* Read the recorded floor, normalizing anything we cannot trust to AUDIT_FLOOR_UNKNOWN. Reads bytes
|
|
497
|
+
* rather than going through the store's value decoder, which would read these eight raw float bytes
|
|
498
|
+
* as an audit entry (and as msgpack on RocksDB). Callers get a number in every case: a NaN or
|
|
499
|
+
* negative floor read as a number would make one of `cursor >= floor` / `cursor < floor` report
|
|
500
|
+
* safety, and which of the two a consumer writes must not decide whether corrupt metadata fails
|
|
501
|
+
* closed.
|
|
502
|
+
*/
|
|
503
|
+
function decodeAuditFloor(stored) {
|
|
504
|
+
// RocksDB's getBinarySync is typed to also return a length; anything that is not exactly the
|
|
505
|
+
// eight bytes we write is metadata written by something else.
|
|
506
|
+
if (stored?.byteLength !== 8)
|
|
507
|
+
return AUDIT_FLOOR_UNKNOWN;
|
|
508
|
+
FLOOR_BUFFER.set(stored);
|
|
509
|
+
const floor = FLOOR_TARGET[0];
|
|
510
|
+
// `Object.is` for -0, which passes `>= 0` and would then read as a permissive zero — every cursor
|
|
511
|
+
// safe — from bytes with the sign bit set that nothing here writes. raiseAuditFloor rejects the
|
|
512
|
+
// same value as a cutoff; the read side has to agree or corrupt metadata fails open.
|
|
513
|
+
if (!Number.isFinite(floor) || floor < 0 || Object.is(floor, -0))
|
|
514
|
+
return AUDIT_FLOOR_UNKNOWN;
|
|
515
|
+
return floor;
|
|
516
|
+
}
|
|
517
|
+
/**
|
|
518
|
+
* Did the floor write land? A record has to be PRESENT, not merely decode to the value we wrote:
|
|
519
|
+
* `decodeAuditFloor(undefined)` is the unknown sentinel too, so on a floorless store — where the
|
|
520
|
+
* resolver writes exactly that sentinel — comparing decoded values alone reported a commit for a
|
|
521
|
+
* write that never happened, and the caller pruned with nothing persisted.
|
|
522
|
+
*/
|
|
523
|
+
function floorWriteLanded(stored, floor) {
|
|
524
|
+
return stored !== undefined && decodeAuditFloor(stored) === floor;
|
|
525
|
+
}
|
|
526
|
+
/** Own eight bytes per write: the store must never be handed a live view of the reused module buffer. */
|
|
527
|
+
function encodeAuditFloor(floor) {
|
|
528
|
+
FLOOR_TARGET[0] = floor;
|
|
529
|
+
return FLOOR_BUFFER.slice();
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* Read-modify-write the floor under one store transaction. `resolve` receives the recorded floor
|
|
533
|
+
* (AUDIT_FLOOR_UNKNOWN when there is none) and returns the value to store, or undefined to leave it
|
|
534
|
+
* alone. `key` selects the record: the floor itself, or the bootstrap-provenance record beside it,
|
|
535
|
+
* which wants the same verified commit rather than a second write path.
|
|
536
|
+
*
|
|
537
|
+
* The transaction is the point: several paths advance the floor from different workers, and two
|
|
538
|
+
* unsynchronized read-then-writes can interleave so the lower cutoff lands last — a floor below
|
|
539
|
+
* history the higher one already removed. Read-only mode writes nothing, and prunes nothing.
|
|
540
|
+
*/
|
|
541
|
+
function updateAuditFloor(auditStore, resolve, key = AUDIT_FLOOR_KEY) {
|
|
542
|
+
// A legacy `auditPath` layout is opened as its own standalone LMDB root (databases.ts) and has no
|
|
543
|
+
// `.rootStore`, so it owns the transaction itself.
|
|
544
|
+
const transactionOwner = auditStore?.rootStore ?? auditStore;
|
|
545
|
+
if (!transactionOwner?.transactionSync)
|
|
546
|
+
throw new Error('Cannot record the audit retention floor: this database has no audit store');
|
|
547
|
+
// Both branches read their own write back and report `false` on mismatch, and the caller demands an
|
|
548
|
+
// explicit `true`. Both halves are load-bearing: a RocksDB transactionSync returns undefined for a
|
|
549
|
+
// swallowed abort rather than throwing (see RecordEncoder.saveStructures), and a write that fails
|
|
550
|
+
// without throwing is otherwise indistinguishable from one that landed — a caller pruning against a
|
|
551
|
+
// floor never recorded. Reads inside a write transaction see their own writes on both engines, so
|
|
552
|
+
// the read-back observes what commit will make durable.
|
|
553
|
+
const committed = auditStore instanceof RocksTransactionLogStore_ts_1.RocksTransactionLogStore
|
|
554
|
+
? transactionOwner.transactionSync((txn) => {
|
|
555
|
+
const stored = txn.getBinarySync(key);
|
|
556
|
+
const floor = resolve(decodeAuditFloor(stored), stored !== undefined);
|
|
557
|
+
if (floor !== undefined) {
|
|
558
|
+
txn.putSync(key, (0, lmdb_1.asBinary)(encodeAuditFloor(floor)));
|
|
559
|
+
if (!floorWriteLanded(txn.getBinarySync(key), floor))
|
|
560
|
+
return false;
|
|
561
|
+
}
|
|
562
|
+
return true;
|
|
563
|
+
}, { retryOnBusy: true })
|
|
564
|
+
: transactionOwner.transactionSync(() => {
|
|
565
|
+
const stored = auditStore.getBinary(key);
|
|
566
|
+
const floor = resolve(decodeAuditFloor(stored), stored !== undefined);
|
|
567
|
+
// `put` rather than `putSync`, and inside the transaction: lmdb's putSync is
|
|
568
|
+
// `put(...) === SYNC_PROMISE_SUCCESS`, so it drops whatever put returns, and a rejected put
|
|
569
|
+
// would leak with no owner. Within a write transaction put writes synchronously and returns
|
|
570
|
+
// an already-resolved sentinel, so the value is visible immediately either way and this
|
|
571
|
+
// only takes ownership of the failure case.
|
|
572
|
+
// asBinary: a legacy standalone audit root's encoder has no Uint8Array passthrough, so raw
|
|
573
|
+
// bytes would reach createAuditEntry and throw. This bypasses both encoders.
|
|
574
|
+
if (floor !== undefined) {
|
|
575
|
+
auditStore
|
|
576
|
+
.put(key, (0, lmdb_1.asBinary)(encodeAuditFloor(floor)))
|
|
577
|
+
?.catch?.((error) => warnContained('Error writing the audit retention floor', error));
|
|
578
|
+
if (!floorWriteLanded(auditStore.getBinary(key), floor))
|
|
579
|
+
return false;
|
|
580
|
+
}
|
|
581
|
+
return true;
|
|
582
|
+
});
|
|
583
|
+
if (committed !== true)
|
|
584
|
+
throw new Error('The audit retention floor transaction did not commit');
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* The bound a prune should use — and record — in place of an unbounded cutoff.
|
|
588
|
+
*
|
|
589
|
+
* `Infinity` is a legitimate thing for a caller to *mean* ("remove all of it") and a ruinous thing to
|
|
590
|
+
* store: it is the unknown sentinel, and the sentinel is absorbing. `raiseAuditFloor`'s lock-free
|
|
591
|
+
* pre-check skips any present record no cutoff exceeds, and `establishAuditFloor` skips any store that
|
|
592
|
+
* has one, so a floor at `Infinity` never comes back down — for the whole database, including sibling
|
|
593
|
+
* tables whose own history was never touched. One `deleteHistory(Infinity)` would otherwise retire the
|
|
594
|
+
* accessor for that database permanently (#2458).
|
|
595
|
+
*
|
|
596
|
+
* So bound it by what exists: strictly above the newest key currently in the log, and never below the
|
|
597
|
+
* clock. Nothing already written can escape that bound, and a caller that passes it as the prune's
|
|
598
|
+
* range end as well as its floor cannot remove anything the floor does not cover — which is what makes
|
|
599
|
+
* the write-ahead ordering hold without an infinite bound. An entry written *after* this returns is
|
|
600
|
+
* simply not history the call asked to remove.
|
|
601
|
+
*
|
|
602
|
+
* `Infinity` is only the extreme case. Any cutoff above this bound is the same defect by degree: a
|
|
603
|
+
* finite year-2286 bound (`Date.now() * 1000`, or a bare `'9999999999999'`) is equally unreachable
|
|
604
|
+
* and equally permanent, and entries written after the prune then land *below* the recorded floor,
|
|
605
|
+
* so the floor's promise — nothing after it was pruned — is false about history that is still there.
|
|
606
|
+
* Every cutoff is therefore clamped, not just the unbounded one.
|
|
607
|
+
*
|
|
608
|
+
* Non-numbers, NaN, negatives and `-0` fall through unchanged to `raiseAuditFloor`, which rejects
|
|
609
|
+
* them: the numeric ones are ordered keys the prune range would honor, not bounds anyone meant, and a
|
|
610
|
+
* non-number is guarded explicitly because `>` would otherwise coerce it into a bound the floor
|
|
611
|
+
* accepts. `cutoff > bound` rather than `Math.min` keeps NaN falling through too.
|
|
612
|
+
*
|
|
613
|
+
* On RocksDB `getKeys` is unimplemented and returns `[]`, so the bound reduces to `Date.now()`. A
|
|
614
|
+
* key above the clock therefore survives a purge that asked for it on that engine. That is the safe
|
|
615
|
+
* direction: the entry is kept and the floor stays honest, where the alternative removes history the
|
|
616
|
+
* floor does not cover.
|
|
617
|
+
*/
|
|
618
|
+
function boundedAuditPruneEnd(auditStore, cutoff) {
|
|
619
|
+
// Non-numbers pass through untouched for `raiseAuditFloor` to reject. `>` coerces, so without this
|
|
620
|
+
// a numeric STRING ('9999999999999', 'Infinity') or a future Date compares true against the bound
|
|
621
|
+
// and comes back AS the bound — a number raiseAuditFloor accepts — turning a type error into a
|
|
622
|
+
// whole-log prune.
|
|
623
|
+
if (typeof cutoff !== 'number')
|
|
624
|
+
return cutoff;
|
|
625
|
+
let bound = Date.now();
|
|
626
|
+
for (const newest of auditStore.getKeys({ reverse: true, limit: 1 })) {
|
|
627
|
+
if (typeof newest === 'number' && newest >= bound)
|
|
628
|
+
bound = newest + 1;
|
|
629
|
+
}
|
|
630
|
+
return cutoff > bound ? bound : cutoff;
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* Raise the floor to `cutoff`, the exclusive lower bound of the history a prune is about to make
|
|
634
|
+
* unreachable.
|
|
635
|
+
*
|
|
636
|
+
* **Call this before removing anything.** A floor written after the removal is lost if the process
|
|
637
|
+
* dies in between, and the surviving lower floor then certifies a cursor whose history is gone.
|
|
638
|
+
* Ordering it first also covers a prune that removes less than `cutoff` spans (a RocksDB purge with
|
|
639
|
+
* no whole droppable file, a retention pass stopping at MAX_DELETES_PER_CLEANUP): over-reporting costs
|
|
640
|
+
* one unnecessary resync, under-reporting loses data silently. The retention paths bound that
|
|
641
|
+
* over-report at the configured horizon (`Date.now() - auditRetention`, never below 0); the two
|
|
642
|
+
* operator-supplied bounds have no ceiling of their own and go through `boundedAuditPruneEnd` first,
|
|
643
|
+
* since a bound above everything reachable would be recorded verbatim and never come down.
|
|
644
|
+
*
|
|
645
|
+
* Throws if the floor cannot be persisted, which is why it is called first — the throw is what stops
|
|
646
|
+
* the prune from proceeding unrecorded. Never lowers the floor, so a narrower prune cannot undo a
|
|
647
|
+
* wider one, and a store whose floor is unknown stays unknown rather than being talked down to a
|
|
648
|
+
* cutoff that says nothing about the history it has already lost.
|
|
649
|
+
*/
|
|
650
|
+
function raiseAuditFloor(auditStore, cutoff) {
|
|
651
|
+
// Throw rather than no-op on a bound we will not store: audit keys are raw float64, so NaN and
|
|
652
|
+
// negatives (sign bit set) sort ABOVE every real timestamp and a range ending there spans the whole
|
|
653
|
+
// log — declining the floor silently would leave the prune deleting everything. `-0` and a non-number
|
|
654
|
+
// slip past a naive `< 0` check but are still ordered keys the range honors, so they are rejected too.
|
|
655
|
+
// Infinity is accepted and stored, decoding back to "unknown": every production caller clamps before
|
|
656
|
+
// reaching here (`boundedAuditPruneEnd`, the bridge's 400), so it stays reachable only for a caller
|
|
657
|
+
// that genuinely cannot bound its prune.
|
|
658
|
+
if (typeof cutoff !== 'number' || Number.isNaN(cutoff) || cutoff < 0 || Object.is(cutoff, -0))
|
|
659
|
+
throw new Error(`Invalid audit prune bound: ${String(cutoff)}`);
|
|
660
|
+
// Read-only mode does not exempt a prune from recording its floor; it means the prune must not
|
|
661
|
+
// happen. Only scheduleAuditCleanup and purgeAgedLogs check read-only themselves, so for
|
|
662
|
+
// deleteHistory and the whole-database purge this throw is the guard.
|
|
663
|
+
if ((0, databases_ts_1.isReadOnlyMode)())
|
|
664
|
+
throw new Error('Cannot record the audit retention floor: the database is read-only');
|
|
665
|
+
// Lock-free pre-check, getBinary-guarded so this optimization never decides the error a store with
|
|
666
|
+
// no audit store reports. Most calls cannot move the floor (a RocksDB reclamation pass on an idle
|
|
667
|
+
// database, a cutoff below one a wider prune already set), and taking the env write lock to
|
|
668
|
+
// discover that serializes every worker's boot and reclamation on it. The in-transaction guards
|
|
669
|
+
// below stay authoritative.
|
|
670
|
+
// Skips only the case it can prove is a no-op: a record that exists and already sits at or above
|
|
671
|
+
// the cutoff. An absent record is NOT decided here — the presence question is settled inside the
|
|
672
|
+
// transaction below, because another worker's establishAuditFloor can land between this read and
|
|
673
|
+
// that write.
|
|
674
|
+
if (auditStore?.getBinary) {
|
|
675
|
+
const stored = auditStore.getBinary(AUDIT_FLOOR_KEY);
|
|
676
|
+
if (stored !== undefined && !(cutoff > decodeAuditFloor(stored)))
|
|
677
|
+
return;
|
|
678
|
+
}
|
|
679
|
+
updateAuditFloor(auditStore, (current, recorded) => {
|
|
680
|
+
// Still no record, and we are about to prune: persist the unknown sentinel. Leaving no marker
|
|
681
|
+
// lets the next open stamp a FINITE epoch, and a prune bound above that epoch (a future
|
|
682
|
+
// `endTime`, or a rolled-back clock) then certifies cursors whose history this prune deleted.
|
|
683
|
+
// Unknown is the honest value, because a store with no record may have been pruned before this
|
|
684
|
+
// run too.
|
|
685
|
+
if (!recorded)
|
|
686
|
+
return AUDIT_FLOOR_UNKNOWN;
|
|
687
|
+
// A record appeared while we were getting here, so this is an ordinary monotonic raise: pruning
|
|
688
|
+
// to `cutoff` against a floor left below it is exactly the silent gap this function prevents.
|
|
689
|
+
return cutoff > current ? cutoff : undefined;
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
/**
|
|
693
|
+
* Give a database a trustworthy floor if it has none: the current time, as a one-time resync epoch.
|
|
694
|
+
*
|
|
695
|
+
* The floor record's *presence* is the trust marker, so a store without one is a store whose
|
|
696
|
+
* retention history we cannot account for. It may have been pruned by a version that recorded no
|
|
697
|
+
* floor; it may be the empty audit store an LMDB→RocksDB migration deliberately leaves behind
|
|
698
|
+
* (`bin/copyDb.ts` does not migrate it, so the records and their resumable cursors outlive their
|
|
699
|
+
* history); or it may be a database restored from a table-scoped backup taken without
|
|
700
|
+
* `include_audit`, which carries records but no audit DBI. Cursors from before this moment are
|
|
701
|
+
* therefore reported stale — not because we know they are, but because we do not know they are not.
|
|
702
|
+
*
|
|
703
|
+
* There is no "brand new store, use a permissive baseline" case: creating the audit DBI proves only
|
|
704
|
+
* that the DBI was absent, which the audit-less backup above also produces. Being conservative on a
|
|
705
|
+
* genuinely new database costs nothing, since its entries are all written after this instant.
|
|
706
|
+
*
|
|
707
|
+
* In read-only mode nothing is written and the floor stays unknown — the fail-closed answer for a
|
|
708
|
+
* process that cannot record what it does not know.
|
|
709
|
+
*
|
|
710
|
+
* **The epoch is also recorded under its own key, so this bootstrap is repairable.** The epoch is a
|
|
711
|
+
* guess bounded by surviving state, and surviving state cannot see history a selective prune already
|
|
712
|
+
* removed (see the clock note below). The record marks the store as one that carried a guess, and
|
|
713
|
+
* preserves the value guessed — the two facts a later release needs to raise such a floor to
|
|
714
|
+
* something it can stand behind. See "Audit retention floor" in DESIGN.md for the full reading.
|
|
715
|
+
*
|
|
716
|
+
* **The mark is the signal, not a comparison against the floor.** A store carrying this record has an
|
|
717
|
+
* unverified pre-tracking window for as long as it exists, however far the floor has since moved: a
|
|
718
|
+
* prune raising the floor above the epoch certifies only what that prune removed, and says nothing
|
|
719
|
+
* about history removed before tracking began — which may sit above the epoch, since that is exactly
|
|
720
|
+
* the case the guess cannot see. Retiring the mark takes a database generation (harper#2451), not a
|
|
721
|
+
* floor that has climbed past it.
|
|
722
|
+
*
|
|
723
|
+
* Ordering. The provenance record is written **first**, so a crash between the two writes leaves a
|
|
724
|
+
* record with no floor — which the next open retries, since the early return above tests the floor.
|
|
725
|
+
* The epoch actually stamped is always read back from the record rather than taken from this call's
|
|
726
|
+
* own `Date.now()`, so a worker whose record lost the race adopts the winner's value and the two
|
|
727
|
+
* always agree. Re-adopting an older record is sound: nothing was pruned in the meantime, or a prune
|
|
728
|
+
* would have written the floor this function returns early on.
|
|
729
|
+
*/
|
|
730
|
+
function establishAuditFloor(auditStore) {
|
|
731
|
+
if ((0, databases_ts_1.isReadOnlyMode)())
|
|
732
|
+
return;
|
|
733
|
+
// Every read and write in here is inside the try: the contract is that a database open never fails
|
|
734
|
+
// over this metadata, and a throwing getBinary/getKeys would escape to initStores just as a
|
|
735
|
+
// throwing write would.
|
|
736
|
+
try {
|
|
737
|
+
// Absence of the record, not `getAuditFloor() === AUDIT_FLOOR_UNKNOWN`: a record that exists but
|
|
738
|
+
// decodes to unknown — corrupt bytes, or the Infinity a prune-everything stored — is already the
|
|
739
|
+
// fail-closed answer, and stamping over it would LOWER a floor that is never supposed to lower.
|
|
740
|
+
// The check is repeated inside the transaction (the `recorded` argument), because between this
|
|
741
|
+
// read and that write another worker's prune can store exactly such a value; this read only keeps
|
|
742
|
+
// the common case — a floor already established, every worker, every database, every boot — off
|
|
743
|
+
// the env write lock.
|
|
744
|
+
if (auditStore.getBinary(AUDIT_FLOOR_KEY) !== undefined)
|
|
745
|
+
return;
|
|
746
|
+
// Not bare Date.now(): a clock that has rolled back would bootstrap a floor BELOW history this
|
|
747
|
+
// database may already have pruned, certifying a stale cursor. The newest retained entry is a
|
|
748
|
+
// lower bound the clock cannot argue with — everything at or above it is demonstrably still here —
|
|
749
|
+
// so take whichever is later.
|
|
750
|
+
//
|
|
751
|
+
// It narrows that hole; it does not close it, because the bound covers what SURVIVES rather than
|
|
752
|
+
// what existed. A legacy `deleteHistory` removes one table's entries below its endTime out of the
|
|
753
|
+
// shared log, so a table whose entries were the newest and all fell below that bound leaves the
|
|
754
|
+
// log's newest survivor being a sibling's OLDER entry — removed history above every surviving key.
|
|
755
|
+
// A clock rolled back to between the two then stamps an epoch below entries that are gone, and a
|
|
756
|
+
// cursor in that window resumes over the gap (#2458). Also unclosed:
|
|
757
|
+
// RocksTransactionLogStore.getKeys() is unimplemented, so on that engine this reduces to
|
|
758
|
+
// Date.now() outright.
|
|
759
|
+
//
|
|
760
|
+
// Neither is a reason to refuse to stamp — the unknown sentinel is absorbing, so that would make
|
|
761
|
+
// every upgraded deployment fail closed forever. They are the reason
|
|
762
|
+
// the guess is RECORDED as a guess: written first, so it cannot be lost behind a floor that
|
|
763
|
+
// outlives it, and left in place afterwards so the repair reading stays available.
|
|
764
|
+
let fresh = Date.now();
|
|
765
|
+
for (const newest of auditStore.getKeys({ reverse: true, limit: 1 })) {
|
|
766
|
+
if (typeof newest === 'number' && newest > fresh)
|
|
767
|
+
fresh = newest;
|
|
768
|
+
}
|
|
769
|
+
// A READABLE record is kept, so the epoch read back below is whatever landed first and the floor
|
|
770
|
+
// always matches it. An unreadable one is replaced: unlike the floor, where a present record may
|
|
771
|
+
// be a deliberate AUDIT_FLOOR_UNKNOWN and overwriting it would lower a floor, this record is only
|
|
772
|
+
// a comparison basis, and undecodable bytes carry nothing worth keeping. Declining to replace
|
|
773
|
+
// them pinned the store's floor to unknown forever — the resolver would skip the write on every
|
|
774
|
+
// later open, the read back would fail identically, and no retry could ever succeed, which is the
|
|
775
|
+
// fail-closed-forever state this bootstrap exists to avoid.
|
|
776
|
+
updateAuditFloor(auditStore, (current, recorded) => (recorded && Number.isFinite(current) ? undefined : fresh), AUDIT_FLOOR_BOOTSTRAP_KEY);
|
|
777
|
+
const epoch = decodeAuditFloor(auditStore.getBinary(AUDIT_FLOOR_BOOTSTRAP_KEY));
|
|
778
|
+
// Never stamp a floor whose provenance cannot be read: that is the one state a later repair cannot
|
|
779
|
+
// act on. Unreachable in principle, since `updateAuditFloor` throws if its write did not land.
|
|
780
|
+
if (!Number.isFinite(epoch))
|
|
781
|
+
return;
|
|
782
|
+
updateAuditFloor(auditStore, (_current, recorded) => (recorded ? undefined : epoch));
|
|
783
|
+
}
|
|
784
|
+
catch (error) {
|
|
785
|
+
// An unrecorded floor already reads as unknown, which is the fail-closed answer; aborting startup
|
|
786
|
+
// instead would turn a metadata failure into an outage. The next open retries.
|
|
787
|
+
warnContained('Error initializing the audit retention floor', error);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
/**
|
|
791
|
+
* The floor of this database's retained audit history: every audit entry at or after the returned
|
|
792
|
+
* time is still retained, so a consumer whose last-processed audit-log cursor is `>=` it can resume
|
|
793
|
+
* incrementally, and one below it must resync — it may have lost nothing, but the floor cannot
|
|
794
|
+
* certify it either way. Returns `Infinity` when
|
|
795
|
+
* the floor is unknown, which fails closed — no cursor compares as safe.
|
|
796
|
+
*
|
|
797
|
+
* **One exception, and it is the only one: history removed before the floor existed.** Every prune
|
|
798
|
+
* that runs with a floor recorded is covered — it raises the floor first, so it cannot remove an entry
|
|
799
|
+
* the floor does not cover. But the first open stamps a floor from what *survives*
|
|
800
|
+
* (`establishAuditFloor`), and a legacy `Table.deleteHistory` that removed a table's newest entries,
|
|
801
|
+
* followed by a clock rollback, leaves that stamp below history that is gone; a cursor in the window
|
|
802
|
+
* is then certified over the gap. So the guarantee is one-directional for tracked prunes and silent
|
|
803
|
+
* about untracked ones. Only a database generation can close that (harper#2451).
|
|
804
|
+
*
|
|
805
|
+
* The time domain is the audit-log key: what `subscribe`'s events carry as `localTime` and what MQTT
|
|
806
|
+
* durable sessions persist as `startTime`, so those compare against the floor directly.
|
|
807
|
+
* **`getHistory` is not in that domain** — it reports each entry's origin `version` under the name
|
|
808
|
+
* `localTime`, which a backdated or replicated write makes differ from the audit-log key. A cursor
|
|
809
|
+
* saved from `getHistory` is not comparable to this floor.
|
|
810
|
+
*
|
|
811
|
+
* **Database-scoped**, and deliberately conservative: the audit store is per-database and its
|
|
812
|
+
* entries carry a `tableId`, so a per-table floor would need a scan for the first entry matching
|
|
813
|
+
* that table. For a valid cursor, `cursor >= floor` therefore means no entry of *any* table in the
|
|
814
|
+
* database was removed *after* the cursor. What it never promises is anything below the FLOOR — that
|
|
815
|
+
* history is exactly what a prune takes. Below the *cursor* is not the same set: for a cursor strictly
|
|
816
|
+
* above the floor, `[floor, cursor)` sits below the cursor and is still covered by the guarantee. `Table.deleteHistory`
|
|
817
|
+
* prunes one table out of that shared log and raises the whole database's floor, which can overstate
|
|
818
|
+
* the floor for its siblings.
|
|
819
|
+
*
|
|
820
|
+
* **A moment-in-time observation.** Retention can advance between this call and whatever the caller
|
|
821
|
+
* does with the answer, so a check-then-resume sequence has a window where the floor moves under it.
|
|
822
|
+
* Closing that requires validating the cursor inside the resume itself (harper#2448); until then a
|
|
823
|
+
* lost race degrades to the truncation that happens today, never to anything worse.
|
|
824
|
+
*
|
|
825
|
+
* **On RocksDB the floor tracks the configured horizon, not retained reality.** That branch purges at
|
|
826
|
+
* whole-log-file granularity, so it cannot know before the fact which entries a purge will drop, and
|
|
827
|
+
* the floor has to be written first — so every retention pass advances it to
|
|
828
|
+
* `Date.now() - auditRetention / (1 + priority²)` whether or not a file was dropped. Entries below
|
|
829
|
+
* that horizon are routinely still on disk, and a consumer holding a cursor among them is told to
|
|
830
|
+
* resync. Conservative in the one safe direction, and the reason the LMDB branch (which can see a
|
|
831
|
+
* single eligible entry) instead raises off the first one it finds.
|
|
832
|
+
*
|
|
833
|
+
* **Not covered: copying a database's state without its history.** `restore_backup` replaces a
|
|
834
|
+
* database with the backup's, floor and all, and a RocksDB checkpoint (a branch database) copies the
|
|
835
|
+
* floor record but no transaction logs — so in both cases a cursor from after the copy point sits
|
|
836
|
+
* above a floor that is present, and therefore trusted, for history that is not there. Nothing here
|
|
837
|
+
* can detect that on its own, and it is not the audit log's problem alone: the same copy rolls back
|
|
838
|
+
* record versions and per-node replication sequence state, so making this one field honest while
|
|
839
|
+
* those stay stale would not give a consumer a coherent answer. It needs a database-level epoch —
|
|
840
|
+
* harper#2451.
|
|
841
|
+
*/
|
|
842
|
+
function getAuditFloor(auditStore) {
|
|
843
|
+
return decodeAuditFloor(auditStore.getBinary(AUDIT_FLOOR_KEY));
|
|
844
|
+
}
|
|
413
845
|
function setAuditRetention(retentionTime, defaultDelay = DEFAULT_AUDIT_CLEANUP_DELAY) {
|
|
414
846
|
exports.auditRetention = retentionTime;
|
|
415
847
|
DEFAULT_AUDIT_CLEANUP_DELAY = defaultDelay;
|
|
416
848
|
}
|
|
849
|
+
/**
|
|
850
|
+
* The retention cutoff a prune uses: `Date.now() - auditRetention`, scaled down by `divisor` for a
|
|
851
|
+
* higher-priority pass, and never below 0. A retention above ~55.7 years (or `Infinity`, to keep
|
|
852
|
+
* logs indefinitely) would otherwise go negative, and a negative bound is not "nothing eligible" —
|
|
853
|
+
* `raiseAuditFloor` rejects it, so every boot purge and retention pass would warn and the floor
|
|
854
|
+
* would never be raised on that install. At 0 the pass is a harmless no-op: nothing sits before
|
|
855
|
+
* the epoch, and a floor that already exists is never lowered to it.
|
|
856
|
+
*/
|
|
857
|
+
function retentionCutoff(divisor = 1) {
|
|
858
|
+
return Math.max(0, Date.now() - exports.auditRetention / divisor);
|
|
859
|
+
}
|
|
417
860
|
/**
|
|
418
861
|
* One-shot purge of transaction-log files already older than the audit retention window,
|
|
419
862
|
* intended to run during startup/recovery before transaction-log replay. The steady-state
|
|
@@ -427,7 +870,10 @@ function purgeAgedLogs(rootStore) {
|
|
|
427
870
|
// Mirror the read-only guard in scheduleAuditCleanup: never delete log files in read-only mode.
|
|
428
871
|
if ((0, databases_ts_1.isReadOnlyMode)())
|
|
429
872
|
return [];
|
|
430
|
-
|
|
873
|
+
const before = retentionCutoff();
|
|
874
|
+
// The audit store is reachable this early because initStores opens it before replayLogs runs this.
|
|
875
|
+
raiseAuditFloor(rootStore.auditStore, before);
|
|
876
|
+
return rootStore.purgeLogs({ before });
|
|
431
877
|
}
|
|
432
878
|
const HAS_RECORD = 16;
|
|
433
879
|
const HAS_PARTIAL_RECORD = 32; // will be used for CRDTs
|
|
@@ -441,13 +887,31 @@ const STRUCTURES = 7;
|
|
|
441
887
|
// Whole-table "reload" marker: a control entry (no record) signalling that a table was bulk-reloaded
|
|
442
888
|
// and subscribers should re-read it. Used after a copyApply base copy, whose per-row snapshot writes
|
|
443
889
|
// carry no audit entry (harper-pro#489). The entry type lives in the low nibble of the action byte
|
|
444
|
-
// (decoded via `action & 0xf`); 1–7 are the record actions above, 8 is reload, leaving
|
|
445
|
-
// future actions.
|
|
890
|
+
// (decoded via `action & 0xf`); 1–7 are the record actions above, 8 is reload, 9 is eviction, leaving 10–15 free for
|
|
891
|
+
// future actions. Reload markers are always written LOCAL_ONLY so an unknown type never reaches a
|
|
892
|
+
// peer; the lock control entries below deliberately are not, and rely on the capability gate instead.
|
|
446
893
|
const RELOAD = 8;
|
|
894
|
+
const EVICT = 9;
|
|
447
895
|
exports.ACTION_32_BIT = 14;
|
|
448
896
|
exports.ACTION_64_BIT = 15;
|
|
449
897
|
/** Used to indicate we have received a remote local time update */
|
|
450
898
|
exports.REMOTE_SEQUENCE_UPDATE = 11;
|
|
899
|
+
/**
|
|
900
|
+
* Cluster record-lock coordination (harper#483 Phase 1). This replicates — unlike the reload marker
|
|
901
|
+
* it is NOT `LOCAL_ONLY` — and carries a control payload rather than a record, so it is written with
|
|
902
|
+
* `recordId: null`: an entry sharing a real record's `(version, tableId, recordId, nodeId)` would be
|
|
903
|
+
* returned by `RocksTransactionLogStore.getSync` ahead of that record's own audit entry and make
|
|
904
|
+
* `_writeUpdate`'s keyed dedup drop the holder's write.
|
|
905
|
+
*
|
|
906
|
+
* Two entries exist: the release, and the barrier — a replicated no-op a member commits on request
|
|
907
|
+
* so its own log position can serve as the §7.2 recovery fence (harper#2625). Nibbles 9 and 10
|
|
908
|
+
* briefly held `lockRequest`/`lockGrant` for the Ricart–Agrawala arbitration rule that
|
|
909
|
+
* `docs/record-lock-ownership.md` replaces; that rule never shipped enabled, so they were retired
|
|
910
|
+
* rather than migrated — and 9 has since been taken by eviction. 10 is spare; 14/15 are the width
|
|
911
|
+
* flags.
|
|
912
|
+
*/
|
|
913
|
+
exports.LOCK_RELEASE = 12;
|
|
914
|
+
exports.LOCK_BARRIER = 13;
|
|
451
915
|
exports.HAS_CURRENT_RESIDENCY_ID = 512;
|
|
452
916
|
exports.HAS_PREVIOUS_RESIDENCY_ID = 1024;
|
|
453
917
|
exports.HAS_ORIGINATING_OPERATION = 2048;
|
|
@@ -481,16 +945,84 @@ const EVENT_TYPES = {
|
|
|
481
945
|
[STRUCTURES]: 'structures',
|
|
482
946
|
reload: RELOAD,
|
|
483
947
|
[RELOAD]: 'reload',
|
|
948
|
+
evict: EVICT,
|
|
949
|
+
[EVICT]: 'evict',
|
|
484
950
|
remoteSequenceUpdate: exports.REMOTE_SEQUENCE_UPDATE,
|
|
485
951
|
[exports.REMOTE_SEQUENCE_UPDATE]: 'remoteSequenceUpdate',
|
|
952
|
+
lockRelease: exports.LOCK_RELEASE | HAS_RECORD,
|
|
953
|
+
[exports.LOCK_RELEASE]: 'lockRelease',
|
|
954
|
+
lockBarrier: exports.LOCK_BARRIER | HAS_RECORD,
|
|
955
|
+
[exports.LOCK_BARRIER]: 'lockBarrier',
|
|
486
956
|
};
|
|
957
|
+
/**
|
|
958
|
+
* The LMDB audit entry states the presence of its leading 8-byte previousVersion field with that
|
|
959
|
+
* field's own first byte. The same test is what harperdb 4.x's reader uses and what both versions'
|
|
960
|
+
* replication senders use to strip the field before framing an entry for the wire, so it is a
|
|
961
|
+
* cross-version contract, not a local convention: a field written with any other leading byte is
|
|
962
|
+
* skipped by every reader and shifts action/nodeId/tableId/recordId/version by 8. See harper#2247.
|
|
963
|
+
*/
|
|
964
|
+
const PREVIOUS_VERSION_FIRST_BYTE = 66;
|
|
965
|
+
/**
|
|
966
|
+
* Which first bytes can begin an action, and so cannot be a previousVersion field. Derived from what
|
|
967
|
+
* the encoding can express, not from the types defined today: nibbles 9-15 are reserved for future
|
|
968
|
+
* entry types, and a peer one version ahead must keep decoding rather than look corrupt here.
|
|
969
|
+
*/
|
|
970
|
+
const ACTION_FIRST_BYTE = new Uint8Array(256);
|
|
971
|
+
// single-byte form; 0 is not an entry type, so a zero low nibble cannot start one
|
|
972
|
+
for (let firstByte = 1; firstByte <= (HAS_RECORD | HAS_PARTIAL_RECORD | 0xf); firstByte++) {
|
|
973
|
+
if (firstByte & 0xf)
|
|
974
|
+
ACTION_FIRST_BYTE[firstByte] = 1;
|
|
975
|
+
}
|
|
976
|
+
// extended form, written as `action | extendedType | 0xc0000000`; 0xff is the five-byte readInt form
|
|
977
|
+
for (let firstByte = 0xc0; firstByte < 0xff; firstByte++)
|
|
978
|
+
ACTION_FIRST_BYTE[firstByte] = 1;
|
|
979
|
+
let knownActionFlags;
|
|
980
|
+
/**
|
|
981
|
+
* Whether an action word decodes wholly into what this version understands. Consulted only to accept
|
|
982
|
+
* or reject a legacy prefix recovery — never to validate a normally-framed entry, where an unknown
|
|
983
|
+
* future flag must stay forwards-compatible rather than corrupt. The mask is built on first use
|
|
984
|
+
* because HAS_STRUCTURE_UPDATE crosses the RecordEncoder import cycle: read at module scope it
|
|
985
|
+
* resolves to undefined whenever RecordEncoder is the cycle's entry, silently dropping that bit.
|
|
986
|
+
*/
|
|
987
|
+
function isDecodableAction(action) {
|
|
988
|
+
knownActionFlags ??=
|
|
989
|
+
0xf |
|
|
990
|
+
HAS_RECORD |
|
|
991
|
+
HAS_PARTIAL_RECORD |
|
|
992
|
+
RecordEncoder_ts_1.HAS_STRUCTURE_UPDATE |
|
|
993
|
+
exports.HAS_CURRENT_RESIDENCY_ID |
|
|
994
|
+
exports.HAS_PREVIOUS_RESIDENCY_ID |
|
|
995
|
+
exports.HAS_ORIGINATING_OPERATION |
|
|
996
|
+
exports.HAS_EXPIRATION_EXTENDED_TYPE |
|
|
997
|
+
exports.HAS_BLOBS |
|
|
998
|
+
exports.HAS_ADDITIONAL_AUDIT_REFS |
|
|
999
|
+
exports.LOCAL_ONLY;
|
|
1000
|
+
return (action & 0xf) !== 0 && !(action & ~knownActionFlags);
|
|
1001
|
+
}
|
|
1002
|
+
/**
|
|
1003
|
+
* Cluster lock coordination entries. They ride the replicated audit stream but describe no record,
|
|
1004
|
+
* so every consumer that surfaces audit entries as record activity — subscriber fan-out, the
|
|
1005
|
+
* `startTime` replay, the `previousCount` backfill, the replicated-event sink — must exclude them.
|
|
1006
|
+
* An equality chain rather than a Set: this runs once per audit entry on the replay and fan-out
|
|
1007
|
+
* paths, where the common answer is false on the first comparison.
|
|
1008
|
+
*/
|
|
1009
|
+
function isLockControlType(type) {
|
|
1010
|
+
return type === 'lockRelease' || type === 'lockBarrier';
|
|
1011
|
+
}
|
|
487
1012
|
const ORIGINATING_OPERATIONS = {
|
|
488
1013
|
insert: 1,
|
|
489
1014
|
update: 2,
|
|
490
1015
|
upsert: 3,
|
|
1016
|
+
// `put` must be persisted, not inferred: the physical write type is also `put` for an `upsert`
|
|
1017
|
+
// that happened to create a record, so the history readers cannot tell the two apart from the
|
|
1018
|
+
// type alone. Without an id here the originating operation decoded as undefined, the readers fell
|
|
1019
|
+
// back to the physical type, and replication catch-up replayed a `put` as an `upsert` — patching
|
|
1020
|
+
// the replica and RETAINING attributes the source had removed.
|
|
1021
|
+
put: 4,
|
|
491
1022
|
1: 'insert',
|
|
492
1023
|
2: 'update',
|
|
493
1024
|
3: 'upsert',
|
|
1025
|
+
4: 'put',
|
|
494
1026
|
};
|
|
495
1027
|
/**
|
|
496
1028
|
* Creates a binary audit entry
|
|
@@ -523,14 +1055,42 @@ function createAuditEntry(auditRecord, start = 0) {
|
|
|
523
1055
|
}
|
|
524
1056
|
action &= ~HAS_RECORD;
|
|
525
1057
|
}
|
|
526
|
-
let
|
|
527
|
-
if (
|
|
528
|
-
|
|
1058
|
+
let hasPreviousVersion;
|
|
1059
|
+
if (start > 0) {
|
|
1060
|
+
// RocksTransactionLogStore states presence with its own prelude flag, derived from this same
|
|
1061
|
+
// truthiness, so this container's field stays unconditional and its value is unconstrained.
|
|
1062
|
+
hasPreviousVersion = !!previousVersion;
|
|
1063
|
+
if (hasPreviousVersion)
|
|
529
1064
|
exports.ENTRY_DATAVIEW.setFloat64(start, previousVersion);
|
|
530
|
-
else
|
|
531
|
-
ENTRY_HEADER.set(RecordEncoder_ts_1.PREVIOUS_TIMESTAMP_PLACEHOLDER, start);
|
|
532
|
-
position = start + 9;
|
|
533
1065
|
}
|
|
1066
|
+
else if (previousVersion == null || previousVersion === 0) {
|
|
1067
|
+
// absence is stated, not inferred from truthiness: NaN is falsy and would otherwise be
|
|
1068
|
+
// silently dropped rather than rejected below
|
|
1069
|
+
hasPreviousVersion = false;
|
|
1070
|
+
}
|
|
1071
|
+
else if (previousVersion === RecordEncoder_ts_1.PENDING_LOCAL_TIME) {
|
|
1072
|
+
// The previous entry has no log position yet, and a format whose presence signal is the value's
|
|
1073
|
+
// own first byte cannot express "to be filled in at commit". The superseded code deferred to
|
|
1074
|
+
// lmdb-js's instructed-write substitution, which resolves to 2.0 whenever no previous time was
|
|
1075
|
+
// recorded — the unreadable entry behind harper-pro#737. It resolved correctly when one was, so
|
|
1076
|
+
// this trades a lost link on that path for never minting the unreadable form. Not a throw: a
|
|
1077
|
+
// pending previous is a producer state, unlike a value the format cannot hold.
|
|
1078
|
+
hasPreviousVersion = false;
|
|
1079
|
+
if (!warnedPendingPreviousVersion.has(`${tableId}:${type}`)) {
|
|
1080
|
+
warnedPendingPreviousVersion.add(`${tableId}:${type}`);
|
|
1081
|
+
warnContained(`Audit entry (${type}) for record ${recordId} in table ${tableId} has a pending previous version; recording it without a previous-version link`, new Error('pending audit previousVersion'));
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
else {
|
|
1085
|
+
exports.ENTRY_DATAVIEW.setFloat64(start, previousVersion);
|
|
1086
|
+
if (ENTRY_HEADER[start] !== PREVIOUS_VERSION_FIRST_BYTE) {
|
|
1087
|
+
throw new Error(`Audit entry previousVersion ${previousVersion} for record ${recordId} in table ${tableId} is not representable. ` +
|
|
1088
|
+
'The LMDB audit format signals this field with its own leading 0x42 byte, so only values in [2**33, 2**49) can be written; ' +
|
|
1089
|
+
'writing any other value produces an entry every reader parses 8 bytes off.');
|
|
1090
|
+
}
|
|
1091
|
+
hasPreviousVersion = true;
|
|
1092
|
+
}
|
|
1093
|
+
let position = start + (hasPreviousVersion ? 9 : 1);
|
|
534
1094
|
if (extendedType) {
|
|
535
1095
|
if (extendedType & 0xff) {
|
|
536
1096
|
throw new Error('Illegal extended type');
|
|
@@ -572,10 +1132,11 @@ function createAuditEntry(auditRecord, start = 0) {
|
|
|
572
1132
|
writeValue(user);
|
|
573
1133
|
else
|
|
574
1134
|
ENTRY_HEADER[position++] = 0;
|
|
1135
|
+
const actionPosition = start + (hasPreviousVersion ? 8 : 0);
|
|
575
1136
|
if (extendedType)
|
|
576
|
-
exports.ENTRY_DATAVIEW.setUint32(
|
|
1137
|
+
exports.ENTRY_DATAVIEW.setUint32(actionPosition, action | extendedType | 0xc0000000);
|
|
577
1138
|
else
|
|
578
|
-
ENTRY_HEADER[
|
|
1139
|
+
ENTRY_HEADER[actionPosition] = action;
|
|
579
1140
|
const header = ENTRY_HEADER.subarray(0, position);
|
|
580
1141
|
if (encodedRecord) {
|
|
581
1142
|
return Buffer.concat([header, encodedRecord]);
|
|
@@ -638,13 +1199,42 @@ function readAuditEntry(buffer, start = 0, end = undefined) {
|
|
|
638
1199
|
(buffer.decoder = new Decoder(buffer.buffer, buffer.byteOffset, buffer.byteLength));
|
|
639
1200
|
decoder.position = start;
|
|
640
1201
|
let previousVersion;
|
|
641
|
-
|
|
642
|
-
|
|
1202
|
+
let entryStart = start;
|
|
1203
|
+
const firstByte = buffer[start];
|
|
1204
|
+
if (firstByte === PREVIOUS_VERSION_FIRST_BYTE) {
|
|
643
1205
|
previousVersion = decoder.readFloat64();
|
|
644
1206
|
}
|
|
1207
|
+
else if (ACTION_FIRST_BYTE[firstByte] !== 1) {
|
|
1208
|
+
// Written before the writer enforced the leading-0x42 contract (harper#2247): the field is
|
|
1209
|
+
// physically here but does not announce itself, so the action sits 8 bytes further on.
|
|
1210
|
+
// Recover only when the bytes cannot be anything else, and never for the RocksDB container,
|
|
1211
|
+
// whose prelude flag already consumed its previousVersion before this offset.
|
|
1212
|
+
if (start !== 0 || start + 9 > buffer.byteLength || ACTION_FIRST_BYTE[buffer[start + 8]] !== 1) {
|
|
1213
|
+
return corruptEntry(buffer, start, end, firstByte);
|
|
1214
|
+
}
|
|
1215
|
+
// > 1 is exactly what the superseded writer's own guard could emit, the tightest available
|
|
1216
|
+
// bound on "these bytes really are an old previousVersion"
|
|
1217
|
+
if (!(decoder.getFloat64(start) > 1))
|
|
1218
|
+
return corruptEntry(buffer, start, end, firstByte);
|
|
1219
|
+
entryStart = decoder.position = start + 8;
|
|
1220
|
+
// The value itself is dropped, not reported. It cannot lead with 0x42 (that is the branch
|
|
1221
|
+
// above), and audit keys carry the same constraint, so it could never have addressed a
|
|
1222
|
+
// retrievable entry; reporting it would only feed an unrepresentable value back to
|
|
1223
|
+
// createAuditEntry through the resolveRecord re-mint in RecordEncoder, which now rejects it.
|
|
1224
|
+
}
|
|
645
1225
|
const action = decoder.readInt();
|
|
1226
|
+
if (entryStart !== start && !isDecodableAction(action)) {
|
|
1227
|
+
return corruptEntry(buffer, start, end, firstByte);
|
|
1228
|
+
}
|
|
646
1229
|
const nodeId = decoder.readInt();
|
|
647
1230
|
const tableId = decoder.readInt();
|
|
1231
|
+
if (entryStart !== start && !warnedRecoveredTables.has(tableId)) {
|
|
1232
|
+
warnedRecoveredTables.add(tableId);
|
|
1233
|
+
warnContained('Audit entry carries an unannounced previousVersion field; recovering its field offsets', {
|
|
1234
|
+
tableId,
|
|
1235
|
+
firstByte,
|
|
1236
|
+
});
|
|
1237
|
+
}
|
|
648
1238
|
let length = decoder.readInt();
|
|
649
1239
|
// A corrupt length field (e.g., a 0xff-prefixed uint32) would otherwise push
|
|
650
1240
|
// decoder.position hundreds of megabytes past the buffer; the next readFloat64
|
|
@@ -690,10 +1280,11 @@ function readAuditEntry(buffer, start = 0, end = undefined) {
|
|
|
690
1280
|
const usernameEnd = (decoder.position += length);
|
|
691
1281
|
let value;
|
|
692
1282
|
return {
|
|
693
|
-
// The entry type is the low nibble of the action byte (1–7 record actions, 8 reload, 9–15
|
|
1283
|
+
// The entry type is the low nibble of the action byte (1–7 record actions, 8 reload, 9 eviction, 10–15
|
|
694
1284
|
// reserved); the flag bits (HAS_RECORD, HAS_PARTIAL_RECORD, …) sit above it. `& 0xf` is
|
|
695
1285
|
// identical to the historical `& 7` for every pre-reload entry (bit 3 was always clear).
|
|
696
1286
|
type: EVENT_TYPES[action & 0xf],
|
|
1287
|
+
logName: undefined,
|
|
697
1288
|
tableId,
|
|
698
1289
|
nodeId,
|
|
699
1290
|
get recordId() {
|
|
@@ -714,7 +1305,6 @@ function readAuditEntry(buffer, start = 0, end = undefined) {
|
|
|
714
1305
|
return buffer.subarray(recordIdStart, recordIdEnd);
|
|
715
1306
|
},
|
|
716
1307
|
version,
|
|
717
|
-
recordVersion: version,
|
|
718
1308
|
previousVersion,
|
|
719
1309
|
get user() {
|
|
720
1310
|
try {
|
|
@@ -728,10 +1318,14 @@ function readAuditEntry(buffer, start = 0, end = undefined) {
|
|
|
728
1318
|
}
|
|
729
1319
|
},
|
|
730
1320
|
get encoded() {
|
|
731
|
-
|
|
1321
|
+
// On a recovered entry this drops the unannounced prefix, so a replication sender —
|
|
1322
|
+
// which strips by the same leading-0x42 test and frames by encoded.length — forwards a
|
|
1323
|
+
// clean entry instead of passing the same misparse to the next hop.
|
|
1324
|
+
return entryStart ? buffer.subarray(entryStart, end) : buffer;
|
|
732
1325
|
},
|
|
733
1326
|
get size() {
|
|
734
|
-
|
|
1327
|
+
// only the recovered prefix is discounted; every other case keeps its existing basis
|
|
1328
|
+
return (end !== undefined ? end - start : buffer.byteLength) - (entryStart - start);
|
|
735
1329
|
},
|
|
736
1330
|
getValue(store, fullRecord, auditTime) {
|
|
737
1331
|
if (action & HAS_RECORD || (action & HAS_PARTIAL_RECORD && !fullRecord)) {
|
|
@@ -776,6 +1370,22 @@ function readAuditEntry(buffer, start = 0, end = undefined) {
|
|
|
776
1370
|
return createCorruptAuditSentinel(buffer, start, end);
|
|
777
1371
|
}
|
|
778
1372
|
}
|
|
1373
|
+
/**
|
|
1374
|
+
* Reject a header whose first byte can be neither an action nor a previousVersion field. Returns the
|
|
1375
|
+
* sentinel directly rather than throwing into readAuditEntry's catch, whose error log is not
|
|
1376
|
+
* contained: a throwing log sink there would escape the decoder and stall the iteration it runs in.
|
|
1377
|
+
*/
|
|
1378
|
+
function corruptEntry(buffer, start, end, firstByte) {
|
|
1379
|
+
if (!warnedUndecodableHeader) {
|
|
1380
|
+
warnedUndecodableHeader = true;
|
|
1381
|
+
warnContained('Audit entry header begins with neither an action nor a previousVersion; treating as corrupt', {
|
|
1382
|
+
firstByte,
|
|
1383
|
+
// the 8 prefix bytes plus the action byte: the classifying bytes, stopping before the recordId
|
|
1384
|
+
header: Buffer.from(buffer.subarray(start, Math.min(start + 9, buffer.byteLength))).toString('hex'),
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
return createCorruptAuditSentinel(buffer, start, end);
|
|
1388
|
+
}
|
|
779
1389
|
/**
|
|
780
1390
|
* Build a structurally complete audit record for an entry that failed to decode. The fields
|
|
781
1391
|
* mirror the happy-path shape so downstream consumers that access (e.g.) `getValue` or the
|
|
@@ -788,6 +1398,7 @@ function readAuditEntry(buffer, start = 0, end = undefined) {
|
|
|
788
1398
|
function createCorruptAuditSentinel(buffer, start, end) {
|
|
789
1399
|
return {
|
|
790
1400
|
type: undefined,
|
|
1401
|
+
logName: undefined,
|
|
791
1402
|
tableId: undefined,
|
|
792
1403
|
nodeId: undefined,
|
|
793
1404
|
recordId: undefined,
|