harper 5.1.20 → 5.2.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/operations.ts +13 -16
- package/components/Application.ts +188 -7
- package/components/ApplicationScope.ts +3 -0
- package/components/Scope.ts +12 -0
- package/components/anthropic/index.ts +9 -2
- package/components/bedrock/index.ts +25 -2
- package/components/componentLoader.ts +56 -7
- package/components/componentSecrets.ts +451 -0
- package/components/deploymentRecorder.ts +7 -1
- package/components/mcp/adapters/fastify.ts +3 -0
- package/components/mcp/adapters/harperHttp.ts +3 -0
- package/components/mcp/audit.ts +21 -6
- package/components/mcp/customResourceRegistry.ts +17 -38
- package/components/mcp/listChanged.ts +37 -20
- package/components/mcp/quota.ts +145 -0
- package/components/mcp/rateLimit.ts +152 -14
- package/components/mcp/resources.ts +2 -4
- package/components/mcp/toolRegistry.ts +73 -4
- package/components/mcp/tools/application.ts +77 -8
- package/components/mcp/tools/operations.ts +106 -36
- package/components/mcp/tools/schemas/operationDescriptions.ts +4 -4
- package/components/mcp/tools/schemas/operations.ts +5 -0
- package/components/mcp/transport.ts +55 -4
- package/components/ollama/index.ts +6 -2
- package/components/openai/index.ts +9 -2
- package/components/operations.js +204 -10
- package/components/operationsValidation.js +173 -0
- package/components/secretOperations.ts +498 -0
- package/config/configUtils.ts +25 -1
- package/config-app.schema.json +81 -0
- package/config-root.schema.json +8 -1
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +4 -3
- package/dataLayer/readAuditLog.ts +14 -0
- package/dataLayer/schemaDescribe.ts +1 -0
- package/dist/agent/operations.d.ts +7 -4
- package/dist/agent/operations.js +13 -15
- package/dist/agent/operations.js.map +1 -1
- package/dist/components/Application.d.ts +25 -2
- package/dist/components/Application.js +166 -7
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +2 -0
- package/dist/components/ApplicationScope.js +3 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/Scope.d.ts +8 -0
- package/dist/components/Scope.js +11 -0
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/anthropic/index.d.ts +4 -1
- package/dist/components/anthropic/index.js +4 -2
- package/dist/components/anthropic/index.js.map +1 -1
- package/dist/components/bedrock/index.d.ts +4 -1
- package/dist/components/bedrock/index.js +22 -2
- package/dist/components/bedrock/index.js.map +1 -1
- package/dist/components/componentLoader.d.ts +2 -0
- package/dist/components/componentLoader.js +52 -10
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentSecrets.d.ts +63 -0
- package/dist/components/componentSecrets.js +405 -0
- package/dist/components/componentSecrets.js.map +1 -0
- package/dist/components/deploymentRecorder.d.ts +3 -0
- package/dist/components/deploymentRecorder.js +4 -3
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/mcp/adapters/fastify.d.ts +2 -0
- package/dist/components/mcp/adapters/fastify.js +1 -0
- package/dist/components/mcp/adapters/fastify.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.d.ts +2 -0
- package/dist/components/mcp/adapters/harperHttp.js +1 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/audit.d.ts +2 -2
- package/dist/components/mcp/audit.js +20 -5
- package/dist/components/mcp/audit.js.map +1 -1
- package/dist/components/mcp/customResourceRegistry.d.ts +5 -12
- package/dist/components/mcp/customResourceRegistry.js +15 -30
- package/dist/components/mcp/customResourceRegistry.js.map +1 -1
- package/dist/components/mcp/listChanged.d.ts +7 -2
- package/dist/components/mcp/listChanged.js +28 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/quota.d.ts +31 -0
- package/dist/components/mcp/quota.js +155 -0
- package/dist/components/mcp/quota.js.map +1 -0
- package/dist/components/mcp/rateLimit.d.ts +19 -3
- package/dist/components/mcp/rateLimit.js +129 -13
- package/dist/components/mcp/rateLimit.js.map +1 -1
- package/dist/components/mcp/resources.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +39 -1
- package/dist/components/mcp/toolRegistry.js +60 -5
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/application.d.ts +8 -1
- package/dist/components/mcp/tools/application.js +65 -5
- package/dist/components/mcp/tools/application.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +15 -3
- package/dist/components/mcp/tools/operations.js +99 -38
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +4 -4
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operations.js +4 -0
- package/dist/components/mcp/tools/schemas/operations.js.map +1 -1
- package/dist/components/mcp/transport.d.ts +6 -0
- package/dist/components/mcp/transport.js +47 -3
- package/dist/components/mcp/transport.js.map +1 -1
- package/dist/components/ollama/index.d.ts +4 -1
- package/dist/components/ollama/index.js +4 -2
- package/dist/components/ollama/index.js.map +1 -1
- package/dist/components/openai/index.d.ts +4 -1
- package/dist/components/openai/index.js +4 -2
- package/dist/components/openai/index.js.map +1 -1
- package/dist/components/operations.d.ts +39 -0
- package/dist/components/operations.js +194 -2
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.d.ts +38 -0
- package/dist/components/operationsValidation.js +155 -0
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/secretOperations.d.ts +113 -0
- package/dist/components/secretOperations.js +475 -0
- package/dist/components/secretOperations.js.map +1 -0
- package/dist/config/configUtils.d.ts +4 -1
- package/dist/config/configUtils.js +18 -1
- package/dist/config/configUtils.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +4 -3
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js.map +1 -1
- package/dist/dataLayer/readAuditLog.js +11 -0
- package/dist/dataLayer/readAuditLog.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +2 -0
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/globals.d.ts +1 -1
- package/dist/globals.js +1 -0
- package/dist/globals.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +38 -0
- package/dist/resources/DatabaseTransaction.d.ts +31 -3
- package/dist/resources/DatabaseTransaction.js +190 -29
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +1 -1
- package/dist/resources/LMDBTransaction.js +43 -5
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +51 -0
- package/dist/resources/PrimaryRocksDatabase.js +194 -0
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -0
- package/dist/resources/RecordEncoder.d.ts +1 -0
- package/dist/resources/RecordEncoder.js +81 -75
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +19 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.d.ts +0 -1
- package/dist/resources/Resource.js +103 -16
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +6 -2
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/Resources.js.map +1 -1
- package/dist/resources/Table.d.ts +6 -4
- package/dist/resources/Table.js +247 -24
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/read.js +25 -28
- package/dist/resources/analytics/read.js.map +1 -1
- package/dist/resources/blob.js +165 -22
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/crdt.d.ts +2 -0
- package/dist/resources/crdt.js +45 -36
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/databases.d.ts +1 -0
- package/dist/resources/databases.js +134 -7
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/graphql.js +8 -0
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +35 -5
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +239 -14
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/jsResource.js +6 -0
- package/dist/resources/jsResource.js.map +1 -1
- package/dist/resources/loadEnv.js +30 -3
- package/dist/resources/loadEnv.js.map +1 -1
- package/dist/resources/login.js +2 -2
- package/dist/resources/login.js.map +1 -1
- package/dist/resources/models/Models.js +14 -3
- package/dist/resources/models/Models.js.map +1 -1
- package/dist/resources/models/embedHook.js +10 -2
- package/dist/resources/models/embedHook.js.map +1 -1
- package/dist/resources/models/openaiStream.d.ts +56 -0
- package/dist/resources/models/openaiStream.js +94 -0
- package/dist/resources/models/openaiStream.js.map +1 -0
- package/dist/resources/replayLogs.js +4 -3
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/search.d.ts +1 -1
- package/dist/resources/search.js +104 -11
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/secretDecryptor.d.ts +54 -0
- package/dist/resources/secretDecryptor.js +131 -0
- package/dist/resources/secretDecryptor.js.map +1 -0
- package/dist/resources/tracked.js +15 -5
- package/dist/resources/tracked.js.map +1 -1
- package/dist/security/auth.js +71 -8
- package/dist/security/auth.js.map +1 -1
- package/dist/security/jsLoader.js +11 -0
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +9 -1
- package/dist/security/tokenAuthentication.js +31 -1
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +1 -1
- package/dist/server/DurableSubscriptionsSession.js +8 -4
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +28 -9
- package/dist/server/REST.js.map +1 -1
- package/dist/server/fastifyRoutes.js +11 -1
- package/dist/server/fastifyRoutes.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -3
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +40 -2
- package/dist/server/http.js +560 -77
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +64 -0
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +12 -0
- package/dist/server/liveSubscriptionAuth.js +140 -0
- package/dist/server/liveSubscriptionAuth.js.map +1 -0
- package/dist/server/loadRootComponents.js +2 -1
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/middlewareChain.d.ts +44 -1
- package/dist/server/middlewareChain.js +86 -10
- package/dist/server/middlewareChain.js.map +1 -1
- package/dist/server/mqtt.js +2 -1
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/operationsServer.js +1 -1
- package/dist/server/operationsServer.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +9 -0
- package/dist/server/serverHelpers/Headers.js +22 -0
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/JSONStream.js +3 -3
- package/dist/server/serverHelpers/JSONStream.js.map +1 -1
- package/dist/server/serverHelpers/Request.d.ts +52 -1
- package/dist/server/serverHelpers/Request.js +122 -1
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.js +3 -8
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/progressEmitter.d.ts +19 -0
- package/dist/server/serverHelpers/progressEmitter.js +52 -4
- package/dist/server/serverHelpers/progressEmitter.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +40 -0
- package/dist/server/serverHelpers/registeredOperations.js +286 -0
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +11 -5
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +1 -0
- package/dist/server/serverHelpers/serverUtilities.js +67 -5
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/uwsServer.d.ts +80 -0
- package/dist/server/serverHelpers/uwsServer.js +433 -0
- package/dist/server/serverHelpers/uwsServer.js.map +1 -0
- package/dist/server/static.d.ts +0 -15
- package/dist/server/static.js +161 -2
- package/dist/server/static.js.map +1 -1
- package/dist/server/status/index.d.ts +4 -1
- package/dist/server/status/index.js +64 -4
- package/dist/server/status/index.js.map +1 -1
- package/dist/server/threads/manageThreads.d.ts +10 -0
- package/dist/server/threads/manageThreads.js +107 -4
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.js +5 -1
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadServer.js +92 -14
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/upgrade/directives/5-2-0.d.ts +7 -0
- package/dist/upgrade/directives/5-2-0.js +108 -0
- package/dist/upgrade/directives/5-2-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/common_utils.js +5 -0
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/envFile.d.ts +41 -0
- package/dist/utility/envFile.js +221 -0
- package/dist/utility/envFile.js.map +1 -0
- package/dist/utility/globalSchema.d.ts +9 -0
- package/dist/utility/hdbTerms.d.ts +25 -0
- package/dist/utility/hdbTerms.js +31 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +21 -0
- package/dist/utility/logging/harper_logger.js +160 -8
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logRotator.js +29 -17
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/logging/readLog.d.ts +1 -1
- package/dist/utility/logging/readLog.js +305 -2
- package/dist/utility/logging/readLog.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +18 -0
- package/dist/utility/operationPermissions.js +35 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +17 -0
- package/dist/utility/operation_authorization.js +38 -0
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/secretEnvelope.d.ts +30 -0
- package/dist/utility/secretEnvelope.js +94 -0
- package/dist/utility/secretEnvelope.js.map +1 -0
- package/dist/utility/signalling.d.ts +7 -0
- package/dist/utility/signalling.js +16 -0
- package/dist/utility/signalling.js.map +1 -1
- package/dist/validation/configValidator.js +70 -24
- package/dist/validation/configValidator.js.map +1 -1
- package/index.ts +4 -0
- package/json/systemSchema.json +38 -0
- package/npm-shrinkwrap.json +15066 -9347
- package/package.json +13 -8
- package/resources/DESIGN.md +15 -12
- package/resources/DatabaseTransaction.ts +206 -37
- package/resources/LMDBTransaction.ts +43 -9
- package/resources/PrimaryRocksDatabase.ts +201 -0
- package/resources/RecordEncoder.ts +81 -69
- package/resources/RequestTarget.ts +21 -0
- package/resources/Resource.ts +109 -25
- package/resources/ResourceInterface.ts +6 -2
- package/resources/Resources.ts +1 -3
- package/resources/Table.ts +238 -29
- package/resources/analytics/read.ts +25 -30
- package/resources/blob.ts +159 -23
- package/resources/crdt.ts +37 -31
- package/resources/databases.ts +143 -6
- package/resources/graphql.ts +8 -0
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +251 -17
- package/resources/jsResource.ts +6 -0
- package/resources/loadEnv.ts +34 -3
- package/resources/login.ts +2 -2
- package/resources/models/Models.ts +19 -3
- package/resources/models/embedHook.ts +13 -2
- package/resources/models/openaiStream.ts +133 -0
- package/resources/replayLogs.ts +4 -3
- package/resources/search.ts +113 -11
- package/resources/secretDecryptor.ts +159 -0
- package/resources/tracked.ts +24 -17
- package/security/auth.ts +70 -10
- package/security/jsLoader.ts +12 -0
- package/security/tokenAuthentication.ts +42 -1
- package/server/DESIGN.md +10 -0
- package/server/DurableSubscriptionsSession.ts +8 -4
- package/server/REST.ts +26 -9
- package/server/fastifyRoutes.ts +10 -1
- package/server/graphqlQuerying.ts +4 -3
- package/server/http.ts +563 -77
- package/server/itc/serverHandlers.js +70 -0
- package/server/liveSubscriptionAuth.ts +152 -0
- package/server/loadRootComponents.js +2 -1
- package/server/middlewareChain.ts +107 -17
- package/server/mqtt.ts +2 -1
- package/server/operationsServer.ts +2 -2
- package/server/serverHelpers/Headers.ts +24 -0
- package/server/serverHelpers/JSONStream.ts +3 -3
- package/server/serverHelpers/Request.ts +127 -0
- package/server/serverHelpers/contentTypes.ts +3 -8
- package/server/serverHelpers/progressEmitter.ts +55 -5
- package/server/serverHelpers/registeredOperations.ts +259 -0
- package/server/serverHelpers/serverHandlers.js +11 -5
- package/server/serverHelpers/serverUtilities.ts +88 -5
- package/server/serverHelpers/uwsServer.ts +488 -0
- package/server/static.ts +180 -2
- package/server/status/index.ts +72 -4
- package/server/threads/manageThreads.js +102 -4
- package/server/threads/socketRouter.ts +5 -1
- package/server/threads/threadServer.js +93 -10
- package/studio/web/assets/{Chat-SdD1EZca.js → Chat-Cv_2paZE.js} +2 -2
- package/studio/web/assets/{Chat-SdD1EZca.js.map → Chat-Cv_2paZE.js.map} +1 -1
- package/studio/web/assets/{FloatingChat-GMq6tHK9.js → FloatingChat-CPyPIcVR.js} +4 -4
- package/studio/web/assets/{FloatingChat-GMq6tHK9.js.map → FloatingChat-CPyPIcVR.js.map} +1 -1
- package/studio/web/assets/{applications-B5hhv7uA.js → applications-C0jT2vdZ.js} +2 -2
- package/studio/web/assets/{applications-B5hhv7uA.js.map → applications-C0jT2vdZ.js.map} +1 -1
- package/studio/web/assets/{index-VFcpNWgq.js → index-D_GKOkhn.js} +6 -6
- package/studio/web/assets/index-D_GKOkhn.js.map +1 -0
- package/studio/web/assets/{index.lazy-DgcJojIA.js → index.lazy-CAmCIA65.js} +4 -4
- package/studio/web/assets/{index.lazy-DgcJojIA.js.map → index.lazy-CAmCIA65.js.map} +1 -1
- package/studio/web/assets/{profile-BqN-8vzm.js → profile-CaF-4aZe.js} +2 -2
- package/studio/web/assets/{profile-BqN-8vzm.js.map → profile-CaF-4aZe.js.map} +1 -1
- package/studio/web/assets/{setComponentFile-CHFvFq9u.js → setComponentFile-DLW1DHCt.js} +2 -2
- package/studio/web/assets/{setComponentFile-CHFvFq9u.js.map → setComponentFile-DLW1DHCt.js.map} +1 -1
- package/studio/web/assets/{setup-CQ3vWJSr.js → setup-TGCErIhq.js} +2 -2
- package/studio/web/assets/{setup-CQ3vWJSr.js.map → setup-TGCErIhq.js.map} +1 -1
- package/studio/web/assets/{status-BsW5fjuh.js → status-DG0Maoao.js} +2 -2
- package/studio/web/assets/{status-BsW5fjuh.js.map → status-DG0Maoao.js.map} +1 -1
- package/studio/web/assets/{swagger-ui-react-DITdeB9-.js → swagger-ui-react-8T4SgQ1m.js} +2 -2
- package/studio/web/assets/{swagger-ui-react-DITdeB9-.js.map → swagger-ui-react-8T4SgQ1m.js.map} +1 -1
- package/studio/web/assets/{tsMode-Dyph-OUs.js → tsMode-DJ6Sl24Q.js} +2 -2
- package/studio/web/assets/{tsMode-Dyph-OUs.js.map → tsMode-DJ6Sl24Q.js.map} +1 -1
- package/studio/web/assets/{useEntityRestURL-D0QTUqex.js → useEntityRestURL-JO2mfWTQ.js} +2 -2
- package/studio/web/assets/{useEntityRestURL-D0QTUqex.js.map → useEntityRestURL-JO2mfWTQ.js.map} +1 -1
- package/studio/web/index.html +1 -1
- package/upgrade/directives/5-2-0.ts +82 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/common_utils.ts +5 -0
- package/utility/envFile.ts +218 -0
- package/utility/hdbTerms.ts +31 -0
- package/utility/logging/harper_logger.ts +156 -8
- package/utility/logging/logRotator.ts +24 -14
- package/utility/logging/readLog.ts +323 -2
- package/utility/operationPermissions.ts +35 -1
- package/utility/operation_authorization.ts +74 -1
- package/utility/secretEnvelope.ts +113 -0
- package/utility/signalling.ts +15 -0
- package/validation/configValidator.ts +78 -25
- package/studio/web/assets/index-VFcpNWgq.js.map +0 -1
package/dist/resources/Table.js
CHANGED
|
@@ -42,6 +42,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
42
42
|
};
|
|
43
43
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
44
44
|
exports.EVICTED = exports.INVALIDATED = void 0;
|
|
45
|
+
exports.freezeRecord = freezeRecord;
|
|
45
46
|
exports.makeTable = makeTable;
|
|
46
47
|
exports.coerceType = coerceType;
|
|
47
48
|
const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
|
|
@@ -120,6 +121,18 @@ function isFrozenRecordObject(value) {
|
|
|
120
121
|
!(value instanceof ArrayBuffer) &&
|
|
121
122
|
Object.isFrozen(value));
|
|
122
123
|
}
|
|
124
|
+
// Freeze a decoded record value for cache integrity, guarding the bare-TypedArray-root case:
|
|
125
|
+
// V8 throws "Cannot freeze array buffer views with elements" on Object.freeze of a non-empty
|
|
126
|
+
// TypedArray/DataView (#1298). _writeUpdate now rejects such roots on write, but this read-side
|
|
127
|
+
// guard still backstops records that bypass validation (source/cache population, replication of
|
|
128
|
+
// legacy data) and lets an already-poisoned table be read again after upgrade. The freeze is
|
|
129
|
+
// shallow anyway, so a typed-array root needs none. Only freeze plain objects: skip ArrayBuffer
|
|
130
|
+
// views/ArrayBuffers, and short-circuit primitives/null/undefined to avoid a needless native
|
|
131
|
+
// Object.freeze call on the hot read/scan path.
|
|
132
|
+
function freezeRecord(value) {
|
|
133
|
+
if (value !== null && typeof value === 'object' && !ArrayBuffer.isView(value) && !(value instanceof ArrayBuffer))
|
|
134
|
+
Object.freeze(value);
|
|
135
|
+
}
|
|
123
136
|
exports.INVALIDATED = 1;
|
|
124
137
|
exports.EVICTED = 8; // note that 2 is reserved for timestamps
|
|
125
138
|
const TEST_WRITE_KEY_BUFFER = Buffer.allocUnsafeSlow(8192);
|
|
@@ -145,7 +158,7 @@ const FULL_PERMISSIONS = {
|
|
|
145
158
|
*/
|
|
146
159
|
// #section: setup-and-factory
|
|
147
160
|
function makeTable(options) {
|
|
148
|
-
const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, } = options;
|
|
161
|
+
const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, } = options;
|
|
149
162
|
let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
|
|
150
163
|
evictionMs ??= 0;
|
|
151
164
|
// Eviction without explicit expiration means expiration:0. Apply at construction so
|
|
@@ -249,6 +262,8 @@ function makeTable(options) {
|
|
|
249
262
|
static description = description;
|
|
250
263
|
static properties = properties;
|
|
251
264
|
static hidden = hidden;
|
|
265
|
+
// default `Cache-Control` for anonymous REST reads (from `@table(cacheControl: "...")`), see REST.ts
|
|
266
|
+
static cacheControl = cacheControl;
|
|
252
267
|
static outputSchemas;
|
|
253
268
|
static mcp;
|
|
254
269
|
static replicate = replicate;
|
|
@@ -669,8 +684,8 @@ function makeTable(options) {
|
|
|
669
684
|
// dictates not to go to source
|
|
670
685
|
if (!this.doesExist())
|
|
671
686
|
throw new hdbError_ts_1.ServerError('Entry is not cached', 504);
|
|
672
|
-
if (hasSourceGet
|
|
673
|
-
target
|
|
687
|
+
if (hasSourceGet)
|
|
688
|
+
setLoadedFromSource(target, false); // mark it as cached
|
|
674
689
|
}
|
|
675
690
|
else if (resourceOptions?.ensureLoaded) {
|
|
676
691
|
const loadingFromSource = ensureLoadedFromSource(this.constructor.source, id, entry, request, this, target);
|
|
@@ -682,7 +697,7 @@ function makeTable(options) {
|
|
|
682
697
|
});
|
|
683
698
|
}
|
|
684
699
|
else if (hasSourceGet)
|
|
685
|
-
target
|
|
700
|
+
setLoadedFromSource(target, false); // mark it as cached
|
|
686
701
|
}
|
|
687
702
|
return this;
|
|
688
703
|
});
|
|
@@ -1139,7 +1154,13 @@ function makeTable(options) {
|
|
|
1139
1154
|
let allowed = true;
|
|
1140
1155
|
if (target?.checkPermission) {
|
|
1141
1156
|
// requesting authorization verification
|
|
1142
|
-
|
|
1157
|
+
try {
|
|
1158
|
+
allowed = this.allowRead(context.user, target, context);
|
|
1159
|
+
}
|
|
1160
|
+
catch {
|
|
1161
|
+
// allow* threw — fail closed rather than letting the request proceed
|
|
1162
|
+
throw new hdbError_ts_1.AccessViolation(context.user);
|
|
1163
|
+
}
|
|
1143
1164
|
}
|
|
1144
1165
|
return (0, when_ts_1.promiseNormalize)((0, when_ts_1.when)((0, when_ts_1.when)(allowed, (allowed) => {
|
|
1145
1166
|
if (!allowed) {
|
|
@@ -1153,6 +1174,8 @@ function makeTable(options) {
|
|
|
1153
1174
|
// dictates not to go to source
|
|
1154
1175
|
if (!entry?.value)
|
|
1155
1176
|
throw new hdbError_ts_1.ServerError('Entry is not cached', 504);
|
|
1177
|
+
if (hasSourceGet)
|
|
1178
|
+
setLoadedFromSource(target, false); // mark it as cached
|
|
1156
1179
|
}
|
|
1157
1180
|
else if (ensureLoaded) {
|
|
1158
1181
|
const loadingFromSource = ensureLoadedFromSource(constructor.source, id, entry, context, this, target);
|
|
@@ -1160,6 +1183,8 @@ function makeTable(options) {
|
|
|
1160
1183
|
txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
|
|
1161
1184
|
return loadingFromSource.then((entry) => entry?.value);
|
|
1162
1185
|
}
|
|
1186
|
+
else if (hasSourceGet)
|
|
1187
|
+
setLoadedFromSource(target, false); // mark it as cached
|
|
1163
1188
|
}
|
|
1164
1189
|
return entry?.value;
|
|
1165
1190
|
});
|
|
@@ -1223,7 +1248,10 @@ function makeTable(options) {
|
|
|
1223
1248
|
property = { name: property };
|
|
1224
1249
|
if (!property.checkPermission && target.checkPermission)
|
|
1225
1250
|
property.checkPermission = target.checkPermission;
|
|
1226
|
-
|
|
1251
|
+
// Invoke the related table's allowRead on a properly-bound instance rather than
|
|
1252
|
+
// `.call(null, ...)` so `this` is a valid resource of the related type.
|
|
1253
|
+
const relatedResource = new relatedTable(undefined, context);
|
|
1254
|
+
if (!relatedResource.allowRead(user, property, context))
|
|
1227
1255
|
return false;
|
|
1228
1256
|
if (!property.select)
|
|
1229
1257
|
return property.name; // no select was applied, just return the name
|
|
@@ -1798,6 +1826,30 @@ function makeTable(options) {
|
|
|
1798
1826
|
if (fullUpdate || (recordUpdate && (0, tracked_ts_1.hasChanges)(this.#changes === recordUpdate ? this : recordUpdate))) {
|
|
1799
1827
|
if (!context?.source) {
|
|
1800
1828
|
transaction.checkOverloaded();
|
|
1829
|
+
// A record must be a plain object. Reject primitive, string/number, bare-binary,
|
|
1830
|
+
// and bare-array roots — e.g. a raw Buffer from an application/octet-stream PUT, a
|
|
1831
|
+
// JSON string/number body, or a top-level JSON array. Such roots carry no primary
|
|
1832
|
+
// key or attributes, are meaningless to SQL/get-attributes, and a bare TypedArray
|
|
1833
|
+
// root additionally throws on Object.freeze during scans (#1298). Binary data belongs
|
|
1834
|
+
// in a Bytes/Blob attribute. (Messages go through _writePublish, not here, so raw
|
|
1835
|
+
// publish payloads are unaffected.)
|
|
1836
|
+
const isBinaryRoot = ArrayBuffer.isView(recordUpdate) || recordUpdate instanceof ArrayBuffer;
|
|
1837
|
+
if (recordUpdate === null ||
|
|
1838
|
+
typeof recordUpdate !== 'object' ||
|
|
1839
|
+
isBinaryRoot ||
|
|
1840
|
+
Array.isArray(recordUpdate)) {
|
|
1841
|
+
// Avoid dumping every byte of a large binary body or huge payload into the error.
|
|
1842
|
+
let received;
|
|
1843
|
+
if (isBinaryRoot) {
|
|
1844
|
+
received = `${recordUpdate.constructor?.name ?? 'binary'} of ${recordUpdate.byteLength} bytes`;
|
|
1845
|
+
}
|
|
1846
|
+
else {
|
|
1847
|
+
const full = stringify(recordUpdate) ?? typeof recordUpdate;
|
|
1848
|
+
received = full.length > 200 ? full.slice(0, 200) + '...' : full;
|
|
1849
|
+
}
|
|
1850
|
+
throw new hdbError_ts_1.ClientError(`A record must be an object, but received ${received}. To store binary data, put it ` +
|
|
1851
|
+
`in a Bytes or Blob attribute (e.g. a record like { "${primaryKey ?? 'id'}": …, "data": <bytes> }).`);
|
|
1852
|
+
}
|
|
1801
1853
|
// Records are intentionally immutable: decoded records are frozen (and 5.2 record
|
|
1802
1854
|
// caching relies on it), so mutating in place would corrupt cached/shared state.
|
|
1803
1855
|
// validate() coerces values and we stamp created/updated times + the primary key
|
|
@@ -1853,6 +1905,16 @@ function makeTable(options) {
|
|
|
1853
1905
|
},
|
|
1854
1906
|
before: writeToSource(),
|
|
1855
1907
|
commit: (txnTime, existingEntry, retry, transaction) => {
|
|
1908
|
+
// Whether a prior attempt of THIS write appended its own audit entry (sticky, set in
|
|
1909
|
+
// save(); log entries are not part of the aborted rocks transaction, so they survive).
|
|
1910
|
+
// Only such a write can find its own orphaned entry in the dedup lookups below and must
|
|
1911
|
+
// not treat it as "already applied". Per-write on purpose: the transaction-wide retry
|
|
1912
|
+
// flag would also suppress dedup for a genuine re-delivered duplicate co-batched with
|
|
1913
|
+
// the conflicting write (double-applying it) and for fresh writes staged through a
|
|
1914
|
+
// reused transaction whose retries counter is stale. Sticky on purpose: a proxy read
|
|
1915
|
+
// from the last attempt's skipped state launders when a recommit round self-skips
|
|
1916
|
+
// (walk identity tie against its own staged record) before a fresh-transaction replay.
|
|
1917
|
+
const stagedOwnAuditEntry = retry && write.appendedAuditEntry === true;
|
|
1856
1918
|
write.skipped = false; // reset on each retry; cleanup happens after commit if still true
|
|
1857
1919
|
if (retry) {
|
|
1858
1920
|
if (context && existingEntry?.version > (context.lastModified || 0))
|
|
@@ -1964,8 +2026,12 @@ function makeTable(options) {
|
|
|
1964
2026
|
// local audit time, not version, so this version-keyed lookup doesn't apply there (LMDB keeps the
|
|
1965
2027
|
// exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
|
|
1966
2028
|
// simply falls through to the walk, so this never changes correctness; the additionalAuditRefs
|
|
1967
|
-
// check above remains the read-your-writes guard.
|
|
1968
|
-
|
|
2029
|
+
// check above remains the read-your-writes guard. Never when this write staged in a prior
|
|
2030
|
+
// failed attempt: that attempt already appended this write's own audit entry, so the lookup
|
|
2031
|
+
// would find it and skip the write as "already applied" when the record was never committed.
|
|
2032
|
+
// A recommit of the same transaction survived that skip only because the old write batch
|
|
2033
|
+
// still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
|
|
2034
|
+
if (isRocksDB && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnTime)) {
|
|
1969
2035
|
const priorAudit = auditStore.get(txnTime, tableId, id, options?.nodeId);
|
|
1970
2036
|
if (priorAudit &&
|
|
1971
2037
|
priorAudit.version === txnTime &&
|
|
@@ -2007,7 +2073,12 @@ function makeTable(options) {
|
|
|
2007
2073
|
// A re-delivered write whose exact (version, nodeId) is already in the audit log was already
|
|
2008
2074
|
// applied; drop it rather than re-applying it (double-applying commutative ops) or writing a
|
|
2009
2075
|
// duplicate audit-only record. Used by the early-out and the depth-cap block below.
|
|
2076
|
+
// Never a duplicate for a write that staged in a prior failed attempt: that attempt already
|
|
2077
|
+
// appended this write's own audit entry, so the lookup would match it while the record was
|
|
2078
|
+
// never committed (see the up-front keyed dedup above).
|
|
2010
2079
|
const isReDeliveredDuplicate = () => {
|
|
2080
|
+
if (stagedOwnAuditEntry)
|
|
2081
|
+
return false;
|
|
2011
2082
|
if (!dedupVersionCouldBeRetained(txnTime))
|
|
2012
2083
|
return false; // pre-retention version — skip the end-of-log scan (best-effort; see above)
|
|
2013
2084
|
const duplicate = auditStore.get(txnTime, tableId, id, options?.nodeId);
|
|
@@ -2390,7 +2461,14 @@ function makeTable(options) {
|
|
|
2390
2461
|
throw target.parseError; // if there was a parse error, we can throw it now
|
|
2391
2462
|
if (target.checkPermission) {
|
|
2392
2463
|
// requesting authorization verification
|
|
2393
|
-
|
|
2464
|
+
let allowed;
|
|
2465
|
+
try {
|
|
2466
|
+
allowed = this.allowRead(context.user, target, context);
|
|
2467
|
+
}
|
|
2468
|
+
catch {
|
|
2469
|
+
// allow* threw — fail closed rather than letting the request proceed
|
|
2470
|
+
throw new hdbError_ts_1.AccessViolation(context.user);
|
|
2471
|
+
}
|
|
2394
2472
|
if (!allowed) {
|
|
2395
2473
|
throw new hdbError_ts_1.AccessViolation(context.user);
|
|
2396
2474
|
}
|
|
@@ -2590,10 +2668,32 @@ function makeTable(options) {
|
|
|
2590
2668
|
// transaction, and we really don't care if the
|
|
2591
2669
|
// counts are done in the same read transaction because they are just estimates) until the search
|
|
2592
2670
|
// results have been iterated and finished.
|
|
2593
|
-
|
|
2594
|
-
|
|
2671
|
+
// When the query opts out of a snapshot (`snapshot: false`, e.g. long-running analytics
|
|
2672
|
+
// scans), the read transaction reads against the latest committed data without pinning a
|
|
2673
|
+
// consistent snapshot, so the scan doesn't hold a snapshot that blocks compaction.
|
|
2674
|
+
const readTxn = txn.useReadTxn(target.snapshot === false);
|
|
2675
|
+
// Record-level read guard (#1241): a resource may define a static allowReadRecord(user, record)
|
|
2676
|
+
// that participates in query execution (pushed into HNSW traversal for vector sorts, applied as a
|
|
2677
|
+
// post-filter otherwise). Resolve off the actual (possibly subclassed) constructor so overrides win.
|
|
2678
|
+
// SCOPE: this is a QUERY-result filter, not a general read-authorization boundary — direct
|
|
2679
|
+
// single-record get(id) does not consult it (use allowRead/instance hooks for that). It must be
|
|
2680
|
+
// synchronous, side-effect free, and fast; it can run once per candidate record during traversal.
|
|
2681
|
+
const allowReadRecord = this.constructor.allowReadRecord;
|
|
2682
|
+
const recordAccess = typeof allowReadRecord === 'function' || typeof target.vectorFilter === 'function'
|
|
2683
|
+
? { allowReadRecord, user: context?.user, vectorFilter: target.vectorFilter }
|
|
2684
|
+
: undefined;
|
|
2685
|
+
const entries = (0, search_ts_1.executeConditions)(conditions, operator, TableResource, readTxn, target, context, (results, filters) => transformToEntries(results, select, context, readTxn, filters), filtered, recordAccess);
|
|
2595
2686
|
const ensure_loaded = target.ensureLoaded !== false;
|
|
2596
|
-
|
|
2687
|
+
// Authoritative RBAC enforcement (#1241): the guards inside executeConditions evaluate the LOCAL
|
|
2688
|
+
// record, but on a caching table transformEntryForSelect may then revalidate an expired/invalidated
|
|
2689
|
+
// row from source and return a DIFFERENT record. An authorization check must hold on the record
|
|
2690
|
+
// actually returned, so allowReadRecord is re-checked there, after materialization (the earlier
|
|
2691
|
+
// evaluation stays as a prune that also bounds HNSW traversal). vectorFilter and condition filters
|
|
2692
|
+
// intentionally keep the local-record semantics all query filters have on caching tables.
|
|
2693
|
+
const recordGuard = typeof allowReadRecord === 'function'
|
|
2694
|
+
? (record) => allowReadRecord(context?.user, record)
|
|
2695
|
+
: undefined;
|
|
2696
|
+
const transformToRecord = TableResource.transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded, true, recordGuard);
|
|
2597
2697
|
let results = TableResource.transformToOrderedSelect(entries, select, postOrdering, context, readTxn, transformToRecord);
|
|
2598
2698
|
// apply any offset/limit after all the sorting and filtering
|
|
2599
2699
|
if (target.offset || target.limit !== undefined)
|
|
@@ -2807,9 +2907,12 @@ function makeTable(options) {
|
|
|
2807
2907
|
* @param filtered
|
|
2808
2908
|
* @param ensure_loaded
|
|
2809
2909
|
* @param canSkip
|
|
2910
|
+
* @param recordGuard record-level read check (#1241) applied to the record actually being
|
|
2911
|
+
* returned — i.e. AFTER any caching-source revalidation replaces a stale local copy — so an
|
|
2912
|
+
* authorization verdict can't be made on bytes that differ from what the caller receives.
|
|
2810
2913
|
* @returns
|
|
2811
2914
|
*/
|
|
2812
|
-
static transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded, canSkip) {
|
|
2915
|
+
static transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded, canSkip, recordGuard) {
|
|
2813
2916
|
let checkLoaded;
|
|
2814
2917
|
if (ensure_loaded &&
|
|
2815
2918
|
hasSourceGet &&
|
|
@@ -2828,6 +2931,7 @@ function makeTable(options) {
|
|
|
2828
2931
|
}
|
|
2829
2932
|
let transformCache;
|
|
2830
2933
|
const source = this.source;
|
|
2934
|
+
const resourceClass = this;
|
|
2831
2935
|
// Transform an entry to a record. Note that *this* instance is intended to be the iterator.
|
|
2832
2936
|
const transform = function (entry) {
|
|
2833
2937
|
let record;
|
|
@@ -2859,7 +2963,23 @@ function makeTable(options) {
|
|
|
2859
2963
|
message: 'This entry has expired',
|
|
2860
2964
|
};
|
|
2861
2965
|
}
|
|
2862
|
-
|
|
2966
|
+
// Stale-while-revalidate is an instance method, but a query has no per-row resource
|
|
2967
|
+
// instance to consult (the single-record `get` path passes `this`). Construct one for this
|
|
2968
|
+
// row — via the same `new constructor(id, context)` the framework's getResource uses — so
|
|
2969
|
+
// the hook sees the current row's identity (this.getId()) and record state, matching the
|
|
2970
|
+
// single-record path. It must be a real instance, not the bare class prototype: every
|
|
2971
|
+
// resource prototype chain ends in a tracked-property Proxy, so reading an absent property
|
|
2972
|
+
// (probing for an undefined `allowStaleWhileRevalidate`, or a hook touching `this.x`) on a
|
|
2973
|
+
// non-instance invokes getChanges() with no backing state and throws (harper#1578). We also
|
|
2974
|
+
// load the stale entry into it (as the single-record path does via _updateResource) so a
|
|
2975
|
+
// hook consulting this.getRecord()/this.<field> sees the stale row, not undefined. `entry`
|
|
2976
|
+
// here may be lazy (its `.value` is a GC-able deref, undefined once collected), so we set the
|
|
2977
|
+
// already-dereferenced `record` explicitly rather than relying on `entry.value`. This runs
|
|
2978
|
+
// only for the expired/invalidated rows already headed to source, so the cost is negligible.
|
|
2979
|
+
const swrResource = new resourceClass(entry.key ?? entry, context);
|
|
2980
|
+
resourceClass._updateResource(swrResource, entry);
|
|
2981
|
+
swrResource.setRecord(record);
|
|
2982
|
+
const loadingFromSource = ensureLoadedFromSource(source, entry.key ?? entry, entry, context, swrResource);
|
|
2863
2983
|
if (loadingFromSource?.then) {
|
|
2864
2984
|
return loadingFromSource.then(transform);
|
|
2865
2985
|
}
|
|
@@ -2867,6 +2987,11 @@ function makeTable(options) {
|
|
|
2867
2987
|
}
|
|
2868
2988
|
if (record == null)
|
|
2869
2989
|
return canSkip ? extended_iterable_1.SKIP : record;
|
|
2990
|
+
// Record-level RBAC (#1241): enforced here because `record` is now the final, materialized
|
|
2991
|
+
// record — a caching table's source revalidation (above) may have replaced the local copy the
|
|
2992
|
+
// query filters evaluated. Fail closed on the record actually being returned.
|
|
2993
|
+
if (recordGuard && !recordGuard(record))
|
|
2994
|
+
return canSkip ? extended_iterable_1.SKIP : undefined;
|
|
2870
2995
|
if (select && !(select[0] === '*' && select.length === 1)) {
|
|
2871
2996
|
let promises;
|
|
2872
2997
|
const selectAttribute = (attribute, callback) => {
|
|
@@ -3033,6 +3158,9 @@ function makeTable(options) {
|
|
|
3033
3158
|
// snapshot:false, which catches any commits that land during yield points; dropping in the
|
|
3034
3159
|
// listener avoids duplicate delivery.
|
|
3035
3160
|
let dropDuringReplay = false;
|
|
3161
|
+
// Coalescing guards for the reload re-snapshot (harper-pro#495), driven from the listener below.
|
|
3162
|
+
let reloadResnapshotRunning = false;
|
|
3163
|
+
let reloadResnapshotPending = false;
|
|
3036
3164
|
const thisId = requestTargetToId(request) ?? null; // treat undefined and null as the root
|
|
3037
3165
|
const subscription = (0, transactionBroadcast_ts_1.addSubscription)(TableResource, thisId, function (id, auditRecord, localTime, beginTxn) {
|
|
3038
3166
|
if (dropDuringReplay)
|
|
@@ -3045,9 +3173,17 @@ function makeTable(options) {
|
|
|
3045
3173
|
// TODO: Do we want to have a limit to how far out-of-order we are willing to send?
|
|
3046
3174
|
value = auditRecord.getValue?.(primaryStore, getFullRecord);
|
|
3047
3175
|
}
|
|
3048
|
-
else if (type
|
|
3049
|
-
//
|
|
3050
|
-
//
|
|
3176
|
+
else if (type === 'reload') {
|
|
3177
|
+
// Whole-table marker with a null id and no record (harper-pro#489): a copyApply base copy
|
|
3178
|
+
// back-filled rows with no per-row audit events. For a user table, re-deliver the current
|
|
3179
|
+
// scope as 'put's so MQTT/SSE/WS subscribers recover the snapshotted records, and do NOT
|
|
3180
|
+
// forward the bare marker (harper-pro#495). System-DB subscribers (knownNodes etc.) instead
|
|
3181
|
+
// get the raw marker and run their own bespoke whole-table rescan.
|
|
3182
|
+
if (databaseName !== 'system')
|
|
3183
|
+
return scheduleReloadResnapshot();
|
|
3184
|
+
// system DB: fall through to forward the bare 'reload' event verbatim.
|
|
3185
|
+
}
|
|
3186
|
+
else if (type !== 'end_txn') {
|
|
3051
3187
|
// these are events that indicate that the primary record has changed. I believe we always want to simply
|
|
3052
3188
|
// send the latest value. Note that it is fine to synchronously access these records, they should have just
|
|
3053
3189
|
// been written, so are fresh in memory.
|
|
@@ -3327,6 +3463,89 @@ function makeTable(options) {
|
|
|
3327
3463
|
}
|
|
3328
3464
|
subscription.send(event);
|
|
3329
3465
|
}
|
|
3466
|
+
// #region reload re-snapshot (harper-pro#495)
|
|
3467
|
+
// A copyApply base copy back-fills rows as snapshots with NO per-row audit entries, so the live
|
|
3468
|
+
// listener above never fires for them and an already-connected subscriber would miss them until
|
|
3469
|
+
// the next direct write. After the copy, a whole-table 'reload' marker is delivered here (id=null).
|
|
3470
|
+
// For a user table we react by re-delivering the subscription's current scope as ordinary 'put'
|
|
3471
|
+
// events — so EVERY consumer that funnels through subscribe() (MQTT, SSE, WS) recovers the records
|
|
3472
|
+
// uniformly, with no per-protocol handling. System-DB reloads are NOT re-snapshotted: their
|
|
3473
|
+
// subscribers (knownNodes peer-discovery, hdb_certificate CA install) run a bespoke whole-table
|
|
3474
|
+
// rescan off the raw marker, which a per-row re-emit cannot express (it can't drop stale rows).
|
|
3475
|
+
// Yields the latest committed value (snapshot:false) and skips tombstones, mirroring the
|
|
3476
|
+
// omitCurrent initial-snapshot scan.
|
|
3477
|
+
async function* currentScopeRecords() {
|
|
3478
|
+
const isCollection = request.isCollection ?? thisId == null;
|
|
3479
|
+
if (isCollection) {
|
|
3480
|
+
let sinceYield = 0;
|
|
3481
|
+
for (const { key: id, value, version, localTime, size } of primaryStore.getRange({
|
|
3482
|
+
start: thisId ?? false,
|
|
3483
|
+
end: thisId == null ? undefined : [thisId, ordered_binary_1.MAXIMUM_KEY],
|
|
3484
|
+
versions: true,
|
|
3485
|
+
snapshot: false, // no need for a snapshot, just want the latest data
|
|
3486
|
+
})) {
|
|
3487
|
+
if (++sinceYield >= REPLAY_YIELD_INTERVAL) {
|
|
3488
|
+
sinceYield = 0;
|
|
3489
|
+
await rest();
|
|
3490
|
+
}
|
|
3491
|
+
if (!value)
|
|
3492
|
+
continue; // skip tombstones
|
|
3493
|
+
yield { id, localTime, value, version, type: 'put', size };
|
|
3494
|
+
}
|
|
3495
|
+
}
|
|
3496
|
+
else {
|
|
3497
|
+
const entry = primaryStore.getEntry(thisId);
|
|
3498
|
+
if (entry?.value)
|
|
3499
|
+
yield { id: thisId, ...entry, type: 'put' };
|
|
3500
|
+
}
|
|
3501
|
+
}
|
|
3502
|
+
// Drain the current scope into the subscription with the same back-pressure as the live path.
|
|
3503
|
+
// Coalesced: a marker that arrives while a re-snapshot is running just re-arms it once more (so
|
|
3504
|
+
// markers for several tables, or a marker landing mid-scan, are not lost), and we never run two
|
|
3505
|
+
// scans concurrently. The first thing it does is `await rest()` (a setImmediate macrotask), so the
|
|
3506
|
+
// scan never executes inside the synchronous broadcast listener — which on the same-thread
|
|
3507
|
+
// aftercommit path holds an inter-thread lock that must not span event-loop turns.
|
|
3508
|
+
async function runReloadResnapshot() {
|
|
3509
|
+
reloadResnapshotRunning = true;
|
|
3510
|
+
try {
|
|
3511
|
+
await rest(); // defer off the broadcast listener's stack before scanning
|
|
3512
|
+
while (reloadResnapshotPending) {
|
|
3513
|
+
reloadResnapshotPending = false;
|
|
3514
|
+
// Subscription.end() nulls `subscriptions`; bail the moment it closes — before, between,
|
|
3515
|
+
// or mid-scan — so we never scan + send into a dead queue. pending is already cleared, so
|
|
3516
|
+
// the finally re-arm won't re-loop.
|
|
3517
|
+
if (!subscription.subscriptions)
|
|
3518
|
+
return;
|
|
3519
|
+
for await (const record of currentScopeRecords()) {
|
|
3520
|
+
if (!subscription.subscriptions)
|
|
3521
|
+
return;
|
|
3522
|
+
send(record);
|
|
3523
|
+
if (subscription.queue?.length > EVENT_HIGH_WATER_MARK) {
|
|
3524
|
+
if ((await subscription.waitForDrain()) === false)
|
|
3525
|
+
return;
|
|
3526
|
+
}
|
|
3527
|
+
}
|
|
3528
|
+
}
|
|
3529
|
+
}
|
|
3530
|
+
catch (error) {
|
|
3531
|
+
harper_logger_ts_1.default.error?.('Error in reload re-snapshot:', error);
|
|
3532
|
+
}
|
|
3533
|
+
finally {
|
|
3534
|
+
reloadResnapshotRunning = false;
|
|
3535
|
+
// A marker that landed after the last pending-check but before we cleared the flag would
|
|
3536
|
+
// otherwise be dropped — re-arm if so (unless the subscription has since closed).
|
|
3537
|
+
if (subscription.subscriptions && reloadResnapshotPending)
|
|
3538
|
+
scheduleReloadResnapshot();
|
|
3539
|
+
}
|
|
3540
|
+
}
|
|
3541
|
+
function scheduleReloadResnapshot() {
|
|
3542
|
+
if (!subscription.subscriptions)
|
|
3543
|
+
return;
|
|
3544
|
+
reloadResnapshotPending = true;
|
|
3545
|
+
if (!reloadResnapshotRunning)
|
|
3546
|
+
runReloadResnapshot();
|
|
3547
|
+
}
|
|
3548
|
+
// #endregion
|
|
3330
3549
|
return subscription;
|
|
3331
3550
|
}
|
|
3332
3551
|
/**
|
|
@@ -3820,7 +4039,7 @@ function makeTable(options) {
|
|
|
3820
4039
|
hasPromises = true;
|
|
3821
4040
|
// for now, we shouldn't be getting promises until rocksdb
|
|
3822
4041
|
if (TableResource.loadAsInstance === false)
|
|
3823
|
-
|
|
4042
|
+
freezeRecord(returnEntry ? value?.value : value);
|
|
3824
4043
|
return value;
|
|
3825
4044
|
});
|
|
3826
4045
|
return relationship.filterMissing
|
|
@@ -3836,7 +4055,7 @@ function makeTable(options) {
|
|
|
3836
4055
|
});
|
|
3837
4056
|
// for now, we shouldn't be getting promises until rocksdb
|
|
3838
4057
|
if (TableResource.loadAsInstance === false)
|
|
3839
|
-
|
|
4058
|
+
freezeRecord(returnEntry ? value?.value : value);
|
|
3840
4059
|
return value;
|
|
3841
4060
|
};
|
|
3842
4061
|
attribute.set = (object, related) => {
|
|
@@ -4213,7 +4432,7 @@ function makeTable(options) {
|
|
|
4213
4432
|
// we need to freeze entry records to ensure the integrity of the cache;
|
|
4214
4433
|
// but we only do this when users have opted into loadAsInstance/freezeRecords to avoid back-compat
|
|
4215
4434
|
// issues
|
|
4216
|
-
|
|
4435
|
+
freezeRecord(entry?.value);
|
|
4217
4436
|
if (entry?.residencyId &&
|
|
4218
4437
|
entry.metadataFlags & exports.INVALIDATED &&
|
|
4219
4438
|
sourceLoad &&
|
|
@@ -4337,6 +4556,12 @@ function makeTable(options) {
|
|
|
4337
4556
|
return table;
|
|
4338
4557
|
}
|
|
4339
4558
|
}
|
|
4559
|
+
function setLoadedFromSource(target, loadedFromSource) {
|
|
4560
|
+
// cache disposition is a per-get result, recorded on the RequestTarget of the get (#1576)
|
|
4561
|
+
// target may be a primitive id on instance-API calls, which can't hold the flag
|
|
4562
|
+
if (target && typeof target === 'object')
|
|
4563
|
+
target.loadedFromSource = loadedFromSource;
|
|
4564
|
+
}
|
|
4340
4565
|
function ensureLoadedFromSource(source, id, entry, context, resource, target) {
|
|
4341
4566
|
if (context?.onlyIfCached) {
|
|
4342
4567
|
if (!entry?.value)
|
|
@@ -4575,8 +4800,7 @@ function makeTable(options) {
|
|
|
4575
4800
|
whenResolved(getFromSource(source, id, primaryStore.getEntry(id), context, target));
|
|
4576
4801
|
else {
|
|
4577
4802
|
// served from cache after waiting for another request to resolve
|
|
4578
|
-
|
|
4579
|
-
target.loadedFromSource = false;
|
|
4803
|
+
setLoadedFromSource(target, false);
|
|
4580
4804
|
whenResolved(entry);
|
|
4581
4805
|
}
|
|
4582
4806
|
};
|
|
@@ -4590,8 +4814,7 @@ function makeTable(options) {
|
|
|
4590
4814
|
});
|
|
4591
4815
|
}
|
|
4592
4816
|
// lock acquired — this request will actually load from source
|
|
4593
|
-
|
|
4594
|
-
target.loadedFromSource = true;
|
|
4817
|
+
setLoadedFromSource(target, true);
|
|
4595
4818
|
const existingRecord = existingEntry?.value;
|
|
4596
4819
|
// it is important to remember that this is _NOT_ part of the current transaction; nothing is changing
|
|
4597
4820
|
// with the canonical data, we are simply fulfilling our local copy of the canonical data, but still don't
|