harper 5.1.20 → 5.2.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/operations.ts +13 -16
- package/components/Application.ts +188 -7
- package/components/ApplicationScope.ts +3 -0
- package/components/Scope.ts +12 -0
- package/components/anthropic/index.ts +9 -2
- package/components/bedrock/index.ts +25 -2
- package/components/componentLoader.ts +56 -7
- package/components/componentSecrets.ts +451 -0
- package/components/deploymentRecorder.ts +7 -1
- package/components/mcp/adapters/fastify.ts +3 -0
- package/components/mcp/adapters/harperHttp.ts +3 -0
- package/components/mcp/audit.ts +21 -6
- package/components/mcp/customResourceRegistry.ts +17 -38
- package/components/mcp/listChanged.ts +37 -20
- package/components/mcp/quota.ts +145 -0
- package/components/mcp/rateLimit.ts +152 -14
- package/components/mcp/resources.ts +2 -4
- package/components/mcp/toolRegistry.ts +73 -4
- package/components/mcp/tools/application.ts +77 -8
- package/components/mcp/tools/operations.ts +106 -36
- package/components/mcp/tools/schemas/operationDescriptions.ts +4 -4
- package/components/mcp/tools/schemas/operations.ts +5 -0
- package/components/mcp/transport.ts +55 -4
- package/components/ollama/index.ts +6 -2
- package/components/openai/index.ts +9 -2
- package/components/operations.js +204 -10
- package/components/operationsValidation.js +173 -0
- package/components/secretOperations.ts +498 -0
- package/config/configUtils.ts +25 -1
- package/config-app.schema.json +81 -0
- package/config-root.schema.json +8 -1
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +4 -3
- package/dataLayer/readAuditLog.ts +14 -0
- package/dataLayer/schemaDescribe.ts +1 -0
- package/dist/agent/operations.d.ts +7 -4
- package/dist/agent/operations.js +13 -15
- package/dist/agent/operations.js.map +1 -1
- package/dist/components/Application.d.ts +25 -2
- package/dist/components/Application.js +166 -7
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +2 -0
- package/dist/components/ApplicationScope.js +3 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/Scope.d.ts +8 -0
- package/dist/components/Scope.js +11 -0
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/anthropic/index.d.ts +4 -1
- package/dist/components/anthropic/index.js +4 -2
- package/dist/components/anthropic/index.js.map +1 -1
- package/dist/components/bedrock/index.d.ts +4 -1
- package/dist/components/bedrock/index.js +22 -2
- package/dist/components/bedrock/index.js.map +1 -1
- package/dist/components/componentLoader.d.ts +2 -0
- package/dist/components/componentLoader.js +52 -10
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentSecrets.d.ts +63 -0
- package/dist/components/componentSecrets.js +405 -0
- package/dist/components/componentSecrets.js.map +1 -0
- package/dist/components/deploymentRecorder.d.ts +3 -0
- package/dist/components/deploymentRecorder.js +4 -3
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/mcp/adapters/fastify.d.ts +2 -0
- package/dist/components/mcp/adapters/fastify.js +1 -0
- package/dist/components/mcp/adapters/fastify.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.d.ts +2 -0
- package/dist/components/mcp/adapters/harperHttp.js +1 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/audit.d.ts +2 -2
- package/dist/components/mcp/audit.js +20 -5
- package/dist/components/mcp/audit.js.map +1 -1
- package/dist/components/mcp/customResourceRegistry.d.ts +5 -12
- package/dist/components/mcp/customResourceRegistry.js +15 -30
- package/dist/components/mcp/customResourceRegistry.js.map +1 -1
- package/dist/components/mcp/listChanged.d.ts +7 -2
- package/dist/components/mcp/listChanged.js +28 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/quota.d.ts +31 -0
- package/dist/components/mcp/quota.js +155 -0
- package/dist/components/mcp/quota.js.map +1 -0
- package/dist/components/mcp/rateLimit.d.ts +19 -3
- package/dist/components/mcp/rateLimit.js +129 -13
- package/dist/components/mcp/rateLimit.js.map +1 -1
- package/dist/components/mcp/resources.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +39 -1
- package/dist/components/mcp/toolRegistry.js +60 -5
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/application.d.ts +8 -1
- package/dist/components/mcp/tools/application.js +65 -5
- package/dist/components/mcp/tools/application.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +15 -3
- package/dist/components/mcp/tools/operations.js +99 -38
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +4 -4
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operations.js +4 -0
- package/dist/components/mcp/tools/schemas/operations.js.map +1 -1
- package/dist/components/mcp/transport.d.ts +6 -0
- package/dist/components/mcp/transport.js +47 -3
- package/dist/components/mcp/transport.js.map +1 -1
- package/dist/components/ollama/index.d.ts +4 -1
- package/dist/components/ollama/index.js +4 -2
- package/dist/components/ollama/index.js.map +1 -1
- package/dist/components/openai/index.d.ts +4 -1
- package/dist/components/openai/index.js +4 -2
- package/dist/components/openai/index.js.map +1 -1
- package/dist/components/operations.d.ts +39 -0
- package/dist/components/operations.js +194 -2
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.d.ts +38 -0
- package/dist/components/operationsValidation.js +155 -0
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/secretOperations.d.ts +113 -0
- package/dist/components/secretOperations.js +475 -0
- package/dist/components/secretOperations.js.map +1 -0
- package/dist/config/configUtils.d.ts +4 -1
- package/dist/config/configUtils.js +18 -1
- package/dist/config/configUtils.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +4 -3
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js.map +1 -1
- package/dist/dataLayer/readAuditLog.js +11 -0
- package/dist/dataLayer/readAuditLog.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +2 -0
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/globals.d.ts +1 -1
- package/dist/globals.js +1 -0
- package/dist/globals.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +38 -0
- package/dist/resources/DatabaseTransaction.d.ts +31 -3
- package/dist/resources/DatabaseTransaction.js +190 -29
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +1 -1
- package/dist/resources/LMDBTransaction.js +43 -5
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +51 -0
- package/dist/resources/PrimaryRocksDatabase.js +194 -0
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -0
- package/dist/resources/RecordEncoder.d.ts +1 -0
- package/dist/resources/RecordEncoder.js +81 -75
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +19 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.d.ts +0 -1
- package/dist/resources/Resource.js +103 -16
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +6 -2
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/Resources.js.map +1 -1
- package/dist/resources/Table.d.ts +6 -4
- package/dist/resources/Table.js +247 -24
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/read.js +25 -28
- package/dist/resources/analytics/read.js.map +1 -1
- package/dist/resources/blob.js +165 -22
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/crdt.d.ts +2 -0
- package/dist/resources/crdt.js +45 -36
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/databases.d.ts +1 -0
- package/dist/resources/databases.js +134 -7
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/graphql.js +8 -0
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +35 -5
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +239 -14
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/jsResource.js +6 -0
- package/dist/resources/jsResource.js.map +1 -1
- package/dist/resources/loadEnv.js +30 -3
- package/dist/resources/loadEnv.js.map +1 -1
- package/dist/resources/login.js +2 -2
- package/dist/resources/login.js.map +1 -1
- package/dist/resources/models/Models.js +14 -3
- package/dist/resources/models/Models.js.map +1 -1
- package/dist/resources/models/embedHook.js +10 -2
- package/dist/resources/models/embedHook.js.map +1 -1
- package/dist/resources/models/openaiStream.d.ts +56 -0
- package/dist/resources/models/openaiStream.js +94 -0
- package/dist/resources/models/openaiStream.js.map +1 -0
- package/dist/resources/replayLogs.js +4 -3
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/search.d.ts +1 -1
- package/dist/resources/search.js +104 -11
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/secretDecryptor.d.ts +54 -0
- package/dist/resources/secretDecryptor.js +131 -0
- package/dist/resources/secretDecryptor.js.map +1 -0
- package/dist/resources/tracked.js +15 -5
- package/dist/resources/tracked.js.map +1 -1
- package/dist/security/auth.js +71 -8
- package/dist/security/auth.js.map +1 -1
- package/dist/security/jsLoader.js +11 -0
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +9 -1
- package/dist/security/tokenAuthentication.js +31 -1
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +1 -1
- package/dist/server/DurableSubscriptionsSession.js +8 -4
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +28 -9
- package/dist/server/REST.js.map +1 -1
- package/dist/server/fastifyRoutes.js +11 -1
- package/dist/server/fastifyRoutes.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -3
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +40 -2
- package/dist/server/http.js +560 -77
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +64 -0
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +12 -0
- package/dist/server/liveSubscriptionAuth.js +140 -0
- package/dist/server/liveSubscriptionAuth.js.map +1 -0
- package/dist/server/loadRootComponents.js +2 -1
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/middlewareChain.d.ts +44 -1
- package/dist/server/middlewareChain.js +86 -10
- package/dist/server/middlewareChain.js.map +1 -1
- package/dist/server/mqtt.js +2 -1
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/operationsServer.js +1 -1
- package/dist/server/operationsServer.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +9 -0
- package/dist/server/serverHelpers/Headers.js +22 -0
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/JSONStream.js +3 -3
- package/dist/server/serverHelpers/JSONStream.js.map +1 -1
- package/dist/server/serverHelpers/Request.d.ts +52 -1
- package/dist/server/serverHelpers/Request.js +122 -1
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.js +3 -8
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/progressEmitter.d.ts +19 -0
- package/dist/server/serverHelpers/progressEmitter.js +52 -4
- package/dist/server/serverHelpers/progressEmitter.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +40 -0
- package/dist/server/serverHelpers/registeredOperations.js +286 -0
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +11 -5
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +1 -0
- package/dist/server/serverHelpers/serverUtilities.js +67 -5
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/uwsServer.d.ts +80 -0
- package/dist/server/serverHelpers/uwsServer.js +433 -0
- package/dist/server/serverHelpers/uwsServer.js.map +1 -0
- package/dist/server/static.d.ts +0 -15
- package/dist/server/static.js +161 -2
- package/dist/server/static.js.map +1 -1
- package/dist/server/status/index.d.ts +4 -1
- package/dist/server/status/index.js +64 -4
- package/dist/server/status/index.js.map +1 -1
- package/dist/server/threads/manageThreads.d.ts +10 -0
- package/dist/server/threads/manageThreads.js +107 -4
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.js +5 -1
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadServer.js +92 -14
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/upgrade/directives/5-2-0.d.ts +7 -0
- package/dist/upgrade/directives/5-2-0.js +108 -0
- package/dist/upgrade/directives/5-2-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/common_utils.js +5 -0
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/envFile.d.ts +41 -0
- package/dist/utility/envFile.js +221 -0
- package/dist/utility/envFile.js.map +1 -0
- package/dist/utility/globalSchema.d.ts +9 -0
- package/dist/utility/hdbTerms.d.ts +25 -0
- package/dist/utility/hdbTerms.js +31 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +21 -0
- package/dist/utility/logging/harper_logger.js +160 -8
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logRotator.js +29 -17
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/logging/readLog.d.ts +1 -1
- package/dist/utility/logging/readLog.js +305 -2
- package/dist/utility/logging/readLog.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +18 -0
- package/dist/utility/operationPermissions.js +35 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +17 -0
- package/dist/utility/operation_authorization.js +38 -0
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/secretEnvelope.d.ts +30 -0
- package/dist/utility/secretEnvelope.js +94 -0
- package/dist/utility/secretEnvelope.js.map +1 -0
- package/dist/utility/signalling.d.ts +7 -0
- package/dist/utility/signalling.js +16 -0
- package/dist/utility/signalling.js.map +1 -1
- package/dist/validation/configValidator.js +70 -24
- package/dist/validation/configValidator.js.map +1 -1
- package/index.ts +4 -0
- package/json/systemSchema.json +38 -0
- package/npm-shrinkwrap.json +15066 -9347
- package/package.json +13 -8
- package/resources/DESIGN.md +15 -12
- package/resources/DatabaseTransaction.ts +206 -37
- package/resources/LMDBTransaction.ts +43 -9
- package/resources/PrimaryRocksDatabase.ts +201 -0
- package/resources/RecordEncoder.ts +81 -69
- package/resources/RequestTarget.ts +21 -0
- package/resources/Resource.ts +109 -25
- package/resources/ResourceInterface.ts +6 -2
- package/resources/Resources.ts +1 -3
- package/resources/Table.ts +238 -29
- package/resources/analytics/read.ts +25 -30
- package/resources/blob.ts +159 -23
- package/resources/crdt.ts +37 -31
- package/resources/databases.ts +143 -6
- package/resources/graphql.ts +8 -0
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +251 -17
- package/resources/jsResource.ts +6 -0
- package/resources/loadEnv.ts +34 -3
- package/resources/login.ts +2 -2
- package/resources/models/Models.ts +19 -3
- package/resources/models/embedHook.ts +13 -2
- package/resources/models/openaiStream.ts +133 -0
- package/resources/replayLogs.ts +4 -3
- package/resources/search.ts +113 -11
- package/resources/secretDecryptor.ts +159 -0
- package/resources/tracked.ts +24 -17
- package/security/auth.ts +70 -10
- package/security/jsLoader.ts +12 -0
- package/security/tokenAuthentication.ts +42 -1
- package/server/DESIGN.md +10 -0
- package/server/DurableSubscriptionsSession.ts +8 -4
- package/server/REST.ts +26 -9
- package/server/fastifyRoutes.ts +10 -1
- package/server/graphqlQuerying.ts +4 -3
- package/server/http.ts +563 -77
- package/server/itc/serverHandlers.js +70 -0
- package/server/liveSubscriptionAuth.ts +152 -0
- package/server/loadRootComponents.js +2 -1
- package/server/middlewareChain.ts +107 -17
- package/server/mqtt.ts +2 -1
- package/server/operationsServer.ts +2 -2
- package/server/serverHelpers/Headers.ts +24 -0
- package/server/serverHelpers/JSONStream.ts +3 -3
- package/server/serverHelpers/Request.ts +127 -0
- package/server/serverHelpers/contentTypes.ts +3 -8
- package/server/serverHelpers/progressEmitter.ts +55 -5
- package/server/serverHelpers/registeredOperations.ts +259 -0
- package/server/serverHelpers/serverHandlers.js +11 -5
- package/server/serverHelpers/serverUtilities.ts +88 -5
- package/server/serverHelpers/uwsServer.ts +488 -0
- package/server/static.ts +180 -2
- package/server/status/index.ts +72 -4
- package/server/threads/manageThreads.js +102 -4
- package/server/threads/socketRouter.ts +5 -1
- package/server/threads/threadServer.js +93 -10
- package/studio/web/assets/{Chat-SdD1EZca.js → Chat-Cv_2paZE.js} +2 -2
- package/studio/web/assets/{Chat-SdD1EZca.js.map → Chat-Cv_2paZE.js.map} +1 -1
- package/studio/web/assets/{FloatingChat-GMq6tHK9.js → FloatingChat-CPyPIcVR.js} +4 -4
- package/studio/web/assets/{FloatingChat-GMq6tHK9.js.map → FloatingChat-CPyPIcVR.js.map} +1 -1
- package/studio/web/assets/{applications-B5hhv7uA.js → applications-C0jT2vdZ.js} +2 -2
- package/studio/web/assets/{applications-B5hhv7uA.js.map → applications-C0jT2vdZ.js.map} +1 -1
- package/studio/web/assets/{index-VFcpNWgq.js → index-D_GKOkhn.js} +6 -6
- package/studio/web/assets/index-D_GKOkhn.js.map +1 -0
- package/studio/web/assets/{index.lazy-DgcJojIA.js → index.lazy-CAmCIA65.js} +4 -4
- package/studio/web/assets/{index.lazy-DgcJojIA.js.map → index.lazy-CAmCIA65.js.map} +1 -1
- package/studio/web/assets/{profile-BqN-8vzm.js → profile-CaF-4aZe.js} +2 -2
- package/studio/web/assets/{profile-BqN-8vzm.js.map → profile-CaF-4aZe.js.map} +1 -1
- package/studio/web/assets/{setComponentFile-CHFvFq9u.js → setComponentFile-DLW1DHCt.js} +2 -2
- package/studio/web/assets/{setComponentFile-CHFvFq9u.js.map → setComponentFile-DLW1DHCt.js.map} +1 -1
- package/studio/web/assets/{setup-CQ3vWJSr.js → setup-TGCErIhq.js} +2 -2
- package/studio/web/assets/{setup-CQ3vWJSr.js.map → setup-TGCErIhq.js.map} +1 -1
- package/studio/web/assets/{status-BsW5fjuh.js → status-DG0Maoao.js} +2 -2
- package/studio/web/assets/{status-BsW5fjuh.js.map → status-DG0Maoao.js.map} +1 -1
- package/studio/web/assets/{swagger-ui-react-DITdeB9-.js → swagger-ui-react-8T4SgQ1m.js} +2 -2
- package/studio/web/assets/{swagger-ui-react-DITdeB9-.js.map → swagger-ui-react-8T4SgQ1m.js.map} +1 -1
- package/studio/web/assets/{tsMode-Dyph-OUs.js → tsMode-DJ6Sl24Q.js} +2 -2
- package/studio/web/assets/{tsMode-Dyph-OUs.js.map → tsMode-DJ6Sl24Q.js.map} +1 -1
- package/studio/web/assets/{useEntityRestURL-D0QTUqex.js → useEntityRestURL-JO2mfWTQ.js} +2 -2
- package/studio/web/assets/{useEntityRestURL-D0QTUqex.js.map → useEntityRestURL-JO2mfWTQ.js.map} +1 -1
- package/studio/web/index.html +1 -1
- package/upgrade/directives/5-2-0.ts +82 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/common_utils.ts +5 -0
- package/utility/envFile.ts +218 -0
- package/utility/hdbTerms.ts +31 -0
- package/utility/logging/harper_logger.ts +156 -8
- package/utility/logging/logRotator.ts +24 -14
- package/utility/logging/readLog.ts +323 -2
- package/utility/operationPermissions.ts +35 -1
- package/utility/operation_authorization.ts +74 -1
- package/utility/secretEnvelope.ts +113 -0
- package/utility/signalling.ts +15 -0
- package/validation/configValidator.ts +78 -25
- package/studio/web/assets/index-VFcpNWgq.js.map +0 -1
|
@@ -138,48 +138,45 @@ async function* coalesceResults(results, window) {
|
|
|
138
138
|
}
|
|
139
139
|
async function get(metric, opts) {
|
|
140
140
|
const { getAttributes, startTime, endTime, additionalConditions, log: logName } = opts ?? {};
|
|
141
|
-
|
|
142
|
-
// have one, lead with it and pin execution order (below) so it drives the scan. `Table.search`
|
|
143
|
-
// otherwise reorders conditions by estimated cost, but the planner estimates a primary-key `between`
|
|
144
|
-
// as a flat 10% of the table (search.ts BETWEEN_ESTIMATE) rather than from the actual range width —
|
|
145
|
-
// so on a large `hdb_analytics` table a narrow window looks costlier than `metric equals`, iteration
|
|
146
|
-
// is driven off the metric index, and the metric's entire history is decoded instead of the window
|
|
147
|
-
// (#1796). The proper long-term fix is a range-width-aware cost estimate in search.ts.
|
|
148
|
-
const boundedWindow = startTime != null && endTime != null;
|
|
149
|
-
const conditions = [];
|
|
150
|
-
if (boundedWindow) {
|
|
151
|
-
conditions.push({ attribute: 'id', comparator: 'between', value: [startTime, endTime] });
|
|
152
|
-
}
|
|
153
|
-
conditions.push({ attribute: 'metric', comparator: 'equals', value: metric });
|
|
141
|
+
const conditions = [{ attribute: 'metric', comparator: 'equals', value: metric }];
|
|
154
142
|
if (logName !== undefined) {
|
|
155
143
|
conditions.push({ attribute: 'log', comparator: 'equals', value: logName });
|
|
156
144
|
}
|
|
157
145
|
if (additionalConditions) {
|
|
158
146
|
conditions.push(...additionalConditions.map(conformCondition));
|
|
159
147
|
}
|
|
160
|
-
// An open-ended range (only start or only end) can span the whole table, so we append it and leave
|
|
161
|
-
// ordering to the planner rather than forcing an unbounded range to drive — which could scan the
|
|
162
|
-
// full table for a selective metric.
|
|
163
|
-
if (!boundedWindow) {
|
|
164
|
-
if (startTime != null) {
|
|
165
|
-
conditions.push({ attribute: 'id', comparator: 'greater_than_equal', value: startTime });
|
|
166
|
-
}
|
|
167
|
-
if (endTime != null) {
|
|
168
|
-
conditions.push({ attribute: 'id', comparator: 'less_than', value: endTime });
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
148
|
const select = getAttributes ?? [];
|
|
172
|
-
// ensure we're always selecting id
|
|
149
|
+
// ensure we're always selecting id
|
|
173
150
|
if (!isSelected(select, 'id')) {
|
|
174
151
|
select.push('id');
|
|
175
152
|
}
|
|
153
|
+
if (startTime && endTime) {
|
|
154
|
+
conditions.push({
|
|
155
|
+
attribute: 'id',
|
|
156
|
+
comparator: 'between',
|
|
157
|
+
value: [startTime, endTime],
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
else {
|
|
161
|
+
if (startTime) {
|
|
162
|
+
conditions.push({
|
|
163
|
+
attribute: 'id',
|
|
164
|
+
comparator: 'greater_than_equal',
|
|
165
|
+
value: startTime,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
if (endTime) {
|
|
169
|
+
conditions.push({
|
|
170
|
+
attribute: 'id',
|
|
171
|
+
comparator: 'less_than',
|
|
172
|
+
value: endTime,
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
176
|
// `snapshot: false` lets these (potentially long-running) analytics scans read against the
|
|
177
177
|
// latest committed data without holding a consistent read snapshot open, so they stay easier
|
|
178
178
|
// on the rest of the system (a pinned snapshot would block compaction for the scan's duration).
|
|
179
179
|
const request = { conditions, allowConditionsOnDynamicAttributes: true, snapshot: false };
|
|
180
|
-
if (boundedWindow) {
|
|
181
|
-
request.enforceExecutionOrder = true;
|
|
182
|
-
}
|
|
183
180
|
if (select.length > 0) {
|
|
184
181
|
request['select'] = select;
|
|
185
182
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read.js","sourceRoot":"","sources":["../../../resources/analytics/read.ts"],"names":[],"mappings":";;;;;AAoDA,sBAuDC;AAqFD,
|
|
1
|
+
{"version":3,"file":"read.js","sourceRoot":"","sources":["../../../resources/analytics/read.ts"],"names":[],"mappings":";;;;;AAoDA,sBAuDC;AAqFD,kBAsEC;AAWD,sCAEC;AAED,kCA2CC;AAeD,4CAEC;AAED,wCAwBC;AA1WD,8FAAkE;AAClE,MAAM,EAAE,YAAY,EAAE,GAAG,0BAAY,CAAC;AACtC,iDAAyE;AAEzE,+CAA+D;AAC/D,2DAA0D;AAC1D,2FAAgF;AAChF,kFAA8E;AAC9E,kEAA6E;AAC7E,0DAA2D;AAE3D,6DAA6D;AAC7D,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1D,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;AACzC,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC;AAE/B,KAAK,UAAU,cAAc,CAAC,MAAc;IAC3C,MAAM,MAAM,GAAG,MAAM,IAAA,wCAAyB,GAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7D,IAAI,MAAM,EAAE,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC;IAC7C,IAAI,MAAM,KAAK,IAAA,2BAAY,EAAC,MAAM,CAAC,QAAQ,CAAC;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC;IACrE,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,WAAqB,EAAE,IAAY;IACtD,OAAO,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAC/D,CAAC;AAoBD;;;;GAIG;AACI,KAAK,UAAU,KAAK,CAAC,GAAwB;IACnD,GAAG,CAAC,KAAK,EAAE,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;IAC3C,MAAM,eAAe,GAAG,IAAA,4CAAoB,EAAC,GAAG,CAAC,CAAC;IAClD,IAAI,eAAe,EAAE,CAAC;QACrB,MAAM,IAAA,4BAAc,EACnB,eAAe,EACf,eAAe,CAAC,OAAO,EACvB,uBAAS,CAAC,iBAAiB,CAAC,WAAW,EACvC,SAAS,EACT,SAAS,EACT,IAAI,CACJ,CAAC;IACH,CAAC;IACD,4FAA4F;IAC5F,iGAAiG;IACjG,IAAI,GAAG,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,oBAAM,CAAC,oBAAoB,EAAE,CAAC;QACzE,MAAM,IAAA,4BAAc,EACnB,IAAI,KAAK,CAAC,+BAA+B,CAAC,EAC1C,+CAA+C,oBAAM,CAAC,oBAAoB,UAAU,EACpF,uBAAS,CAAC,iBAAiB,CAAC,WAAW,EACvC,SAAS,EACT,SAAS,EACT,IAAI,CACJ,CAAC;IACH,CAAC;IACD,4EAA4E;IAC5E,uEAAuE;IACvE,qCAAqC;IACrC,qFAAqF;IACrF,kFAAkF;IAClF,+EAA+E;IAC/E,8EAA8E;IAC9E,qFAAqF;IACrF,MAAM,uBAAuB,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,KAAK,KAAK,CAAC;IACnF,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,IAAI,CAAC,uBAAuB,IAAI,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5G,+EAA+E;IAC/E,kFAAkF;IAClF,oFAAoF;IACpF,IAAI,aAAa,GAAG,GAAG,CAAC,cAAc,CAAC;IACvC,IAAI,KAAK,IAAI,aAAa,EAAE,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACvE,aAAa,GAAG,CAAC,GAAG,aAAa,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE;QAC1C,aAAa;QACb,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,YAAY,EAAE,GAAG,CAAC,aAAa;QAC/B,oBAAoB,EAAE,GAAG,CAAC,UAAU;QACpC,GAAG,EAAE,GAAG,CAAC,GAAG;KACZ,CAAC,CAAC;IAEH,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAChC,OAAO,uBAAuB,CAAC,YAAY,EAAE,EAAE,GAAG,GAAG,EAAE,cAAc,EAAE,aAAa,EAAE,EAAE,KAAK,CAAC,CAAC;AAChG,CAAC;AAED;;;;;GAKG;AACH,KAAK,SAAS,CAAC,CAAC,uBAAuB,CACtC,YAAsD,EACtD,GAAwB,EACxB,KAAyB;IAEzB,MAAM,QAAQ,GAAG,IAAA,6BAAe,GAAE,CAAC;IACnC,MAAM,OAAO,GAAG,EAAE,GAAG,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IAC9C,6EAA6E;IAC7E,8EAA8E;IAC9E,wCAAwC;IACxC,MAAM,mBAAmB,GAAG,MAAM,CAAC,WAAW,CAAC,mBAGA,CAAC;IAEhD,MAAM,WAAW,GAAG,KAAK;SACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;SACxC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACb,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI;IACtC,8EAA8E;IAC9E,2EAA2E;IAC3E,wBAAwB;IACxB,CAAC,QAAQ,EAAY,EAAE,CACtB,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAC9F,CAAC,KAAY,EAAY,EAAE;QAC1B,MAAM,CAAC,IAAI,CAAC,sCAAsC,IAAI,CAAC,IAAI,gCAAgC,EAAE,KAAK,CAAC,CAAC;QACpG,OAAO,EAAE,CAAC;IACX,CAAC,CACD,CACD,CAAC;IAEH,KAAK,CAAC,CAAC,YAAY,CAAC;IACpB,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,KAAK,CAAC,CAAC,MAAM,UAAU,CAAC;IACzB,CAAC;AACF,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAoB;IAC7C,IAAI,YAAY,IAAI,SAAS,EAAE,CAAC;QAC/B,OAAO;YACN,GAAG,SAAS;YACZ,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,gBAAgB,CAAC;SACtD,CAAC;IACH,CAAC;IACD,OAAO;QACN,SAAS,EAAE,SAAS,CAAC,gBAAgB,IAAI,SAAS,CAAC,SAAS;QAC5D,UAAU,EAAE,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,UAAU;QACzD,KAAK,EAAE,SAAS,CAAC,YAAY,IAAI,SAAS,CAAC,KAAK;KAChD,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAAiB,EAAE,MAAc;IAChE,IAAI,UAAe,CAAC;IACpB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QACpC,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,UAAU,GAAG,EAAE,CAAC;QACjB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,MAAM,EAAE,CAAC;YACpB,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;QACrC,CAAC;aAAM,CAAC;YACP,MAAM,MAAM,CAAC;YACb,UAAU,GAAG,EAAE,CAAC;QACjB,CAAC;IACF,CAAC;AACF,CAAC;AAWM,KAAK,UAAU,GAAG,CAAC,MAAc,EAAE,IAAuB;IAChE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,EAAE,oBAAoB,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7F,MAAM,UAAU,GAAe,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9F,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC3B,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,oBAAoB,EAAE,CAAC;QAC1B,UAAU,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAChE,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,IAAI,EAAE,CAAC;IAEnC,mCAAmC;IACnC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;QAC1B,UAAU,CAAC,IAAI,CAAC;YACf,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,SAAS;YACrB,KAAK,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;SAC3B,CAAC,CAAC;IACJ,CAAC;SAAM,CAAC;QACP,IAAI,SAAS,EAAE,CAAC;YACf,UAAU,CAAC,IAAI,CAAC;gBACf,SAAS,EAAE,IAAI;gBACf,UAAU,EAAE,oBAAoB;gBAChC,KAAK,EAAE,SAAS;aAChB,CAAC,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACb,UAAU,CAAC,IAAI,CAAC;gBACf,SAAS,EAAE,IAAI;gBACf,UAAU,EAAE,WAAW;gBACvB,KAAK,EAAE,OAAO;aACd,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,2FAA2F;IAC3F,6FAA6F;IAC7F,gGAAgG;IAChG,MAAM,OAAO,GAAQ,EAAE,UAAU,EAAE,kCAAkC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAC/F,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;IAC5B,CAAC;IACD,GAAG,CAAC,KAAK,EAAE,CAAC,6CAA6C,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACpF,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE3E,IAAI,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,MAAc,EAAE,EAAE;QACxD,yEAAyE;QACzE,yCAAyC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YAChC,GAAG,CAAC,KAAK,EAAE,CAAC,6CAA6C,MAAM,EAAE,CAAC,CAAC;YACnE,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,GAAG,QAAQ,IAAI,MAAM,CAAC;QACrC,CAAC;QACD,GAAG,CAAC,KAAK,EAAE,CAAC,uBAAuB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7D,OAAO,MAAM,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,EAAE,YAAY,EAAE,CAAC;QACxB,sEAAsE;QACtE,MAAM,MAAM,GAAG,IAAA,2BAAM,EAAC,2BAAa,CAAC,yBAAyB,CAAC,GAAG,IAAI,CAAC;QACtE,OAAO,GAAG,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,OAAO,CAAC;AAChB,CAAC;AAWD,SAAgB,aAAa,CAAC,GAAuB;IACpD,OAAO,WAAW,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAC;AACjE,CAAC;AAEM,KAAK,UAAU,WAAW,CAChC,cAA4B,CAAC,SAAS,CAAC,EACvC,eAAuB,yBAAyB;IAEhD,IAAI,OAAO,GAAa,EAAE,CAAC;IAE3B,MAAM,QAAQ,GAAwB,MAAM,CAAC,MAAM,CAAC,oBAAM,CAAC,CAAC;IAE5D,IAAI,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,OAAO,GAAG,QAAQ,CAAC;IACpB,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;QACjD,MAAM,UAAU,GAAe;YAC9B;gBACC,SAAS,EAAE,IAAI;gBACf,UAAU,EAAE,cAAc;gBAC1B,KAAK,EAAE,cAAc;aACrB;SACD,CAAC;QACF,MAAM,gBAAgB,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3C,OAAO;gBACN,SAAS,EAAE,QAAQ;gBACnB,UAAU,EAAE,WAAW;gBACvB,KAAK,EAAE,CAAC;aACK,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,UAAU,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;QACrC,MAAM,mBAAmB,GAAQ;YAChC,MAAM,EAAE,CAAC,QAAQ,CAAC;YAClB,UAAU,EAAE,UAAU;SACtB,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;QACvF,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;YAC1C,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,OAAO,CAAC;AAChB,CAAC;AAeD,SAAgB,gBAAgB,CAAC,GAA0B;IAC1D,OAAO,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,MAAc;IAClD,MAAM,eAAe,GAAQ;QAC5B,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAC1E,IAAI,EAAE;YACL,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,IAAI;SAChB;QACD,KAAK,EAAE,CAAC;KACR,CAAC;IACF,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACvE,4DAA4D;IAC5D,MAAM,UAAU,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IACtD,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,IAAI,GAAG;YACZ,UAAU;SACV,CAAC;QACF,GAAG,CAAC,KAAK,EAAE,CAAC,yBAAyB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7D,OAAO,IAAI,CAAC;IACb,CAAC;IACD,qCAAqC;IACrC,OAAO,EAAE,CAAC;AACX,CAAC"}
|
package/dist/resources/blob.js
CHANGED
|
@@ -286,11 +286,17 @@ class FileBackedBlob extends exports.Blob {
|
|
|
286
286
|
rawBytes = Buffer.alloc(0);
|
|
287
287
|
fileMissing = true;
|
|
288
288
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
289
|
+
// The file may still be mid-write. Wait for the writer's `:blob` lock to release, then re-read.
|
|
290
|
+
// `incomplete` is the caller's "definitely not done yet" signal: for an uncompressed blob the
|
|
291
|
+
// on-disk body length must reach the (uncompressed) header size. For a compressed blob the body
|
|
292
|
+
// is shorter than the uncompressed size AND the header size is finalized up front when the size
|
|
293
|
+
// is known, so neither is a reliable completeness signal — the writer's lock is. The caller
|
|
294
|
+
// passes `mustVerifyViaLock` for compressed blobs so we probe the lock even when the bytes look
|
|
295
|
+
// plausible, and only treat the file as complete once the writer is confirmed done.
|
|
296
|
+
function waitForCompletion(incomplete, mustVerifyViaLock = false) {
|
|
297
|
+
const store = storageInfo.store;
|
|
298
|
+
const lockKey = storageInfo.fileId + ':blob';
|
|
299
|
+
if (incomplete) {
|
|
294
300
|
if (writeFinished) {
|
|
295
301
|
// the writer released its lock but the content is still short: the file is cleanly gone
|
|
296
302
|
// (404) or confidently incomplete/corrupt (500) — not merely mid-write (#1423/#1424).
|
|
@@ -327,25 +333,79 @@ class FileBackedBlob extends exports.Blob {
|
|
|
327
333
|
}
|
|
328
334
|
});
|
|
329
335
|
}
|
|
336
|
+
else if (mustVerifyViaLock && !writeFinished) {
|
|
337
|
+
// Bytes look complete but we can't prove it for a compressed body; probe whether the writer
|
|
338
|
+
// still holds the lock. If we acquire it, the write has finished, but the bytes we read may
|
|
339
|
+
// predate the final flush, so re-read once to get the complete body. Otherwise fall through
|
|
340
|
+
// to wait for the lock to release.
|
|
341
|
+
const acquiredImmediately = store.tryLock(lockKey);
|
|
342
|
+
if (acquiredImmediately) {
|
|
343
|
+
writeFinished = true;
|
|
344
|
+
store.unlock(lockKey);
|
|
345
|
+
return readContents(); // writer done — re-read to ensure we have the final bytes
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
else {
|
|
349
|
+
return undefined; // confirmed complete (uncompressed exact-size match)
|
|
350
|
+
}
|
|
351
|
+
return new Promise((resolve, reject) => {
|
|
352
|
+
let settled = false;
|
|
353
|
+
const timer = setTimeout(() => {
|
|
354
|
+
if (settled)
|
|
355
|
+
return;
|
|
356
|
+
settled = true;
|
|
357
|
+
reject(new BlobReadError(`Blob ${storageInfo.fileId} is unavailable; the in-progress write did not complete in time`, BLOB_UNAVAILABLE_STATUS));
|
|
358
|
+
}, getBlobReadTimeout());
|
|
359
|
+
timer.unref();
|
|
360
|
+
const callback = () => {
|
|
361
|
+
if (settled)
|
|
362
|
+
return;
|
|
363
|
+
settled = true;
|
|
364
|
+
clearTimeout(timer);
|
|
365
|
+
writeFinished = true;
|
|
366
|
+
resolve(readContents());
|
|
367
|
+
};
|
|
368
|
+
const lockAcquired = store.tryLock(lockKey, callback);
|
|
369
|
+
if (lockAcquired) {
|
|
370
|
+
settled = true;
|
|
371
|
+
clearTimeout(timer);
|
|
372
|
+
writeFinished = true;
|
|
373
|
+
store.unlock(lockKey);
|
|
374
|
+
resolve(readContents());
|
|
375
|
+
}
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
function sliceContent(bytes) {
|
|
330
379
|
if (end != undefined || start != undefined) {
|
|
331
|
-
|
|
380
|
+
bytes = bytes.subarray(start ?? 0, end ?? bytes.length);
|
|
332
381
|
}
|
|
333
|
-
return
|
|
382
|
+
return bytes;
|
|
334
383
|
}
|
|
335
384
|
// Only sniff the storage-type byte and take the decompress branch when a full header is present.
|
|
336
385
|
// A file shorter than the header (truncated/corrupted, #1424) would otherwise be mis-read as a
|
|
337
386
|
// deflate body and yield garbage; instead it falls through to checkCompletion as incomplete.
|
|
338
387
|
if (rawBytes.length >= HEADER_SIZE && rawBytes[1] === DEFLATE_TYPE) {
|
|
388
|
+
// An in-flight compressed file still carrying the UNKNOWN_SIZE placeholder is definitely
|
|
389
|
+
// incomplete; otherwise the size is finalized but the compressed body may still be streaming,
|
|
390
|
+
// so verify via the write lock before inflating (inflating a partial deflate stream errors).
|
|
391
|
+
const pending = waitForCompletion(size === UNKNOWN_SIZE, true);
|
|
392
|
+
if (pending)
|
|
393
|
+
return pending;
|
|
339
394
|
return new Promise((resolve, reject) => {
|
|
340
|
-
|
|
395
|
+
// start/end index into the UNCOMPRESSED content, so inflate first, then slice
|
|
396
|
+
(0, node_zlib_1.inflate)(rawBytes.subarray(HEADER_SIZE), (error, result) => {
|
|
341
397
|
if (error)
|
|
342
398
|
reject(error);
|
|
343
399
|
else
|
|
344
|
-
resolve(
|
|
400
|
+
resolve(sliceContent(result));
|
|
345
401
|
});
|
|
346
402
|
});
|
|
347
403
|
}
|
|
348
|
-
|
|
404
|
+
const body = rawBytes.subarray(HEADER_SIZE);
|
|
405
|
+
const pending = waitForCompletion(size > body.length);
|
|
406
|
+
if (pending)
|
|
407
|
+
return pending;
|
|
408
|
+
return sliceContent(body);
|
|
349
409
|
};
|
|
350
410
|
return readContents();
|
|
351
411
|
}
|
|
@@ -376,7 +436,24 @@ class FileBackedBlob extends exports.Blob {
|
|
|
376
436
|
let totalContentRead = 0;
|
|
377
437
|
let watcher;
|
|
378
438
|
let timer;
|
|
439
|
+
// The start() open-retry timer lives in a different scope/phase than pull()'s `timer`; track it
|
|
440
|
+
// separately so a cancel() during the file-creation wait clears it instead of leaking an fd (#1457).
|
|
441
|
+
let openTimer;
|
|
442
|
+
// Stream-wide cancellation gate. cancel() can null `fd` but cannot reach pull()'s per-pull
|
|
443
|
+
// `settled` flag, so this is the cross-phase signal that stops start()/pull() from opening or
|
|
444
|
+
// reading once the consumer has cancelled (e.g. an aborted ranged response) (#1457).
|
|
445
|
+
let cancelled = false;
|
|
379
446
|
let isBeingWritten;
|
|
447
|
+
// Close the descriptor at most once. Without nulling `fd`, a later close()/cancel() (e.g. a
|
|
448
|
+
// GC-driven cancel after a successful read) would close a descriptor the OS may have reassigned to
|
|
449
|
+
// an unrelated file/socket; and a cancel() before start()'s open() resolves would call
|
|
450
|
+
// close(undefined) and throw. Guarding on `fd != null` fixes both (#1457).
|
|
451
|
+
const closeFd = () => {
|
|
452
|
+
if (fd != null) {
|
|
453
|
+
(0, node_fs_1.close)(fd);
|
|
454
|
+
fd = null;
|
|
455
|
+
}
|
|
456
|
+
};
|
|
380
457
|
let previouslyFinishedWriting = false;
|
|
381
458
|
const blob = this;
|
|
382
459
|
// Authoritative (uncompressed) content length from the record descriptor, captured before the
|
|
@@ -393,6 +470,8 @@ class FileBackedBlob extends exports.Blob {
|
|
|
393
470
|
// lookup) is computed lazily on the first miss, so a healthy read never pays for it.
|
|
394
471
|
let deadline = 0;
|
|
395
472
|
const openFile = (resolve, reject) => {
|
|
473
|
+
if (cancelled)
|
|
474
|
+
return resolve(undefined); // consumer cancelled before/while opening; stop retrying
|
|
396
475
|
(0, node_fs_1.open)(filePath, 'r', (error, openedFd) => {
|
|
397
476
|
if (error) {
|
|
398
477
|
if (error.code === 'ENOENT' && isBeingWritten !== false) {
|
|
@@ -401,10 +480,11 @@ class FileBackedBlob extends exports.Blob {
|
|
|
401
480
|
if (Date.now() < deadline) {
|
|
402
481
|
logger_ts_1.logger.debug?.('File does not exist yet, waiting for it to be created', filePath);
|
|
403
482
|
// the file doesn't exist, so we need to wait for it to be created
|
|
404
|
-
|
|
483
|
+
openTimer = setTimeout(() => {
|
|
405
484
|
checkIfIsBeingWritten();
|
|
406
485
|
openFile(resolve, reject);
|
|
407
486
|
}, 20).unref();
|
|
487
|
+
return openTimer;
|
|
408
488
|
}
|
|
409
489
|
}
|
|
410
490
|
// ENOENT: 503 if a local writer holds the lock or a replication receive is in
|
|
@@ -421,6 +501,10 @@ class FileBackedBlob extends exports.Blob {
|
|
|
421
501
|
}
|
|
422
502
|
else {
|
|
423
503
|
fd = openedFd;
|
|
504
|
+
// the consumer cancelled while open() was in flight: pull()/cancel() will not run
|
|
505
|
+
// again, so close the descriptor we just acquired rather than leaking it (#1457)
|
|
506
|
+
if (cancelled)
|
|
507
|
+
closeFd();
|
|
424
508
|
resolve(openedFd);
|
|
425
509
|
}
|
|
426
510
|
});
|
|
@@ -435,9 +519,20 @@ class FileBackedBlob extends exports.Blob {
|
|
|
435
519
|
// it; bounds the case where the header reports a known size but the bytes never arrive (a
|
|
436
520
|
// present-but-truncated blob whose writer lock stays held) (#1454).
|
|
437
521
|
let incompleteDeadline = 0;
|
|
522
|
+
// Settle this pull exactly once. A slow async read that completes with an error after the
|
|
523
|
+
// watcher-timeout path already called onError() (and closed the fd) would otherwise run
|
|
524
|
+
// onError twice — double close + duplicated #onError callbacks (#1457).
|
|
525
|
+
let settled = false;
|
|
438
526
|
return new Promise(function readMore(resolve, reject) {
|
|
527
|
+
// A queued timer/watcher callback can re-enter readMore after the stream was cancelled (fd
|
|
528
|
+
// is gone); stop here so we never issue read() on a closed/nulled descriptor (#1457).
|
|
529
|
+
if (cancelled || fd == null)
|
|
530
|
+
return resolve();
|
|
439
531
|
function onError(error) {
|
|
440
|
-
|
|
532
|
+
if (settled)
|
|
533
|
+
return;
|
|
534
|
+
settled = true;
|
|
535
|
+
closeFd();
|
|
441
536
|
clearTimeout(timer);
|
|
442
537
|
if (watcher)
|
|
443
538
|
watcher.close();
|
|
@@ -447,7 +542,12 @@ class FileBackedBlob extends exports.Blob {
|
|
|
447
542
|
// allocate a buffer for reading. Note that we could do a stat to get the size, but that is a little more complicated, and might be a little extra overhead
|
|
448
543
|
const buffer = Buffer.allocUnsafe(0x40000);
|
|
449
544
|
(0, node_fs_1.read)(fd, buffer, 0, buffer.length, position, (error, bytesRead, buffer) => {
|
|
450
|
-
//
|
|
545
|
+
// A late read completion after the pull already settled (via onError or terminal
|
|
546
|
+
// close) or after the stream was cancelled (fd nulled, EBADF/partial read) must not
|
|
547
|
+
// re-enter the state machine — otherwise a recursive read() would hit a null fd and
|
|
548
|
+
// throw synchronously, or onError() would reject an already-cancelled stream (#1457).
|
|
549
|
+
if (settled || cancelled)
|
|
550
|
+
return resolve();
|
|
451
551
|
totalContentRead += bytesRead;
|
|
452
552
|
if (error)
|
|
453
553
|
return onError(error);
|
|
@@ -478,6 +578,36 @@ class FileBackedBlob extends exports.Blob {
|
|
|
478
578
|
// half-replicated blob whose source stream aborted; the bytes are still expected — retry (harper-pro#481)
|
|
479
579
|
return onError(new BlobReadError('Blob pending replication for ' + filePath, BLOB_UNAVAILABLE_STATUS));
|
|
480
580
|
}
|
|
581
|
+
if (buffer[1] === DEFLATE_TYPE) {
|
|
582
|
+
// We can't seek/slice a deflate stream by uncompressed offset, so hand off to the
|
|
583
|
+
// buffered inflate path (bytes() inflates then slices) and emit it as one chunk.
|
|
584
|
+
// Safe by construction: the read loop never streams the raw compressed body.
|
|
585
|
+
return blob.bytes().then((bytes) => {
|
|
586
|
+
// bytes() resolves asynchronously; the consumer may have cancelled meanwhile.
|
|
587
|
+
// Settle exactly once and route the close through closeFd() so we never touch a
|
|
588
|
+
// reassigned/nulled descriptor (#1457).
|
|
589
|
+
if (settled || cancelled)
|
|
590
|
+
return resolve();
|
|
591
|
+
settled = true;
|
|
592
|
+
closeFd();
|
|
593
|
+
if (bytes.length > 0) {
|
|
594
|
+
try {
|
|
595
|
+
controller.enqueue(bytes);
|
|
596
|
+
}
|
|
597
|
+
catch (error) {
|
|
598
|
+
logger_ts_1.logger.debug?.('Error enqueuing chunk', error);
|
|
599
|
+
return resolve();
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
try {
|
|
603
|
+
controller.close();
|
|
604
|
+
}
|
|
605
|
+
catch {
|
|
606
|
+
// controller may already be closed
|
|
607
|
+
}
|
|
608
|
+
resolve();
|
|
609
|
+
}, (error) => onError(error));
|
|
610
|
+
}
|
|
481
611
|
size = Number(headerValue & 0xffffffffffffn);
|
|
482
612
|
if (size < UNKNOWN_SIZE) {
|
|
483
613
|
if (isFullRead && descriptorSize != null && descriptorSize < UNKNOWN_SIZE && size !== descriptorSize) {
|
|
@@ -500,6 +630,8 @@ class FileBackedBlob extends exports.Blob {
|
|
|
500
630
|
else if (bytesRead === 0) {
|
|
501
631
|
const buffer = Buffer.allocUnsafe(8);
|
|
502
632
|
return (0, node_fs_1.read)(fd, buffer, 0, HEADER_SIZE, 0, (error) => {
|
|
633
|
+
if (settled || cancelled)
|
|
634
|
+
return resolve();
|
|
503
635
|
if (error)
|
|
504
636
|
return onError(error);
|
|
505
637
|
size = Number(new DataView(buffer.buffer, buffer.byteOffset, 8).getBigUint64(0) & 0xffffffffffffn);
|
|
@@ -591,7 +723,8 @@ class FileBackedBlob extends exports.Blob {
|
|
|
591
723
|
}
|
|
592
724
|
return;
|
|
593
725
|
}
|
|
594
|
-
|
|
726
|
+
settled = true;
|
|
727
|
+
closeFd();
|
|
595
728
|
controller.close();
|
|
596
729
|
resolve();
|
|
597
730
|
});
|
|
@@ -600,20 +733,27 @@ class FileBackedBlob extends exports.Blob {
|
|
|
600
733
|
buffer = buffer.subarray(0, bytesRead);
|
|
601
734
|
}
|
|
602
735
|
if (start !== undefined || end !== undefined) {
|
|
736
|
+
// content offset of the first byte currently in `buffer`. The 8-byte header is only
|
|
737
|
+
// present in — and stripped from — the first read at file position 0; every later read
|
|
738
|
+
// starts HEADER_SIZE into the file relative to its content offset.
|
|
739
|
+
const contentStart = position === 0 ? 0 : position - HEADER_SIZE;
|
|
603
740
|
if (start && totalContentRead < start) {
|
|
604
|
-
// we are before the start of the slice
|
|
605
|
-
|
|
741
|
+
// we are before the start of the slice. Seek straight to it (HEADER_SIZE + start)
|
|
742
|
+
// instead of reading and discarding a 256KB buffer per chunk from byte 0. The header
|
|
743
|
+
// has already been validated and the size detected on the first read (#1457).
|
|
744
|
+
position = HEADER_SIZE + start;
|
|
745
|
+
totalContentRead = start;
|
|
606
746
|
return readMore(resolve, reject);
|
|
607
747
|
}
|
|
608
748
|
if (end && totalContentRead >= end) {
|
|
609
749
|
// we are past or reached the end of the slice, so we have reached the end, indicate
|
|
610
750
|
if (totalContentRead > end)
|
|
611
|
-
buffer = buffer.subarray(0, end -
|
|
751
|
+
buffer = buffer.subarray(0, end - contentStart);
|
|
612
752
|
totalContentRead = size = end;
|
|
613
753
|
}
|
|
614
|
-
if (start && start >
|
|
615
|
-
// we need to skip ahead to the start of the slice
|
|
616
|
-
buffer = buffer.subarray(start -
|
|
754
|
+
if (start && start > contentStart) {
|
|
755
|
+
// we need to skip ahead to the start of the slice within this chunk
|
|
756
|
+
buffer = buffer.subarray(start - contentStart);
|
|
617
757
|
}
|
|
618
758
|
}
|
|
619
759
|
position += bytesRead;
|
|
@@ -627,7 +767,8 @@ class FileBackedBlob extends exports.Blob {
|
|
|
627
767
|
return resolve();
|
|
628
768
|
}
|
|
629
769
|
if (totalContentRead === size) {
|
|
630
|
-
|
|
770
|
+
settled = true;
|
|
771
|
+
closeFd();
|
|
631
772
|
controller.close();
|
|
632
773
|
}
|
|
633
774
|
resolve();
|
|
@@ -635,8 +776,10 @@ class FileBackedBlob extends exports.Blob {
|
|
|
635
776
|
});
|
|
636
777
|
},
|
|
637
778
|
cancel() {
|
|
638
|
-
|
|
779
|
+
cancelled = true;
|
|
780
|
+
closeFd();
|
|
639
781
|
clearTimeout(timer);
|
|
782
|
+
clearTimeout(openTimer);
|
|
640
783
|
if (watcher)
|
|
641
784
|
watcher.close();
|
|
642
785
|
},
|