harper 5.1.20 → 5.2.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/operations.ts +13 -16
- package/components/Application.ts +188 -7
- package/components/ApplicationScope.ts +3 -0
- package/components/Scope.ts +12 -0
- package/components/anthropic/index.ts +9 -2
- package/components/bedrock/index.ts +25 -2
- package/components/componentLoader.ts +56 -7
- package/components/componentSecrets.ts +451 -0
- package/components/deploymentRecorder.ts +7 -1
- package/components/mcp/adapters/fastify.ts +3 -0
- package/components/mcp/adapters/harperHttp.ts +3 -0
- package/components/mcp/audit.ts +21 -6
- package/components/mcp/customResourceRegistry.ts +17 -38
- package/components/mcp/listChanged.ts +37 -20
- package/components/mcp/quota.ts +145 -0
- package/components/mcp/rateLimit.ts +152 -14
- package/components/mcp/resources.ts +2 -4
- package/components/mcp/toolRegistry.ts +73 -4
- package/components/mcp/tools/application.ts +77 -8
- package/components/mcp/tools/operations.ts +106 -36
- package/components/mcp/tools/schemas/operationDescriptions.ts +4 -4
- package/components/mcp/tools/schemas/operations.ts +5 -0
- package/components/mcp/transport.ts +55 -4
- package/components/ollama/index.ts +6 -2
- package/components/openai/index.ts +9 -2
- package/components/operations.js +204 -10
- package/components/operationsValidation.js +173 -0
- package/components/secretOperations.ts +498 -0
- package/config/configUtils.ts +25 -1
- package/config-app.schema.json +81 -0
- package/config-root.schema.json +8 -1
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +4 -3
- package/dataLayer/readAuditLog.ts +14 -0
- package/dataLayer/schemaDescribe.ts +1 -0
- package/dist/agent/operations.d.ts +7 -4
- package/dist/agent/operations.js +13 -15
- package/dist/agent/operations.js.map +1 -1
- package/dist/components/Application.d.ts +25 -2
- package/dist/components/Application.js +166 -7
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +2 -0
- package/dist/components/ApplicationScope.js +3 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/Scope.d.ts +8 -0
- package/dist/components/Scope.js +11 -0
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/anthropic/index.d.ts +4 -1
- package/dist/components/anthropic/index.js +4 -2
- package/dist/components/anthropic/index.js.map +1 -1
- package/dist/components/bedrock/index.d.ts +4 -1
- package/dist/components/bedrock/index.js +22 -2
- package/dist/components/bedrock/index.js.map +1 -1
- package/dist/components/componentLoader.d.ts +2 -0
- package/dist/components/componentLoader.js +52 -10
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentSecrets.d.ts +63 -0
- package/dist/components/componentSecrets.js +405 -0
- package/dist/components/componentSecrets.js.map +1 -0
- package/dist/components/deploymentRecorder.d.ts +3 -0
- package/dist/components/deploymentRecorder.js +4 -3
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/mcp/adapters/fastify.d.ts +2 -0
- package/dist/components/mcp/adapters/fastify.js +1 -0
- package/dist/components/mcp/adapters/fastify.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.d.ts +2 -0
- package/dist/components/mcp/adapters/harperHttp.js +1 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/audit.d.ts +2 -2
- package/dist/components/mcp/audit.js +20 -5
- package/dist/components/mcp/audit.js.map +1 -1
- package/dist/components/mcp/customResourceRegistry.d.ts +5 -12
- package/dist/components/mcp/customResourceRegistry.js +15 -30
- package/dist/components/mcp/customResourceRegistry.js.map +1 -1
- package/dist/components/mcp/listChanged.d.ts +7 -2
- package/dist/components/mcp/listChanged.js +28 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/quota.d.ts +31 -0
- package/dist/components/mcp/quota.js +155 -0
- package/dist/components/mcp/quota.js.map +1 -0
- package/dist/components/mcp/rateLimit.d.ts +19 -3
- package/dist/components/mcp/rateLimit.js +129 -13
- package/dist/components/mcp/rateLimit.js.map +1 -1
- package/dist/components/mcp/resources.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +39 -1
- package/dist/components/mcp/toolRegistry.js +60 -5
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/application.d.ts +8 -1
- package/dist/components/mcp/tools/application.js +65 -5
- package/dist/components/mcp/tools/application.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +15 -3
- package/dist/components/mcp/tools/operations.js +99 -38
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +4 -4
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operations.js +4 -0
- package/dist/components/mcp/tools/schemas/operations.js.map +1 -1
- package/dist/components/mcp/transport.d.ts +6 -0
- package/dist/components/mcp/transport.js +47 -3
- package/dist/components/mcp/transport.js.map +1 -1
- package/dist/components/ollama/index.d.ts +4 -1
- package/dist/components/ollama/index.js +4 -2
- package/dist/components/ollama/index.js.map +1 -1
- package/dist/components/openai/index.d.ts +4 -1
- package/dist/components/openai/index.js +4 -2
- package/dist/components/openai/index.js.map +1 -1
- package/dist/components/operations.d.ts +39 -0
- package/dist/components/operations.js +194 -2
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.d.ts +38 -0
- package/dist/components/operationsValidation.js +155 -0
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/secretOperations.d.ts +113 -0
- package/dist/components/secretOperations.js +475 -0
- package/dist/components/secretOperations.js.map +1 -0
- package/dist/config/configUtils.d.ts +4 -1
- package/dist/config/configUtils.js +18 -1
- package/dist/config/configUtils.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +4 -3
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js.map +1 -1
- package/dist/dataLayer/readAuditLog.js +11 -0
- package/dist/dataLayer/readAuditLog.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +2 -0
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/globals.d.ts +1 -1
- package/dist/globals.js +1 -0
- package/dist/globals.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +38 -0
- package/dist/resources/DatabaseTransaction.d.ts +31 -3
- package/dist/resources/DatabaseTransaction.js +190 -29
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +1 -1
- package/dist/resources/LMDBTransaction.js +43 -5
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +51 -0
- package/dist/resources/PrimaryRocksDatabase.js +194 -0
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -0
- package/dist/resources/RecordEncoder.d.ts +1 -0
- package/dist/resources/RecordEncoder.js +81 -75
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +19 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.d.ts +0 -1
- package/dist/resources/Resource.js +103 -16
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +6 -2
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/Resources.js.map +1 -1
- package/dist/resources/Table.d.ts +6 -4
- package/dist/resources/Table.js +247 -24
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/read.js +25 -28
- package/dist/resources/analytics/read.js.map +1 -1
- package/dist/resources/blob.js +165 -22
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/crdt.d.ts +2 -0
- package/dist/resources/crdt.js +45 -36
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/databases.d.ts +1 -0
- package/dist/resources/databases.js +134 -7
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/graphql.js +8 -0
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +35 -5
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +239 -14
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/jsResource.js +6 -0
- package/dist/resources/jsResource.js.map +1 -1
- package/dist/resources/loadEnv.js +30 -3
- package/dist/resources/loadEnv.js.map +1 -1
- package/dist/resources/login.js +2 -2
- package/dist/resources/login.js.map +1 -1
- package/dist/resources/models/Models.js +14 -3
- package/dist/resources/models/Models.js.map +1 -1
- package/dist/resources/models/embedHook.js +10 -2
- package/dist/resources/models/embedHook.js.map +1 -1
- package/dist/resources/models/openaiStream.d.ts +56 -0
- package/dist/resources/models/openaiStream.js +94 -0
- package/dist/resources/models/openaiStream.js.map +1 -0
- package/dist/resources/replayLogs.js +4 -3
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/search.d.ts +1 -1
- package/dist/resources/search.js +104 -11
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/secretDecryptor.d.ts +54 -0
- package/dist/resources/secretDecryptor.js +131 -0
- package/dist/resources/secretDecryptor.js.map +1 -0
- package/dist/resources/tracked.js +15 -5
- package/dist/resources/tracked.js.map +1 -1
- package/dist/security/auth.js +71 -8
- package/dist/security/auth.js.map +1 -1
- package/dist/security/jsLoader.js +11 -0
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +9 -1
- package/dist/security/tokenAuthentication.js +31 -1
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +1 -1
- package/dist/server/DurableSubscriptionsSession.js +8 -4
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +28 -9
- package/dist/server/REST.js.map +1 -1
- package/dist/server/fastifyRoutes.js +11 -1
- package/dist/server/fastifyRoutes.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -3
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +40 -2
- package/dist/server/http.js +560 -77
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +64 -0
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +12 -0
- package/dist/server/liveSubscriptionAuth.js +140 -0
- package/dist/server/liveSubscriptionAuth.js.map +1 -0
- package/dist/server/loadRootComponents.js +2 -1
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/middlewareChain.d.ts +44 -1
- package/dist/server/middlewareChain.js +86 -10
- package/dist/server/middlewareChain.js.map +1 -1
- package/dist/server/mqtt.js +2 -1
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/operationsServer.js +1 -1
- package/dist/server/operationsServer.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +9 -0
- package/dist/server/serverHelpers/Headers.js +22 -0
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/JSONStream.js +3 -3
- package/dist/server/serverHelpers/JSONStream.js.map +1 -1
- package/dist/server/serverHelpers/Request.d.ts +52 -1
- package/dist/server/serverHelpers/Request.js +122 -1
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.js +3 -8
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/progressEmitter.d.ts +19 -0
- package/dist/server/serverHelpers/progressEmitter.js +52 -4
- package/dist/server/serverHelpers/progressEmitter.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +40 -0
- package/dist/server/serverHelpers/registeredOperations.js +286 -0
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +11 -5
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +1 -0
- package/dist/server/serverHelpers/serverUtilities.js +67 -5
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/uwsServer.d.ts +80 -0
- package/dist/server/serverHelpers/uwsServer.js +433 -0
- package/dist/server/serverHelpers/uwsServer.js.map +1 -0
- package/dist/server/static.d.ts +0 -15
- package/dist/server/static.js +161 -2
- package/dist/server/static.js.map +1 -1
- package/dist/server/status/index.d.ts +4 -1
- package/dist/server/status/index.js +64 -4
- package/dist/server/status/index.js.map +1 -1
- package/dist/server/threads/manageThreads.d.ts +10 -0
- package/dist/server/threads/manageThreads.js +107 -4
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.js +5 -1
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadServer.js +92 -14
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/upgrade/directives/5-2-0.d.ts +7 -0
- package/dist/upgrade/directives/5-2-0.js +108 -0
- package/dist/upgrade/directives/5-2-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/common_utils.js +5 -0
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/envFile.d.ts +41 -0
- package/dist/utility/envFile.js +221 -0
- package/dist/utility/envFile.js.map +1 -0
- package/dist/utility/globalSchema.d.ts +9 -0
- package/dist/utility/hdbTerms.d.ts +25 -0
- package/dist/utility/hdbTerms.js +31 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +21 -0
- package/dist/utility/logging/harper_logger.js +160 -8
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logRotator.js +29 -17
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/logging/readLog.d.ts +1 -1
- package/dist/utility/logging/readLog.js +305 -2
- package/dist/utility/logging/readLog.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +18 -0
- package/dist/utility/operationPermissions.js +35 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +17 -0
- package/dist/utility/operation_authorization.js +38 -0
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/secretEnvelope.d.ts +30 -0
- package/dist/utility/secretEnvelope.js +94 -0
- package/dist/utility/secretEnvelope.js.map +1 -0
- package/dist/utility/signalling.d.ts +7 -0
- package/dist/utility/signalling.js +16 -0
- package/dist/utility/signalling.js.map +1 -1
- package/dist/validation/configValidator.js +70 -24
- package/dist/validation/configValidator.js.map +1 -1
- package/index.ts +4 -0
- package/json/systemSchema.json +38 -0
- package/npm-shrinkwrap.json +15066 -9347
- package/package.json +13 -8
- package/resources/DESIGN.md +15 -12
- package/resources/DatabaseTransaction.ts +206 -37
- package/resources/LMDBTransaction.ts +43 -9
- package/resources/PrimaryRocksDatabase.ts +201 -0
- package/resources/RecordEncoder.ts +81 -69
- package/resources/RequestTarget.ts +21 -0
- package/resources/Resource.ts +109 -25
- package/resources/ResourceInterface.ts +6 -2
- package/resources/Resources.ts +1 -3
- package/resources/Table.ts +238 -29
- package/resources/analytics/read.ts +25 -30
- package/resources/blob.ts +159 -23
- package/resources/crdt.ts +37 -31
- package/resources/databases.ts +143 -6
- package/resources/graphql.ts +8 -0
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +251 -17
- package/resources/jsResource.ts +6 -0
- package/resources/loadEnv.ts +34 -3
- package/resources/login.ts +2 -2
- package/resources/models/Models.ts +19 -3
- package/resources/models/embedHook.ts +13 -2
- package/resources/models/openaiStream.ts +133 -0
- package/resources/replayLogs.ts +4 -3
- package/resources/search.ts +113 -11
- package/resources/secretDecryptor.ts +159 -0
- package/resources/tracked.ts +24 -17
- package/security/auth.ts +70 -10
- package/security/jsLoader.ts +12 -0
- package/security/tokenAuthentication.ts +42 -1
- package/server/DESIGN.md +10 -0
- package/server/DurableSubscriptionsSession.ts +8 -4
- package/server/REST.ts +26 -9
- package/server/fastifyRoutes.ts +10 -1
- package/server/graphqlQuerying.ts +4 -3
- package/server/http.ts +563 -77
- package/server/itc/serverHandlers.js +70 -0
- package/server/liveSubscriptionAuth.ts +152 -0
- package/server/loadRootComponents.js +2 -1
- package/server/middlewareChain.ts +107 -17
- package/server/mqtt.ts +2 -1
- package/server/operationsServer.ts +2 -2
- package/server/serverHelpers/Headers.ts +24 -0
- package/server/serverHelpers/JSONStream.ts +3 -3
- package/server/serverHelpers/Request.ts +127 -0
- package/server/serverHelpers/contentTypes.ts +3 -8
- package/server/serverHelpers/progressEmitter.ts +55 -5
- package/server/serverHelpers/registeredOperations.ts +259 -0
- package/server/serverHelpers/serverHandlers.js +11 -5
- package/server/serverHelpers/serverUtilities.ts +88 -5
- package/server/serverHelpers/uwsServer.ts +488 -0
- package/server/static.ts +180 -2
- package/server/status/index.ts +72 -4
- package/server/threads/manageThreads.js +102 -4
- package/server/threads/socketRouter.ts +5 -1
- package/server/threads/threadServer.js +93 -10
- package/studio/web/assets/{Chat-SdD1EZca.js → Chat-Cv_2paZE.js} +2 -2
- package/studio/web/assets/{Chat-SdD1EZca.js.map → Chat-Cv_2paZE.js.map} +1 -1
- package/studio/web/assets/{FloatingChat-GMq6tHK9.js → FloatingChat-CPyPIcVR.js} +4 -4
- package/studio/web/assets/{FloatingChat-GMq6tHK9.js.map → FloatingChat-CPyPIcVR.js.map} +1 -1
- package/studio/web/assets/{applications-B5hhv7uA.js → applications-C0jT2vdZ.js} +2 -2
- package/studio/web/assets/{applications-B5hhv7uA.js.map → applications-C0jT2vdZ.js.map} +1 -1
- package/studio/web/assets/{index-VFcpNWgq.js → index-D_GKOkhn.js} +6 -6
- package/studio/web/assets/index-D_GKOkhn.js.map +1 -0
- package/studio/web/assets/{index.lazy-DgcJojIA.js → index.lazy-CAmCIA65.js} +4 -4
- package/studio/web/assets/{index.lazy-DgcJojIA.js.map → index.lazy-CAmCIA65.js.map} +1 -1
- package/studio/web/assets/{profile-BqN-8vzm.js → profile-CaF-4aZe.js} +2 -2
- package/studio/web/assets/{profile-BqN-8vzm.js.map → profile-CaF-4aZe.js.map} +1 -1
- package/studio/web/assets/{setComponentFile-CHFvFq9u.js → setComponentFile-DLW1DHCt.js} +2 -2
- package/studio/web/assets/{setComponentFile-CHFvFq9u.js.map → setComponentFile-DLW1DHCt.js.map} +1 -1
- package/studio/web/assets/{setup-CQ3vWJSr.js → setup-TGCErIhq.js} +2 -2
- package/studio/web/assets/{setup-CQ3vWJSr.js.map → setup-TGCErIhq.js.map} +1 -1
- package/studio/web/assets/{status-BsW5fjuh.js → status-DG0Maoao.js} +2 -2
- package/studio/web/assets/{status-BsW5fjuh.js.map → status-DG0Maoao.js.map} +1 -1
- package/studio/web/assets/{swagger-ui-react-DITdeB9-.js → swagger-ui-react-8T4SgQ1m.js} +2 -2
- package/studio/web/assets/{swagger-ui-react-DITdeB9-.js.map → swagger-ui-react-8T4SgQ1m.js.map} +1 -1
- package/studio/web/assets/{tsMode-Dyph-OUs.js → tsMode-DJ6Sl24Q.js} +2 -2
- package/studio/web/assets/{tsMode-Dyph-OUs.js.map → tsMode-DJ6Sl24Q.js.map} +1 -1
- package/studio/web/assets/{useEntityRestURL-D0QTUqex.js → useEntityRestURL-JO2mfWTQ.js} +2 -2
- package/studio/web/assets/{useEntityRestURL-D0QTUqex.js.map → useEntityRestURL-JO2mfWTQ.js.map} +1 -1
- package/studio/web/index.html +1 -1
- package/upgrade/directives/5-2-0.ts +82 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/common_utils.ts +5 -0
- package/utility/envFile.ts +218 -0
- package/utility/hdbTerms.ts +31 -0
- package/utility/logging/harper_logger.ts +156 -8
- package/utility/logging/logRotator.ts +24 -14
- package/utility/logging/readLog.ts +323 -2
- package/utility/operationPermissions.ts +35 -1
- package/utility/operation_authorization.ts +74 -1
- package/utility/secretEnvelope.ts +113 -0
- package/utility/signalling.ts +15 -0
- package/validation/configValidator.ts +78 -25
- package/studio/web/assets/index-VFcpNWgq.js.map +0 -1
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `openaiStream()` — format an internal `generateStream()` token iterator into the
|
|
3
|
+
* OpenAI-compatible Server-Sent Events shape so unmodified OpenAI / LangChain.js /
|
|
4
|
+
* Vercel AI SDK clients can consume a Harper chat-completions stream (#514, #510).
|
|
5
|
+
*
|
|
6
|
+
* The yielded `{ data }` messages pass through Harper's existing `text/event-stream`
|
|
7
|
+
* serializer (`server/serverHelpers/contentTypes.ts`) unchanged: an object `data`
|
|
8
|
+
* is JSON-stringified to `data: {json}\n\n`, and the terminal `{ data: '[DONE]' }`
|
|
9
|
+
* sentinel serializes to `data: [DONE]\n\n` — exactly OpenAI's wire format. We emit
|
|
10
|
+
* NO SSE `event:` or `id:` lines: OpenAI's stream is `data:`-only and the completion
|
|
11
|
+
* id lives inside the JSON payload, not as an SSE field.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { randomUUID } from 'node:crypto';
|
|
15
|
+
import type { GenerateChunk, GenerateResult } from './types.ts';
|
|
16
|
+
|
|
17
|
+
type OpenAIFinishReason = GenerateResult['finishReason'];
|
|
18
|
+
|
|
19
|
+
export interface OpenAIStreamOptions {
|
|
20
|
+
/** Advertised model name echoed back on every chunk. */
|
|
21
|
+
model?: string;
|
|
22
|
+
/** Reuse a caller-supplied completion id across all chunks; one is generated when omitted. */
|
|
23
|
+
id?: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
interface OpenAIToolCallDelta {
|
|
27
|
+
index: number;
|
|
28
|
+
id: string;
|
|
29
|
+
type: 'function';
|
|
30
|
+
function: { name: string; arguments: string };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
interface OpenAIDelta {
|
|
34
|
+
role?: 'assistant';
|
|
35
|
+
content?: string;
|
|
36
|
+
tool_calls?: OpenAIToolCallDelta[];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface OpenAIChunk {
|
|
40
|
+
id: string;
|
|
41
|
+
object: 'chat.completion.chunk';
|
|
42
|
+
created: number;
|
|
43
|
+
model: string;
|
|
44
|
+
choices: Array<{ index: number; delta: OpenAIDelta; finish_reason: OpenAIFinishReason | null }>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** SSE message envelope consumed by Harper's `text/event-stream` serializer. */
|
|
48
|
+
export interface OpenAIStreamMessage {
|
|
49
|
+
data: OpenAIChunk | string;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Wrap a `GenerateChunk` async iterable as OpenAI `chat.completion.chunk` SSE messages,
|
|
54
|
+
* terminated by the `[DONE]` sentinel. Content deltas stream inline; tool calls are
|
|
55
|
+
* assembled and flushed once (see the tool-call note below).
|
|
56
|
+
*/
|
|
57
|
+
export async function* openaiStream(
|
|
58
|
+
tokens: AsyncIterable<GenerateChunk>,
|
|
59
|
+
opts: OpenAIStreamOptions = {}
|
|
60
|
+
): AsyncGenerator<OpenAIStreamMessage> {
|
|
61
|
+
const id = opts.id ?? `chatcmpl-${randomUUID().replaceAll('-', '')}`;
|
|
62
|
+
const created = Math.floor(Date.now() / 1000);
|
|
63
|
+
const model = opts.model ?? '';
|
|
64
|
+
|
|
65
|
+
let roleSent = false;
|
|
66
|
+
let finishReason: OpenAIFinishReason | undefined;
|
|
67
|
+
|
|
68
|
+
// Tool-call assembly. Backends pre-parse arguments to objects and may re-send the
|
|
69
|
+
// same id with partial fields (see `mergeToolCallDelta` in agentLoop.ts), so we
|
|
70
|
+
// accumulate by id here and emit each call's arguments as ONE stringified blob.
|
|
71
|
+
// Emitting incremental fragments would corrupt the OpenAI client's concatenation
|
|
72
|
+
// (`{"a":1}` + `{"b":2}` → invalid JSON) — Harper's already-buffered upstream model
|
|
73
|
+
// means we cannot faithfully reproduce per-token argument fragments anyway.
|
|
74
|
+
const toolAssembly = new Map<string, { index: number; name?: string; arguments: object }>();
|
|
75
|
+
|
|
76
|
+
const chunk = (delta: OpenAIDelta, finish: OpenAIFinishReason | null): OpenAIStreamMessage => ({
|
|
77
|
+
data: {
|
|
78
|
+
id,
|
|
79
|
+
object: 'chat.completion.chunk',
|
|
80
|
+
created,
|
|
81
|
+
model,
|
|
82
|
+
choices: [{ index: 0, delta, finish_reason: finish }],
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
for await (const token of tokens) {
|
|
87
|
+
if (token.deltaContent !== undefined) {
|
|
88
|
+
const delta: OpenAIDelta = {};
|
|
89
|
+
if (!roleSent) {
|
|
90
|
+
delta.role = 'assistant';
|
|
91
|
+
roleSent = true;
|
|
92
|
+
}
|
|
93
|
+
delta.content = token.deltaContent;
|
|
94
|
+
yield chunk(delta, null);
|
|
95
|
+
}
|
|
96
|
+
if (token.deltaToolCalls) {
|
|
97
|
+
for (const incoming of token.deltaToolCalls) {
|
|
98
|
+
if (!incoming.id) continue;
|
|
99
|
+
const existing = toolAssembly.get(incoming.id) ?? { index: toolAssembly.size, arguments: {} };
|
|
100
|
+
if (incoming.name) existing.name = incoming.name;
|
|
101
|
+
if (incoming.arguments) existing.arguments = { ...existing.arguments, ...incoming.arguments };
|
|
102
|
+
toolAssembly.set(incoming.id, existing);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (token.finishReason) finishReason = token.finishReason;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (toolAssembly.size > 0) {
|
|
109
|
+
const toolCalls: OpenAIToolCallDelta[] = [];
|
|
110
|
+
for (const [callId, call] of toolAssembly) {
|
|
111
|
+
toolCalls.push({
|
|
112
|
+
index: call.index,
|
|
113
|
+
id: callId,
|
|
114
|
+
type: 'function',
|
|
115
|
+
function: { name: call.name ?? '', arguments: JSON.stringify(call.arguments) },
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
const delta: OpenAIDelta = {};
|
|
119
|
+
if (!roleSent) {
|
|
120
|
+
delta.role = 'assistant';
|
|
121
|
+
roleSent = true;
|
|
122
|
+
}
|
|
123
|
+
delta.tool_calls = toolCalls;
|
|
124
|
+
yield chunk(delta, null);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const finish: OpenAIFinishReason = finishReason ?? (toolAssembly.size > 0 ? 'tool_calls' : 'stop');
|
|
128
|
+
const terminalDelta: OpenAIDelta = {};
|
|
129
|
+
if (!roleSent) terminalDelta.role = 'assistant';
|
|
130
|
+
yield chunk(terminalDelta, finish);
|
|
131
|
+
|
|
132
|
+
yield { data: '[DONE]' };
|
|
133
|
+
}
|
package/resources/replayLogs.ts
CHANGED
|
@@ -190,10 +190,11 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
|
|
|
190
190
|
transaction = new DatabaseTransaction();
|
|
191
191
|
transaction.db = primaryStore;
|
|
192
192
|
transaction.timestamp = version;
|
|
193
|
-
//
|
|
193
|
+
// retries=1 routes operation.commit() through its retry path (no duplicate audit staging)
|
|
194
194
|
transaction.retries = 1;
|
|
195
|
-
// Explicit replay marker
|
|
196
|
-
//
|
|
195
|
+
// Explicit replay marker: skips schema validation (harper#1316) and makes save() stamp
|
|
196
|
+
// the native transaction isRetry, so replayed writes are never re-appended to the
|
|
197
|
+
// transaction log being iterated (re-appending would prevent replay convergence).
|
|
197
198
|
transaction.isReplay = true;
|
|
198
199
|
}
|
|
199
200
|
context.transaction = transaction;
|
package/resources/search.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { ClientError, IndexRebuildingError, Violation } from '../utility/errors/
|
|
|
2
2
|
import { OVERFLOW_MARKER, MAX_SEARCH_KEY_LENGTH, SEARCH_TYPES } from '../utility/lmdb/terms.ts';
|
|
3
3
|
import { compareKeys, MAXIMUM_KEY } from 'ordered-binary';
|
|
4
4
|
import { SKIP } from '@harperfast/extended-iterable';
|
|
5
|
-
import { INVALIDATED, EVICTED } from './Table.ts';
|
|
5
|
+
import { INVALIDATED, EVICTED, freezeRecord } from './Table.ts';
|
|
6
6
|
import type { DirectCondition, Id } from './ResourceInterface.ts';
|
|
7
7
|
import { RequestTarget } from './RequestTarget.ts';
|
|
8
8
|
import { lastMetadata } from './RecordEncoder.ts';
|
|
@@ -44,8 +44,24 @@ export const COERCIBLE_OPERATORS = {
|
|
|
44
44
|
ne: true,
|
|
45
45
|
eq: true,
|
|
46
46
|
};
|
|
47
|
-
export function executeConditions(
|
|
47
|
+
export function executeConditions(
|
|
48
|
+
conditions,
|
|
49
|
+
operator,
|
|
50
|
+
table,
|
|
51
|
+
txn,
|
|
52
|
+
request,
|
|
53
|
+
context,
|
|
54
|
+
transformToEntries,
|
|
55
|
+
filtered,
|
|
56
|
+
recordAccess?
|
|
57
|
+
) {
|
|
48
58
|
const firstSearch = conditions[0];
|
|
59
|
+
// Record-level guards (a caller-supplied vectorFilter + RBAC allowReadRecord) apply to every record
|
|
60
|
+
// the query returns, independent of which condition leads (#1241). `recordAccess` is supplied only on
|
|
61
|
+
// the top-level executeConditions call (Table.search) and deliberately NOT threaded into the recursive
|
|
62
|
+
// calls below, so the guards run exactly once — on the final result set — rather than redundantly at
|
|
63
|
+
// each nested group.
|
|
64
|
+
const recordGuards = buildRecordGuards(recordAccess);
|
|
49
65
|
// both AND and OR start by getting an iterator for the ids for first condition
|
|
50
66
|
// and then things diverge...
|
|
51
67
|
if (operator === 'or') {
|
|
@@ -58,7 +74,7 @@ export function executeConditions(conditions, operator, table, txn, request, con
|
|
|
58
74
|
results = results.concat(nextResults);
|
|
59
75
|
}
|
|
60
76
|
const returnedIds = new Set();
|
|
61
|
-
|
|
77
|
+
const deduped = results.filter((entry) => {
|
|
62
78
|
const id = entry.key ?? entry;
|
|
63
79
|
if (returnedIds.has(id))
|
|
64
80
|
// skip duplicate ids
|
|
@@ -66,14 +82,35 @@ export function executeConditions(conditions, operator, table, txn, request, con
|
|
|
66
82
|
returnedIds.add(id);
|
|
67
83
|
return true;
|
|
68
84
|
});
|
|
85
|
+
// Record guards must hold on the OR union too — loading each surviving record and dropping the
|
|
86
|
+
// ones the guards reject (transformToEntries loads the record to evaluate a filter). Without this
|
|
87
|
+
// a record-level RBAC check would be bypassed by any OR query.
|
|
88
|
+
return recordGuards ? transformToEntries(deduped, recordGuards) : deduped;
|
|
69
89
|
} else {
|
|
70
|
-
// AND
|
|
90
|
+
// AND: use the indexed query for the first condition and filter by all subsequent conditions.
|
|
91
|
+
if (isFilterablePushdown(firstSearch, table)) {
|
|
92
|
+
// The lead is a filterable custom index (HNSW). It doesn't populate the join map (`filtered`)
|
|
93
|
+
// that sibling filters read, so — unlike the general case below — we can build the sibling
|
|
94
|
+
// filters BEFORE executing the lead and push them into the traversal, so filtering happens
|
|
95
|
+
// during the graph walk instead of post-filtering an under-filled candidate set (#1241). They
|
|
96
|
+
// also stay in the post-filter chain as a deterministic, harmless re-check.
|
|
97
|
+
const filters = mapConditionsToFilters(conditions.slice(1), true, firstSearch.estimated_count);
|
|
98
|
+
const recordFilters = recordGuards ? filters.concat(recordGuards) : filters;
|
|
99
|
+
// Execute a COPY carrying the pushed-down predicate, never mutating the caller's condition
|
|
100
|
+
// object (which may be reused across requests with different users / guards).
|
|
101
|
+
const lead =
|
|
102
|
+
recordFilters.length > 0
|
|
103
|
+
? { ...firstSearch, recordFilter: composeRecordFilter(recordFilters, table, context) }
|
|
104
|
+
: firstSearch;
|
|
105
|
+
const results = executeCondition(lead);
|
|
106
|
+
return recordFilters.length > 0 ? transformToEntries(results, recordFilters) : results;
|
|
107
|
+
}
|
|
108
|
+
// General path: execute the lead FIRST — a leading join populates `filtered`, which the sibling
|
|
109
|
+
// filters read as they are built — then apply the sibling filters plus any record guards.
|
|
71
110
|
const results = executeCondition(firstSearch);
|
|
72
|
-
// get the intersection of condition searches by using the indexed query for the first condition
|
|
73
|
-
// and then filtering by all subsequent conditions.
|
|
74
|
-
// now apply filters that require looking up records
|
|
75
111
|
const filters = mapConditionsToFilters(conditions.slice(1), true, firstSearch.estimated_count);
|
|
76
|
-
|
|
112
|
+
const recordFilters = recordGuards ? filters.concat(recordGuards) : filters;
|
|
113
|
+
return recordFilters.length > 0 ? transformToEntries(results, recordFilters) : results;
|
|
77
114
|
}
|
|
78
115
|
function executeCondition(condition) {
|
|
79
116
|
if (condition.conditions)
|
|
@@ -86,6 +123,7 @@ export function executeConditions(conditions, operator, table, txn, request, con
|
|
|
86
123
|
context,
|
|
87
124
|
transformToEntries,
|
|
88
125
|
filtered
|
|
126
|
+
// recordAccess intentionally omitted: guards run once, at the top level (see above).
|
|
89
127
|
);
|
|
90
128
|
return searchByIndex(
|
|
91
129
|
condition,
|
|
@@ -122,6 +160,65 @@ export function executeConditions(conditions, operator, table, txn, request, con
|
|
|
122
160
|
}
|
|
123
161
|
}
|
|
124
162
|
|
|
163
|
+
/**
|
|
164
|
+
* Build the record-level guards that apply to a query independent of its conditions (#1241):
|
|
165
|
+
* - `vectorFilter`: a caller-supplied `(record) => boolean` predicate (JS-API only).
|
|
166
|
+
* - `allowReadRecord`: a resource's static record-level RBAC check `(user, record) => boolean`.
|
|
167
|
+
* Both arrive already resolved on `recordAccess` (assembled once in Table.search). Returns an array of
|
|
168
|
+
* `(record) => boolean` predicates, or undefined when neither is defined (the common case — zero
|
|
169
|
+
* overhead). Predicates must be synchronous and side-effect free; records passed in are frozen.
|
|
170
|
+
*/
|
|
171
|
+
function buildRecordGuards(recordAccess): ((record: any) => boolean)[] | undefined {
|
|
172
|
+
if (!recordAccess) return undefined;
|
|
173
|
+
const guards: ((record: any) => boolean)[] = [];
|
|
174
|
+
const vectorFilter = recordAccess.vectorFilter;
|
|
175
|
+
if (typeof vectorFilter === 'function') guards.push((record) => vectorFilter(record));
|
|
176
|
+
const allowReadRecord = recordAccess.allowReadRecord;
|
|
177
|
+
if (typeof allowReadRecord === 'function') {
|
|
178
|
+
const user = recordAccess.user;
|
|
179
|
+
guards.push((record) => allowReadRecord(user, record));
|
|
180
|
+
}
|
|
181
|
+
return guards.length > 0 ? guards : undefined;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** True when a condition's index is a custom index that participates in predicate-aware traversal (HNSW). */
|
|
185
|
+
function isFilterablePushdown(condition, table): boolean {
|
|
186
|
+
const attributeName = condition?.attribute ?? condition?.[0];
|
|
187
|
+
if (attributeName == null || table == null) return false;
|
|
188
|
+
const index = attributeName === table.primaryKey ? table.primaryStore : table.indices?.[attributeName];
|
|
189
|
+
return Boolean(index?.customIndex?.filteredSearch);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Compose a set of record predicates into the single `(primaryKey) => boolean` the custom index calls
|
|
194
|
+
* during traversal (#1241). The record is loaded lazily (only when the index reaches a node) via
|
|
195
|
+
* `primaryStore.getEntry` and frozen before the predicates see it. Verdicts are memoized per query since
|
|
196
|
+
* the graph can reach a node from multiple neighbors. A missing/deleted record fails the predicate.
|
|
197
|
+
*/
|
|
198
|
+
function composeRecordFilter(recordFilters, table, context): (primaryKey: Id) => boolean {
|
|
199
|
+
const memo = new Map<Id, boolean>();
|
|
200
|
+
const transaction = context && table._readTxnForContext ? table._readTxnForContext(context) : undefined;
|
|
201
|
+
return (primaryKey: Id) => {
|
|
202
|
+
const cached = memo.get(primaryKey);
|
|
203
|
+
if (cached !== undefined) return cached;
|
|
204
|
+
let verdict = false;
|
|
205
|
+
const entry = table.primaryStore.getEntry(primaryKey, { transaction });
|
|
206
|
+
const record = entry?.value;
|
|
207
|
+
if (record != null) {
|
|
208
|
+
freezeRecord(record);
|
|
209
|
+
verdict = true;
|
|
210
|
+
for (const filter of recordFilters) {
|
|
211
|
+
if (!filter(record, entry)) {
|
|
212
|
+
verdict = false;
|
|
213
|
+
break;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
memo.set(primaryKey, verdict);
|
|
218
|
+
return verdict;
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
125
222
|
/**
|
|
126
223
|
* Search for records or keys, based on the search condition, using an index if available
|
|
127
224
|
* @param searchCondition
|
|
@@ -375,7 +472,7 @@ export function searchByIndex(
|
|
|
375
472
|
let result: any;
|
|
376
473
|
if (entry.value == null && !(entry.metadataFlags & (INVALIDATED | EVICTED))) result = SKIP;
|
|
377
474
|
else {
|
|
378
|
-
|
|
475
|
+
freezeRecord(entry.value);
|
|
379
476
|
recordRead(entry);
|
|
380
477
|
result = entry;
|
|
381
478
|
}
|
|
@@ -387,7 +484,12 @@ export function searchByIndex(
|
|
|
387
484
|
return results;
|
|
388
485
|
} else if (index && !skipIndex) {
|
|
389
486
|
if (index.customIndex) {
|
|
390
|
-
|
|
487
|
+
// Predicate-aware traversal (#1241): a record filter composed from companion AND conditions,
|
|
488
|
+
// a caller-supplied vectorFilter, and record-level RBAC is pushed into the index so it keeps
|
|
489
|
+
// exploring until it has enough MATCHING results, rather than post-filtering an under-filled
|
|
490
|
+
// candidate set. Only indexes that opt in (filteredSearch) receive it; others post-filter as before.
|
|
491
|
+
const recordFilter = index.customIndex.filteredSearch ? searchCondition.recordFilter : undefined;
|
|
492
|
+
const loaded = index.customIndex.search(searchCondition, context, recordFilter).map((entry) => {
|
|
391
493
|
// if the custom index returns an entry with metadata, merge it with the loaded entry
|
|
392
494
|
if (typeof entry === 'object' && entry) {
|
|
393
495
|
const { key, ...otherProps } = entry;
|
|
@@ -396,7 +498,7 @@ export function searchByIndex(
|
|
|
396
498
|
transaction: context && Table._readTxnForContext(context),
|
|
397
499
|
});
|
|
398
500
|
if (!loadedEntry) return SKIP; // record was deleted/expired or not yet visible
|
|
399
|
-
|
|
501
|
+
freezeRecord(loadedEntry?.value);
|
|
400
502
|
recordRead(loadedEntry);
|
|
401
503
|
return { ...otherProps, ...loadedEntry };
|
|
402
504
|
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extension point for decrypting `enc:v1:` secret values at runtime. `.env` loading is the first
|
|
3
|
+
* consumer; the same hook serves any store of `enc:v1:` ciphertext (secret tables, deploy
|
|
4
|
+
* credentials) so key custody stays in one place.
|
|
5
|
+
*
|
|
6
|
+
* Core (Apache) ships NO decryptor: encrypted values stay opaque and are skipped at load time.
|
|
7
|
+
* The Harper Pro secrets component registers a real decryptor here at startup (it holds the
|
|
8
|
+
* cluster's secrets private key), at which point consumers like `loadEnv` transparently decrypt
|
|
9
|
+
* encrypted values. Without Pro the hook is empty and the feature is dormant.
|
|
10
|
+
*
|
|
11
|
+
* The decryptor is synchronous on purpose — `loadEnv` runs synchronously and Node's RSA/AES
|
|
12
|
+
* primitives are synchronous, so this avoids reworking the load path. It receives the full value
|
|
13
|
+
* (including the `enc:v1:` prefix) and returns the plaintext, or throws if it cannot decrypt.
|
|
14
|
+
*
|
|
15
|
+
* Registration is per-process: the Pro component registers in each worker where secrets are
|
|
16
|
+
* consumed, so registration and use share the same module instance.
|
|
17
|
+
*
|
|
18
|
+
* Registration order is NOT load-bearing for `.env` values: entries that load before a decryptor
|
|
19
|
+
* exists are queued (see `deferEncryptedEnvValue`) and replayed into `process.env` the moment one
|
|
20
|
+
* registers, so a custody provider that comes up after component `.env` loading heals the skipped
|
|
21
|
+
* values instead of leaving them missing.
|
|
22
|
+
*/
|
|
23
|
+
import logger from '../utility/logging/harper_logger.ts';
|
|
24
|
+
|
|
25
|
+
export type SecretDecryptor = (value: string) => string;
|
|
26
|
+
|
|
27
|
+
/** An encrypted `.env` entry that was loaded before any decryptor was registered. */
|
|
28
|
+
export interface DeferredEncryptedEnvValue {
|
|
29
|
+
key: string;
|
|
30
|
+
rawValue: string;
|
|
31
|
+
sourcePath: string;
|
|
32
|
+
override: boolean;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Defensive cap: the queue only ever holds encrypted `.env` entries loaded before registration,
|
|
36
|
+
// so growth beyond this indicates a bug or abuse — drop (with an error) rather than grow.
|
|
37
|
+
const MAX_DEFERRED_SECRETS = 1000;
|
|
38
|
+
let deferredSecrets: DeferredEncryptedEnvValue[] = [];
|
|
39
|
+
|
|
40
|
+
let decryptor: SecretDecryptor | undefined;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Queue an encrypted `.env` entry that could not be decrypted because no decryptor is registered
|
|
44
|
+
* yet. When one registers, the queue is replayed into `process.env` with the same
|
|
45
|
+
* override/conflict semantics `loadEnv` applied to the original load.
|
|
46
|
+
*/
|
|
47
|
+
export function deferEncryptedEnvValue(entry: DeferredEncryptedEnvValue): void {
|
|
48
|
+
// A re-evaluated env file (reload, watcher re-add) defers the same key again: replace the
|
|
49
|
+
// earlier entry in place so the queue holds one entry per {key, sourcePath} with the newest
|
|
50
|
+
// ciphertext, instead of wasting the cap and replaying stale values.
|
|
51
|
+
const existingIndex = deferredSecrets.findIndex(
|
|
52
|
+
(queued) => queued.key === entry.key && queued.sourcePath === entry.sourcePath
|
|
53
|
+
);
|
|
54
|
+
if (existingIndex >= 0) {
|
|
55
|
+
deferredSecrets[existingIndex] = entry;
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
if (deferredSecrets.length >= MAX_DEFERRED_SECRETS) {
|
|
59
|
+
logger.error(
|
|
60
|
+
`Deferred encrypted env queue is full (${MAX_DEFERRED_SECRETS}); dropping ${entry.key} from ${entry.sourcePath}`
|
|
61
|
+
);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
deferredSecrets.push(entry);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* A redacted snapshot of the queued entries awaiting a decryptor, for tests/diagnostics: a copy,
|
|
69
|
+
* without the ciphertext (`rawValue`), so holders can neither mutate pending entries nor retain
|
|
70
|
+
* the queued values past the flush.
|
|
71
|
+
*/
|
|
72
|
+
export function getDeferredEncryptedEnvValues(): Omit<DeferredEncryptedEnvValue, 'rawValue'>[] {
|
|
73
|
+
return deferredSecrets.map(({ key, sourcePath, override }) => ({ key, sourcePath, override }));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Replay queued encrypted env entries through a newly registered decryptor, mirroring loadEnv's
|
|
77
|
+
// conflict semantics: a failed decrypt logs and skips that entry; an already-set key is only
|
|
78
|
+
// overwritten when the entry's env file was loaded with `override`.
|
|
79
|
+
function replayDeferredSecrets(newDecryptor: SecretDecryptor): void {
|
|
80
|
+
const entries = deferredSecrets;
|
|
81
|
+
deferredSecrets = [];
|
|
82
|
+
for (const { key, rawValue, sourcePath, override } of entries) {
|
|
83
|
+
let value: string;
|
|
84
|
+
try {
|
|
85
|
+
value = newDecryptor(rawValue);
|
|
86
|
+
} catch (error) {
|
|
87
|
+
logger.error(
|
|
88
|
+
`Failed to decrypt deferred environment variable ${key} from ${sourcePath}: ${(error as Error).message}; skipping`
|
|
89
|
+
);
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
if (process.env[key] !== undefined) {
|
|
93
|
+
if (process.env[key] === value) continue; // already holds the decrypted value — nothing to report
|
|
94
|
+
logger.warn(`Environment variable conflict: ${key} from ${sourcePath} is already set on process.env`);
|
|
95
|
+
if (!override) continue;
|
|
96
|
+
}
|
|
97
|
+
process.env[key] = value;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Install the env-secret decryptor (called by the Pro env-secrets component at startup). Any
|
|
103
|
+
* encrypted `.env` entries that loaded before this point are immediately decrypted into
|
|
104
|
+
* `process.env`.
|
|
105
|
+
*/
|
|
106
|
+
export function registerSecretDecryptor(fn: SecretDecryptor): void {
|
|
107
|
+
decryptor = fn;
|
|
108
|
+
replayDeferredSecrets(fn);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The registered decryptor, or undefined when no Pro env-secrets component is active. */
|
|
112
|
+
export function getSecretDecryptor(): SecretDecryptor | undefined {
|
|
113
|
+
return decryptor;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Remove the registered decryptor and drop any deferred entries. Intended for tests. */
|
|
117
|
+
export function clearSecretDecryptor(): void {
|
|
118
|
+
decryptor = undefined;
|
|
119
|
+
deferredSecrets = [];
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The public half of the cluster secrets keypair, with its stable SHA-256 fingerprint (`kid`). */
|
|
123
|
+
export interface SecretCustodyPublicKey {
|
|
124
|
+
publicKey: string;
|
|
125
|
+
fingerprint: string;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Full key custody: decrypt plus access to the public key, so the node can encrypt on ingest
|
|
130
|
+
* (`set_secret` with a plaintext value) and serve `get_secrets_public_key`. Core ships no custody;
|
|
131
|
+
* the Pro secrets component registers one per process where secrets are used — including the MAIN
|
|
132
|
+
* thread, where the operations API dispatches the secret operations.
|
|
133
|
+
*/
|
|
134
|
+
export interface SecretCustody {
|
|
135
|
+
decrypt: SecretDecryptor;
|
|
136
|
+
getPublicKey(): SecretCustodyPublicKey;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
let custody: SecretCustody | undefined;
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Install key custody. Also installs `custody.decrypt` via the decryptor slot above, so consumers
|
|
143
|
+
* like `loadEnv` that only need decryption keep working through the existing hook.
|
|
144
|
+
*/
|
|
145
|
+
export function registerSecretCustody(newCustody: SecretCustody): void {
|
|
146
|
+
custody = newCustody;
|
|
147
|
+
registerSecretDecryptor(newCustody.decrypt);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** The registered custody, or undefined when no secrets key is held by this process. */
|
|
151
|
+
export function getSecretCustody(): SecretCustody | undefined {
|
|
152
|
+
return custody;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Remove the registered custody (and the decryptor it installed). Intended for tests. */
|
|
156
|
+
export function clearSecretCustody(): void {
|
|
157
|
+
custody = undefined;
|
|
158
|
+
decryptor = undefined;
|
|
159
|
+
}
|
package/resources/tracked.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ClientError } from '../utility/errors/hdbError.ts';
|
|
2
2
|
import * as crdtOperations from './crdt.ts';
|
|
3
3
|
import { Blob } from './blob.ts';
|
|
4
|
+
import * as harperLogger from '../utility/logging/harper_logger.ts';
|
|
4
5
|
|
|
5
6
|
function getChanges(target) {
|
|
6
7
|
let changes = target.getChanges();
|
|
@@ -59,13 +60,11 @@ export function assignTrackedAccessors(Target, typeDef, useFullPropertyProxy = f
|
|
|
59
60
|
break;
|
|
60
61
|
case 'ID':
|
|
61
62
|
set = function (value) {
|
|
62
|
-
if (
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
)
|
|
68
|
-
)
|
|
63
|
+
if (!(
|
|
64
|
+
typeof value === 'string' ||
|
|
65
|
+
(value?.length > 0 && value.every?.((value) => typeof value === 'string')) ||
|
|
66
|
+
(value == null && attribute.nullable !== false)
|
|
67
|
+
))
|
|
69
68
|
throw new ClientError(`${name} must be a string, attempt to assign ${value}`);
|
|
70
69
|
getChanges(this)[name] = value;
|
|
71
70
|
};
|
|
@@ -99,12 +98,10 @@ export function assignTrackedAccessors(Target, typeDef, useFullPropertyProxy = f
|
|
|
99
98
|
case 'Long':
|
|
100
99
|
set = function (value) {
|
|
101
100
|
let scalarValue = value?.__op__ ? value.value : value;
|
|
102
|
-
if (
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
)
|
|
107
|
-
) {
|
|
101
|
+
if (!(
|
|
102
|
+
(Math.round(scalarValue) === value && Math.abs(scalarValue) <= 9007199254740992) ||
|
|
103
|
+
(value == null && attribute.nullable !== false)
|
|
104
|
+
)) {
|
|
108
105
|
if (typeof scalarValue === 'number' && Math.abs(scalarValue) <= 9007199254740992) {
|
|
109
106
|
// if it just needs to be rounded, do the conversion without complaining
|
|
110
107
|
scalarValue = Math.round(scalarValue);
|
|
@@ -427,9 +424,18 @@ export function updateAndFreeze(target, changes = target.getChanges?.()) {
|
|
|
427
424
|
let value = changes[key];
|
|
428
425
|
if (value && typeof value === 'object') {
|
|
429
426
|
if (value.__op__) {
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
427
|
+
// Resolve against the explicit operation registry, not the crdt.ts export namespace,
|
|
428
|
+
// so a crafted/corrupt `__op__` naming another export can't be invoked as an operation.
|
|
429
|
+
const operation = crdtOperations.operations[value.__op__];
|
|
430
|
+
if (operation) operation(mergedUpdatedObject, key, value);
|
|
431
|
+
else {
|
|
432
|
+
// An unrecognized CRDT operation — corruption, or an op type from a newer node this
|
|
433
|
+
// version can't apply. This runs on the write/replication apply path, where throwing
|
|
434
|
+
// would abort the commit and can wedge a subscription, so skip the op instead: the
|
|
435
|
+
// field keeps its base value and a full-copy re-converges the record. Read-path
|
|
436
|
+
// reconstruction (crdt.applyForward/applyReverse) still throws loudly by design.
|
|
437
|
+
harperLogger.warn(`Skipping unrecognized CRDT operation "${value.__op__}" on property "${key}"`);
|
|
438
|
+
}
|
|
433
439
|
continue;
|
|
434
440
|
} else value = updateAndFreeze(value);
|
|
435
441
|
}
|
|
@@ -588,6 +594,7 @@ export class Addition {
|
|
|
588
594
|
this.value = value;
|
|
589
595
|
}
|
|
590
596
|
update(previousValue) {
|
|
591
|
-
|
|
597
|
+
// Shared with crdt.add so the read path folds identically to the storage path.
|
|
598
|
+
return crdtOperations.addValues(previousValue, this.value);
|
|
592
599
|
}
|
|
593
600
|
}
|