@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/BinObjects.js +17 -0
- package/bin/cliOperations.js +157 -0
- package/bin/copyDb.ts +280 -0
- package/bin/harper.js +156 -0
- package/bin/install.js +15 -0
- package/bin/lite.js +5 -0
- package/bin/restart.js +201 -0
- package/bin/run.js +409 -0
- package/bin/status.js +65 -0
- package/bin/stop.js +22 -0
- package/bin/upgrade.js +134 -0
- package/components/Application.ts +646 -0
- package/components/ApplicationScope.ts +49 -0
- package/components/Component.ts +53 -0
- package/components/ComponentV1.ts +342 -0
- package/components/DEFAULT_CONFIG.ts +18 -0
- package/components/EntryHandler.ts +227 -0
- package/components/Logger.ts +14 -0
- package/components/OptionsWatcher.ts +354 -0
- package/components/PluginModule.ts +6 -0
- package/components/Scope.ts +329 -0
- package/components/componentLoader.ts +529 -0
- package/components/deriveCommonPatternBase.ts +31 -0
- package/components/deriveGlobOptions.ts +44 -0
- package/components/deriveURLPath.ts +57 -0
- package/components/operations.js +658 -0
- package/components/operationsValidation.js +246 -0
- package/components/packageComponent.ts +39 -0
- package/components/requestRestart.ts +26 -0
- package/components/resolveBaseURLPath.ts +38 -0
- package/components/status/ComponentStatus.ts +110 -0
- package/components/status/ComponentStatusRegistry.ts +251 -0
- package/components/status/api.ts +153 -0
- package/components/status/crossThread.ts +405 -0
- package/components/status/errors.ts +152 -0
- package/components/status/index.ts +44 -0
- package/components/status/internal.ts +65 -0
- package/components/status/registry.ts +12 -0
- package/components/status/types.ts +96 -0
- package/config/RootConfigWatcher.ts +59 -0
- package/config/configHelpers.ts +11 -0
- package/config/configUtils.js +967 -0
- package/config/harperConfigEnvVars.ts +641 -0
- package/dataLayer/CreateAttributeObject.js +25 -0
- package/dataLayer/CreateTableObject.js +11 -0
- package/dataLayer/DataLayerObjects.js +43 -0
- package/dataLayer/DeleteBeforeObject.js +22 -0
- package/dataLayer/DeleteObject.js +25 -0
- package/dataLayer/DropAttributeObject.js +11 -0
- package/dataLayer/GetBackupObject.js +22 -0
- package/dataLayer/InsertObject.js +24 -0
- package/dataLayer/ReadAuditLogObject.js +24 -0
- package/dataLayer/SQLSearch.js +1335 -0
- package/dataLayer/SearchByConditionsObject.js +61 -0
- package/dataLayer/SearchByHashObject.js +21 -0
- package/dataLayer/SearchObject.js +45 -0
- package/dataLayer/SqlSearchObject.js +14 -0
- package/dataLayer/UpdateObject.js +23 -0
- package/dataLayer/UpsertObject.js +23 -0
- package/dataLayer/bulkLoad.js +813 -0
- package/dataLayer/dataObjects/BulkLoadObjects.js +27 -0
- package/dataLayer/dataObjects/UpsertObject.js +23 -0
- package/dataLayer/delete.js +164 -0
- package/dataLayer/export.js +381 -0
- package/dataLayer/getBackup.js +40 -0
- package/dataLayer/harperBridge/BridgeMethods.js +81 -0
- package/dataLayer/harperBridge/ResourceBridge.ts +633 -0
- package/dataLayer/harperBridge/bridgeUtility/insertUpdateReturnObj.js +28 -0
- package/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +88 -0
- package/dataLayer/harperBridge/harperBridge.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/LMDBBridge.js +119 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +19 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +112 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +67 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +31 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +94 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +98 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +89 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +109 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +107 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +137 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +35 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +111 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +28 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +29 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +207 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +156 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +30 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +19 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +64 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +70 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +22 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +23 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +22 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBTransactionObject.js +23 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +24 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +24 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +25 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +157 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +94 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +39 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +34 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +100 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +371 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +109 -0
- package/dataLayer/hdbInfoController.js +254 -0
- package/dataLayer/insert.js +266 -0
- package/dataLayer/readAuditLog.js +59 -0
- package/dataLayer/schema.js +366 -0
- package/dataLayer/schemaDescribe.js +289 -0
- package/dataLayer/search.js +60 -0
- package/dataLayer/transaction.js +17 -0
- package/dataLayer/update.js +124 -0
- package/dist/components/Logger.d.ts +12 -0
- package/dist/components/Logger.js +3 -0
- package/dist/components/Logger.js.map +1 -0
- package/dist/components/Scope.d.ts +14 -4
- package/dist/components/Scope.js +18 -10
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/componentLoader.js +16 -9
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/operations.js +2 -2
- package/dist/components/operations.js.map +1 -1
- package/dist/config/configUtils.d.ts +1 -1
- package/dist/config/configUtils.js +1 -1
- package/dist/config/configUtils.js.map +1 -1
- package/dist/dataLayer/CreateTableObject.d.ts +2 -2
- package/dist/dataLayer/CreateTableObject.js +2 -2
- package/dist/dataLayer/CreateTableObject.js.map +1 -1
- package/dist/dataLayer/delete.d.ts +1 -1
- package/dist/dataLayer/schema.js +6 -5
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +1 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/resources/DatabaseTransaction.d.ts +1 -1
- package/dist/resources/IterableEventQueue.d.ts +1 -1
- package/dist/resources/LMDBTransaction.d.ts +5 -1
- package/dist/resources/Resource.d.ts +1 -1
- package/dist/resources/RocksIndexStore.d.ts +3 -3
- package/dist/resources/RocksTransactionLogStore.d.ts +6 -3
- package/dist/resources/Table.d.ts +15 -6
- package/dist/resources/Table.js +4 -1
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/read.js +32 -22
- package/dist/resources/analytics/read.js.map +1 -1
- package/dist/resources/analytics/write.js +3 -6
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +3 -3
- package/dist/resources/blob.d.ts +25 -2
- package/dist/resources/databases.d.ts +12 -2
- package/dist/resources/databases.js +22 -19
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/search.js +11 -5
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transaction.d.ts +2 -1
- package/dist/security/auth.js +1 -1
- package/dist/security/auth.js.map +1 -1
- package/dist/security/cryptoHash.d.ts +2 -2
- package/dist/security/jsLoader.js +243 -66
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +4 -5
- package/dist/security/keys.js.map +1 -1
- package/dist/security/user.js +3 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/REST.js +16 -2
- package/dist/server/REST.js.map +1 -1
- package/dist/server/Server.d.ts +2 -1
- package/dist/server/Server.js.map +1 -1
- package/dist/server/fastifyRoutes/plugins/hdbCore.d.ts +6 -1
- package/dist/server/fastifyRoutes.js +2 -0
- package/dist/server/fastifyRoutes.js.map +1 -1
- package/dist/server/http.js +12 -6
- package/dist/server/http.js.map +1 -1
- package/dist/server/jobs/JobObject.d.ts +3 -3
- package/dist/server/loadRootComponents.js +1 -0
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/operationsServer.js +3 -1
- package/dist/server/operationsServer.js.map +1 -1
- package/dist/server/serverHelpers/JSONStream.d.ts +3 -3
- package/dist/server/serverHelpers/Request.d.ts +5 -5
- package/dist/server/serverHelpers/requestTimePlugin.d.ts +1 -1
- package/dist/server/threads/manageThreads.d.ts +2 -2
- package/dist/server/threads/manageThreads.js +50 -35
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.d.ts +1 -1
- package/dist/sqlTranslator/deleteTranslator.d.ts +1 -1
- package/dist/utility/AWS/AWSConnector.d.ts +3 -2
- package/dist/utility/common_utils.d.ts +3 -3
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/functions/date/dateFunctions.d.ts +11 -11
- package/dist/utility/globalSchema.d.ts +1 -1
- package/dist/utility/hdbTerms.d.ts +3 -0
- package/dist/utility/hdbTerms.js +3 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/installation.d.ts +2 -4
- package/dist/utility/installation.js.map +1 -1
- package/dist/utility/lmdb/commonUtility.d.ts +1 -0
- package/dist/utility/lmdb/deleteUtility.d.ts +1 -0
- package/dist/utility/lmdb/environmentUtility.d.ts +1 -0
- package/dist/utility/lmdb/searchUtility.d.ts +2 -1
- package/dist/utility/lmdb/writeUtility.d.ts +1 -0
- package/dist/utility/logging/harper_logger.d.ts +6 -6
- package/dist/utility/processManagement/processManagement.d.ts +1 -1
- package/dist/utility/processManagement/servicesConfig.d.ts +12 -6
- package/dist/validation/common_validators.d.ts +4 -3
- package/dist/validation/configValidator.d.ts +3 -2
- package/index.d.ts +56 -0
- package/index.js +41 -0
- package/json/systemSchema.json +373 -0
- package/launchServiceScripts/launchHarperDB.js +3 -0
- package/launchServiceScripts/utility/checkNodeVersion.js +15 -0
- package/package.json +21 -3
- package/resources/DatabaseTransaction.ts +378 -0
- package/resources/ErrorResource.ts +57 -0
- package/resources/IterableEventQueue.ts +94 -0
- package/resources/LMDBTransaction.ts +349 -0
- package/resources/RecordEncoder.ts +702 -0
- package/resources/RequestTarget.ts +134 -0
- package/resources/Resource.ts +789 -0
- package/resources/ResourceInterface.ts +221 -0
- package/resources/ResourceInterfaceV2.ts +53 -0
- package/resources/ResourceV2.ts +67 -0
- package/resources/Resources.ts +162 -0
- package/resources/RocksIndexStore.ts +70 -0
- package/resources/RocksTransactionLogStore.ts +352 -0
- package/resources/Table.ts +4527 -0
- package/resources/analytics/hostnames.ts +72 -0
- package/resources/analytics/metadata.ts +10 -0
- package/resources/analytics/read.ts +252 -0
- package/resources/analytics/write.ts +803 -0
- package/resources/auditStore.ts +556 -0
- package/resources/blob.ts +1268 -0
- package/resources/crdt.ts +125 -0
- package/resources/dataLoader.ts +527 -0
- package/resources/databases.ts +1290 -0
- package/resources/graphql.ts +221 -0
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +638 -0
- package/resources/indexes/customIndexes.ts +7 -0
- package/resources/indexes/vector.ts +38 -0
- package/resources/jsResource.ts +86 -0
- package/resources/loadEnv.ts +22 -0
- package/resources/login.ts +18 -0
- package/resources/openApi.ts +409 -0
- package/resources/registrationDeprecated.ts +8 -0
- package/resources/replayLogs.ts +136 -0
- package/resources/roles.ts +98 -0
- package/resources/search.ts +1301 -0
- package/resources/tracked.ts +584 -0
- package/resources/transaction.ts +89 -0
- package/resources/transactionBroadcast.ts +258 -0
- package/security/auth.ts +376 -0
- package/security/certificateVerification/certificateVerificationSource.ts +84 -0
- package/security/certificateVerification/configValidation.ts +107 -0
- package/security/certificateVerification/crlVerification.ts +623 -0
- package/security/certificateVerification/index.ts +121 -0
- package/security/certificateVerification/ocspVerification.ts +148 -0
- package/security/certificateVerification/pkijs-ed25519-patch.ts +188 -0
- package/security/certificateVerification/types.ts +128 -0
- package/security/certificateVerification/verificationConfig.ts +138 -0
- package/security/certificateVerification/verificationUtils.ts +447 -0
- package/security/cryptoHash.js +42 -0
- package/security/data_objects/PermissionAttributeResponseObject.js +15 -0
- package/security/data_objects/PermissionResponseObject.js +115 -0
- package/security/data_objects/PermissionTableResponseObject.js +20 -0
- package/security/fastifyAuth.js +169 -0
- package/security/impersonation.ts +160 -0
- package/security/jsLoader.ts +716 -0
- package/security/keys.js +948 -0
- package/security/permissionsTranslator.js +300 -0
- package/security/role.js +218 -0
- package/security/tokenAuthentication.ts +228 -0
- package/security/user.ts +449 -0
- package/server/DurableSubscriptionsSession.ts +503 -0
- package/server/REST.ts +407 -0
- package/server/Server.ts +89 -0
- package/server/fastifyRoutes/helpers/getCORSOptions.js +36 -0
- package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +15 -0
- package/server/fastifyRoutes/helpers/getServerOptions.js +33 -0
- package/server/fastifyRoutes/plugins/hdbCore.js +39 -0
- package/server/fastifyRoutes.ts +205 -0
- package/server/graphqlQuerying.ts +700 -0
- package/server/http.ts +640 -0
- package/server/itc/serverHandlers.js +161 -0
- package/server/itc/utility/ITCEventObject.js +10 -0
- package/server/jobs/JobObject.js +24 -0
- package/server/jobs/jobProcess.js +69 -0
- package/server/jobs/jobRunner.js +162 -0
- package/server/jobs/jobs.js +304 -0
- package/server/loadRootComponents.js +44 -0
- package/server/mqtt.ts +485 -0
- package/server/nodeName.ts +75 -0
- package/server/operationsServer.ts +313 -0
- package/server/serverHelpers/Headers.ts +108 -0
- package/server/serverHelpers/JSONStream.ts +269 -0
- package/server/serverHelpers/OperationFunctionObject.ts +13 -0
- package/server/serverHelpers/Request.ts +158 -0
- package/server/serverHelpers/contentTypes.ts +637 -0
- package/server/serverHelpers/requestTimePlugin.js +57 -0
- package/server/serverHelpers/serverHandlers.js +148 -0
- package/server/serverHelpers/serverUtilities.ts +473 -0
- package/server/serverRegistry.ts +8 -0
- package/server/static.ts +187 -0
- package/server/status/definitions.ts +37 -0
- package/server/status/index.ts +125 -0
- package/server/storageReclamation.ts +93 -0
- package/server/threads/itc.js +89 -0
- package/server/threads/manageThreads.js +594 -0
- package/server/threads/socketRouter.ts +360 -0
- package/server/threads/threadServer.js +279 -0
- package/server/throttle.ts +73 -0
- package/sqlTranslator/SelectValidator.js +330 -0
- package/sqlTranslator/alasqlFunctionImporter.js +62 -0
- package/sqlTranslator/deleteTranslator.js +67 -0
- package/sqlTranslator/index.js +242 -0
- package/sqlTranslator/sql_statement_bucket.js +472 -0
- package/static/defaultConfig.yaml +3 -0
- package/studio/web/HDBDogOnly.svg +78 -0
- package/studio/web/assets/PPRadioGrotesk-Bold-DDaUYG8E.woff +0 -0
- package/studio/web/assets/fa-brands-400-CEJbCg16.woff +0 -0
- package/studio/web/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
- package/studio/web/assets/fa-brands-400-DnkPfk3o.eot +0 -0
- package/studio/web/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
- package/studio/web/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
- package/studio/web/assets/fa-regular-400-BhTwtT8w.eot +0 -0
- package/studio/web/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
- package/studio/web/assets/fa-regular-400-DFnMcJPd.woff +0 -0
- package/studio/web/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
- package/studio/web/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
- package/studio/web/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
- package/studio/web/assets/fa-solid-900-B6Axprfb.eot +0 -0
- package/studio/web/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
- package/studio/web/assets/fa-solid-900-DOXgCApm.woff +0 -0
- package/studio/web/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
- package/studio/web/assets/index-BTgXJX9d.js +235 -0
- package/studio/web/assets/index-BTgXJX9d.js.map +1 -0
- package/studio/web/assets/index-C-GXfcup.js +37 -0
- package/studio/web/assets/index-C-GXfcup.js.map +1 -0
- package/studio/web/assets/index-PFlNdimM.js +2 -0
- package/studio/web/assets/index-PFlNdimM.js.map +1 -0
- package/studio/web/assets/index-Y2g_iFpU.css +1 -0
- package/studio/web/assets/index-jiPwkrsB.css +1 -0
- package/studio/web/assets/index.lazy-C3TJZJ4o.js +266 -0
- package/studio/web/assets/index.lazy-C3TJZJ4o.js.map +1 -0
- package/studio/web/assets/profiler-DotzgiCJ.js +2 -0
- package/studio/web/assets/profiler-DotzgiCJ.js.map +1 -0
- package/studio/web/assets/react-redux-VxUEx_mU.js +6 -0
- package/studio/web/assets/react-redux-VxUEx_mU.js.map +1 -0
- package/studio/web/assets/startRecording-B_9J9Csd.js +3 -0
- package/studio/web/assets/startRecording-B_9J9Csd.js.map +1 -0
- package/studio/web/fabric-signup-background.webp +0 -0
- package/studio/web/fabric-signup-text.png +0 -0
- package/studio/web/favicon_purple.png +0 -0
- package/studio/web/github-icon.svg +15 -0
- package/studio/web/harper-fabric_black.png +0 -0
- package/studio/web/harper-fabric_white.png +0 -0
- package/studio/web/harper-studio_white.png +0 -0
- package/studio/web/index.html +16 -0
- package/studio/web/running.css +148 -0
- package/studio/web/running.html +147 -0
- package/studio/web/running.js +111 -0
- package/upgrade/UpgradeObjects.js +13 -0
- package/upgrade/directives/directivesController.js +90 -0
- package/upgrade/directivesManager.js +139 -0
- package/upgrade/upgradePrompt.js +124 -0
- package/upgrade/upgradeUtilities.js +28 -0
- package/utility/AWS/AWSConnector.js +29 -0
- package/utility/OperationFunctionCaller.js +63 -0
- package/utility/assignCmdEnvVariables.js +62 -0
- package/utility/common_utils.js +867 -0
- package/utility/environment/environmentManager.js +208 -0
- package/utility/environment/systemInformation.js +355 -0
- package/utility/errors/commonErrors.js +267 -0
- package/utility/errors/hdbError.js +146 -0
- package/utility/functions/date/dateFunctions.js +65 -0
- package/utility/functions/geo.js +355 -0
- package/utility/functions/sql/alaSQLExtension.js +104 -0
- package/utility/globalSchema.js +35 -0
- package/utility/hdbTerms.ts +819 -0
- package/utility/install/checkJWTTokensExist.js +62 -0
- package/utility/install/harperdb.conf +15 -0
- package/utility/install/harperdb.service +14 -0
- package/utility/install/installer.js +635 -0
- package/utility/installation.ts +30 -0
- package/utility/lmdb/DBIDefinition.js +20 -0
- package/utility/lmdb/DeleteRecordsResponseObject.js +25 -0
- package/utility/lmdb/InsertRecordsResponseObject.js +22 -0
- package/utility/lmdb/OpenDBIObject.js +31 -0
- package/utility/lmdb/OpenEnvironmentObject.js +41 -0
- package/utility/lmdb/UpdateRecordsResponseObject.js +25 -0
- package/utility/lmdb/UpsertRecordsResponseObject.js +22 -0
- package/utility/lmdb/cleanLMDBMap.js +65 -0
- package/utility/lmdb/commonUtility.js +119 -0
- package/utility/lmdb/deleteUtility.js +128 -0
- package/utility/lmdb/environmentUtility.js +477 -0
- package/utility/lmdb/searchCursorFunctions.js +187 -0
- package/utility/lmdb/searchUtility.js +918 -0
- package/utility/lmdb/terms.js +57 -0
- package/utility/lmdb/writeUtility.js +407 -0
- package/utility/logging/harper_logger.js +876 -0
- package/utility/logging/logRotator.js +157 -0
- package/utility/logging/logger.ts +24 -0
- package/utility/logging/readLog.js +355 -0
- package/utility/logging/transactionLog.js +57 -0
- package/utility/mount_hdb.js +59 -0
- package/utility/npmUtilities.js +102 -0
- package/utility/operationPermissions.ts +112 -0
- package/utility/operation_authorization.js +836 -0
- package/utility/packageUtils.js +55 -0
- package/utility/password.ts +99 -0
- package/utility/processManagement/processManagement.js +187 -0
- package/utility/processManagement/servicesConfig.js +56 -0
- package/utility/scripts/restartHdb.js +24 -0
- package/utility/scripts/user_data.sh +13 -0
- package/utility/signalling.js +36 -0
- package/utility/terms/certificates.js +81 -0
- package/utility/when.ts +20 -0
- package/v1.d.ts +39 -0
- package/v1.js +41 -0
- package/v2.d.ts +39 -0
- package/v2.js +41 -0
- package/validation/bulkDeleteValidator.js +24 -0
- package/validation/check_permissions.js +19 -0
- package/validation/common_validators.js +95 -0
- package/validation/configValidator.js +331 -0
- package/validation/deleteValidator.js +15 -0
- package/validation/fileLoadValidator.js +153 -0
- package/validation/insertValidator.js +40 -0
- package/validation/installValidator.js +37 -0
- package/validation/readLogValidator.js +64 -0
- package/validation/role_validation.js +320 -0
- package/validation/schemaMetadataValidator.js +42 -0
- package/validation/searchValidator.js +166 -0
- package/validation/statusValidator.ts +66 -0
- package/validation/transactionLogValidator.js +33 -0
- package/validation/user_validation.js +55 -0
- package/validation/validationWrapper.js +105 -0
- package/dist/resources/analytics/profile.d.ts +0 -2
- package/dist/resources/analytics/profile.js +0 -144
- package/dist/resources/analytics/profile.js.map +0 -1
|
@@ -0,0 +1,637 @@
|
|
|
1
|
+
import { streamAsJSON, stringify, parse } from './JSONStream.ts';
|
|
2
|
+
import { pack, unpack, encodeIter } from 'msgpackr';
|
|
3
|
+
import { decode, Encoder, EncoderStream } from 'cbor-x';
|
|
4
|
+
import { createBrotliCompress, brotliCompress, constants } from 'zlib';
|
|
5
|
+
import { ClientError } from '../../utility/errors/hdbError.js';
|
|
6
|
+
import stream, { Readable } from 'stream';
|
|
7
|
+
import { server } from '../Server.ts';
|
|
8
|
+
import { _assignPackageExport } from '../../globals.js';
|
|
9
|
+
import envMgr from '../../utility/environment/environmentManager.js';
|
|
10
|
+
import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts';
|
|
11
|
+
import * as YAML from 'yaml';
|
|
12
|
+
import { logger } from '../../utility/logging/logger.ts';
|
|
13
|
+
import { Blob } from '../../resources/blob.ts';
|
|
14
|
+
import { Transform } from 'json2csv';
|
|
15
|
+
// TODO: Only load this if fastify is loaded
|
|
16
|
+
import fp from 'fastify-plugin';
|
|
17
|
+
const SERIALIZATION_BIGINT = envMgr.get(CONFIG_PARAMS.SERIALIZATION_BIGINT) !== false;
|
|
18
|
+
const JSONStringify = SERIALIZATION_BIGINT ? stringify : JSON.stringify;
|
|
19
|
+
const JSONParse = SERIALIZATION_BIGINT ? parse : JSON.parse;
|
|
20
|
+
|
|
21
|
+
const PUBLIC_ENCODE_OPTIONS = {
|
|
22
|
+
useRecords: false,
|
|
23
|
+
useToJSON: true,
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
type Deserialize = (data: Buffer) => { contentType?: string; data: unknown } | unknown;
|
|
27
|
+
|
|
28
|
+
const mediaTypes = new Map<
|
|
29
|
+
string,
|
|
30
|
+
{
|
|
31
|
+
serialize?: unknown;
|
|
32
|
+
deserialize?: Deserialize;
|
|
33
|
+
serializeStream?: unknown;
|
|
34
|
+
compressible?: boolean;
|
|
35
|
+
q?: number;
|
|
36
|
+
}
|
|
37
|
+
>();
|
|
38
|
+
|
|
39
|
+
export const contentTypes = mediaTypes;
|
|
40
|
+
server.contentTypes = contentTypes;
|
|
41
|
+
_assignPackageExport('contentTypes', contentTypes);
|
|
42
|
+
// TODO: Make these monomorphic for faster access. And use a Map
|
|
43
|
+
mediaTypes.set('application/json', {
|
|
44
|
+
serializeStream: streamAsJSON,
|
|
45
|
+
serialize: JSONStringify,
|
|
46
|
+
deserialize(data) {
|
|
47
|
+
return JSONParse(data);
|
|
48
|
+
},
|
|
49
|
+
q: 0.8,
|
|
50
|
+
});
|
|
51
|
+
const cborEncoder = new Encoder(PUBLIC_ENCODE_OPTIONS);
|
|
52
|
+
mediaTypes.set('application/cbor', {
|
|
53
|
+
serializeStream(data) {
|
|
54
|
+
if (data[Symbol.asyncIterator]) data[Symbol.iterator] = null; // choose async iteration if possible
|
|
55
|
+
return new EncoderStream(PUBLIC_ENCODE_OPTIONS).end(data);
|
|
56
|
+
},
|
|
57
|
+
serialize: cborEncoder.encode,
|
|
58
|
+
deserialize: cborEncoder.decode,
|
|
59
|
+
q: 1,
|
|
60
|
+
});
|
|
61
|
+
mediaTypes.set('application/x-msgpack', {
|
|
62
|
+
serializeStream(data: any) {
|
|
63
|
+
if ((data?.[Symbol.iterator] || data?.[Symbol.asyncIterator]) && !Array.isArray(data)) {
|
|
64
|
+
return Readable.from(encodeIter(data, PUBLIC_ENCODE_OPTIONS));
|
|
65
|
+
}
|
|
66
|
+
return pack(data);
|
|
67
|
+
},
|
|
68
|
+
serialize: pack,
|
|
69
|
+
deserialize: unpack,
|
|
70
|
+
q: 0.9,
|
|
71
|
+
});
|
|
72
|
+
mediaTypes.set('text/csv', {
|
|
73
|
+
serializeStream(data: any, response: Response) {
|
|
74
|
+
response.headers.set('Content-Disposition', 'attachment; filename="data.csv"');
|
|
75
|
+
return toCsvStream(data, data?.getColumns?.());
|
|
76
|
+
},
|
|
77
|
+
serialize(data: any, response: Response) {
|
|
78
|
+
response.headers.set('Content-Disposition', 'attachment; filename="data.csv"');
|
|
79
|
+
if (data && !data[Symbol.iterator]) data = [data.toJSON ? data.toJSON() : data];
|
|
80
|
+
return toCsvStream(data, data?.getColumns?.());
|
|
81
|
+
},
|
|
82
|
+
q: 0.1,
|
|
83
|
+
});
|
|
84
|
+
mediaTypes.set('text/plain', {
|
|
85
|
+
serialize(data: any) {
|
|
86
|
+
return data.toString();
|
|
87
|
+
},
|
|
88
|
+
serializeStream(data: any) {
|
|
89
|
+
return Readable.from(data.map ? data.map((d) => d.toString()) : data);
|
|
90
|
+
},
|
|
91
|
+
deserialize(data: Buffer) {
|
|
92
|
+
return data.toString();
|
|
93
|
+
},
|
|
94
|
+
q: 0.2,
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
mediaTypes.set('text/yaml', {
|
|
98
|
+
serialize(data) {
|
|
99
|
+
return YAML.stringify(data, { aliasDuplicateObjects: false });
|
|
100
|
+
},
|
|
101
|
+
|
|
102
|
+
q: 0.7,
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
mediaTypes.set('text/event-stream', {
|
|
106
|
+
// Server-Sent Events (SSE)
|
|
107
|
+
serializeStream: function (iterable) {
|
|
108
|
+
// create a readable stream that we use to stream out events from our subscription
|
|
109
|
+
return Readable.from(transformIterable(iterable, this.serialize));
|
|
110
|
+
},
|
|
111
|
+
serialize: function (message) {
|
|
112
|
+
if (message.acknowledge) message.acknowledge();
|
|
113
|
+
if (typeof message === 'object' && 'value' in message && message.timestamp) {
|
|
114
|
+
// native messages
|
|
115
|
+
message = {
|
|
116
|
+
data: message.value,
|
|
117
|
+
event: message.type,
|
|
118
|
+
id: message.timestamp,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
if (message.data || message.event) {
|
|
122
|
+
let serialized = '';
|
|
123
|
+
if (message.event) serialized += 'event: ' + message.event + '\n';
|
|
124
|
+
if (message.data) {
|
|
125
|
+
let data = message.data;
|
|
126
|
+
if (typeof data === 'object') data = JSONStringify(data);
|
|
127
|
+
serialized += 'data: ' + data + '\n';
|
|
128
|
+
}
|
|
129
|
+
if (message.id) serialized += 'id: ' + message.id + '\n';
|
|
130
|
+
if (message.retry) serialized += 'retry: ' + message.retry + '\n';
|
|
131
|
+
return serialized + '\n';
|
|
132
|
+
} else {
|
|
133
|
+
if (typeof message === 'object') return `data: ${JSONStringify(message)}\n\n`;
|
|
134
|
+
return `data: ${message}\n\n`;
|
|
135
|
+
}
|
|
136
|
+
},
|
|
137
|
+
compressible: false,
|
|
138
|
+
q: 0.8,
|
|
139
|
+
});
|
|
140
|
+
// TODO: Support this as well:
|
|
141
|
+
//'multipart/form-data'
|
|
142
|
+
mediaTypes.set('application/x-www-form-urlencoded', {
|
|
143
|
+
deserialize(data) {
|
|
144
|
+
const stringData = Buffer.isBuffer(data) ? data.toString('utf8') : data;
|
|
145
|
+
const object: Record<string, string | string[]> = {};
|
|
146
|
+
for (const [key, value] of new URLSearchParams(stringData)) {
|
|
147
|
+
if (object.hasOwnProperty(key)) {
|
|
148
|
+
// in case there are multiple query params with the same name, convert them to an array
|
|
149
|
+
const last = object[key];
|
|
150
|
+
if (Array.isArray(last)) last.push(value);
|
|
151
|
+
else object.key = [last, value];
|
|
152
|
+
} else object[key] = value;
|
|
153
|
+
}
|
|
154
|
+
return object;
|
|
155
|
+
},
|
|
156
|
+
serialize(data) {
|
|
157
|
+
const usp = new URLSearchParams();
|
|
158
|
+
for (const key in data) {
|
|
159
|
+
usp.set(key, data);
|
|
160
|
+
}
|
|
161
|
+
return usp.toString();
|
|
162
|
+
},
|
|
163
|
+
});
|
|
164
|
+
const genericHandler = {
|
|
165
|
+
type: 'application/json',
|
|
166
|
+
serializeStream: streamAsJSON,
|
|
167
|
+
serialize: JSONStringify,
|
|
168
|
+
deserialize: tryJSONParse,
|
|
169
|
+
q: 0.5,
|
|
170
|
+
};
|
|
171
|
+
mediaTypes.set('*/*', genericHandler);
|
|
172
|
+
mediaTypes.set('', genericHandler);
|
|
173
|
+
// try to JSON parse, but since we don't know for sure, this will return the body
|
|
174
|
+
// otherwise
|
|
175
|
+
function tryJSONParse(input) {
|
|
176
|
+
try {
|
|
177
|
+
if (input?.[0] === 123) return JSONParse(input);
|
|
178
|
+
else return input;
|
|
179
|
+
} catch {
|
|
180
|
+
return input;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
export function registerContentHandlers(app) {
|
|
184
|
+
app.register(registerFastifySerializers, {
|
|
185
|
+
serializers: [
|
|
186
|
+
{
|
|
187
|
+
regex: /^application\/json$/,
|
|
188
|
+
serializer: streamAsJSON,
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
regex: /^application\/cbor$/,
|
|
192
|
+
serializer: function (data) {
|
|
193
|
+
return new EncoderStream(PUBLIC_ENCODE_OPTIONS).end(data);
|
|
194
|
+
},
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
regex: /^application\/(x-)?msgpack$/,
|
|
198
|
+
serializer: function (data) {
|
|
199
|
+
if ((data?.[Symbol.iterator] || data?.[Symbol.asyncIterator]) && !Array.isArray(data)) {
|
|
200
|
+
return Readable.from(encodeIter(data, PUBLIC_ENCODE_OPTIONS));
|
|
201
|
+
}
|
|
202
|
+
return pack(data);
|
|
203
|
+
},
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
regex: /^text\/csv$/,
|
|
207
|
+
serializer: function (data) {
|
|
208
|
+
this.header('Content-Disposition', 'attachment; filename="data.csv"');
|
|
209
|
+
return toCsvStream(data);
|
|
210
|
+
},
|
|
211
|
+
},
|
|
212
|
+
],
|
|
213
|
+
});
|
|
214
|
+
app.addContentTypeParser('application/x-msgpack', { parseAs: 'buffer' }, (req, body, done) => {
|
|
215
|
+
try {
|
|
216
|
+
done(null, unpack(body));
|
|
217
|
+
} catch (error) {
|
|
218
|
+
error.statusCode = 400;
|
|
219
|
+
done(error);
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
app.addContentTypeParser('application/cbor', { parseAs: 'buffer' }, (req, body, done) => {
|
|
224
|
+
try {
|
|
225
|
+
done(null, decode(body));
|
|
226
|
+
} catch (error) {
|
|
227
|
+
error.statusCode = 400;
|
|
228
|
+
done(error);
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const registerFastifySerializers = fp(
|
|
234
|
+
function (fastify, opts, done) {
|
|
235
|
+
// eslint-disable-next-line require-await
|
|
236
|
+
fastify.addHook('preSerialization', async (request, reply) => {
|
|
237
|
+
const contentType = reply.raw.getHeader('content-type');
|
|
238
|
+
if (contentType) return;
|
|
239
|
+
const { serializer, type } = findBestSerializer(request.raw);
|
|
240
|
+
reply.type(type);
|
|
241
|
+
reply.serializer(function (data: any) {
|
|
242
|
+
let serialize: (data: any, context: any) => any;
|
|
243
|
+
if (
|
|
244
|
+
typeof data === 'object' &&
|
|
245
|
+
data &&
|
|
246
|
+
(data[Symbol.iterator] || data[Symbol.asyncIterator]) &&
|
|
247
|
+
serializer.serializeStream
|
|
248
|
+
) {
|
|
249
|
+
if (data.mapError) {
|
|
250
|
+
// indicate that we want iterator errors to be returned so we can serialize them in a meaningful way, if possible
|
|
251
|
+
const getColumns = data.getColumns;
|
|
252
|
+
data = data.mapError((error) => {
|
|
253
|
+
// make errors serializable in a descriptive way
|
|
254
|
+
error.toJSON = () => ({ error: error.name, message: error.message, ...error.partialObject });
|
|
255
|
+
return error;
|
|
256
|
+
});
|
|
257
|
+
data.getColumns = getColumns;
|
|
258
|
+
}
|
|
259
|
+
serialize = serializer.serializeStream;
|
|
260
|
+
} else serialize = serializer.serialize;
|
|
261
|
+
|
|
262
|
+
return serialize(data, {
|
|
263
|
+
// a small header shim to allow us to set headers in serializers
|
|
264
|
+
headers: {
|
|
265
|
+
set: (key, value) => {
|
|
266
|
+
reply.header(key, value);
|
|
267
|
+
},
|
|
268
|
+
},
|
|
269
|
+
});
|
|
270
|
+
});
|
|
271
|
+
});
|
|
272
|
+
done();
|
|
273
|
+
},
|
|
274
|
+
{ name: 'content-type-negotiation' }
|
|
275
|
+
);
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* This is returns the best serializer for the request's Accept header (content negotiation)
|
|
279
|
+
* @param incomingMessage
|
|
280
|
+
* @returns {{serializer, type: string, parameters: {q: number}}|{serializer(): void}}
|
|
281
|
+
*/
|
|
282
|
+
export function findBestSerializer(incomingMessage) {
|
|
283
|
+
const headersObject = incomingMessage.headers.asObject || incomingMessage.headers;
|
|
284
|
+
const acceptType = incomingMessage.requestedContentType ?? headersObject.accept;
|
|
285
|
+
let bestSerializer;
|
|
286
|
+
let bestQuality = 0;
|
|
287
|
+
let bestType;
|
|
288
|
+
let bestParameters;
|
|
289
|
+
const acceptTypes = acceptType ? acceptType.toLowerCase().split(/\s*,\s*/) : [];
|
|
290
|
+
for (const acceptType of acceptTypes) {
|
|
291
|
+
const [type, ...parameterParts] = acceptType.split(/\s*;\s*/);
|
|
292
|
+
let clientQuality = 1;
|
|
293
|
+
const parameters = { q: 1 };
|
|
294
|
+
for (const part of parameterParts) {
|
|
295
|
+
const equalIndex = part.indexOf('=');
|
|
296
|
+
parameters[part.substring(0, equalIndex)] = part.substring(equalIndex + 1);
|
|
297
|
+
}
|
|
298
|
+
clientQuality = +parameters.q;
|
|
299
|
+
const serializer = mediaTypes.get(type);
|
|
300
|
+
if (serializer) {
|
|
301
|
+
const quality = (serializer.q || 1) * clientQuality;
|
|
302
|
+
if (quality > bestQuality) {
|
|
303
|
+
bestSerializer = serializer;
|
|
304
|
+
bestType = serializer.type || type;
|
|
305
|
+
bestQuality = quality;
|
|
306
|
+
bestParameters = parameters;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
if (!bestSerializer) {
|
|
311
|
+
if (acceptType) {
|
|
312
|
+
throw new ClientError(
|
|
313
|
+
'No supported content types found in Accept header, supported types include: ' +
|
|
314
|
+
Array.from(mediaTypes.keys()).join(', '),
|
|
315
|
+
406
|
|
316
|
+
);
|
|
317
|
+
} else {
|
|
318
|
+
// default if Accept header is absent
|
|
319
|
+
bestSerializer = mediaTypes.get('application/json');
|
|
320
|
+
bestType = 'application/json';
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
return { serializer: bestSerializer, type: bestType, parameters: bestParameters };
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// about an average TCP packet size (if headers included)
|
|
328
|
+
const COMPRESSION_THRESHOLD = envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD);
|
|
329
|
+
/**
|
|
330
|
+
* Serialize a response
|
|
331
|
+
* @param responseData
|
|
332
|
+
* @param request
|
|
333
|
+
* @param responseObject
|
|
334
|
+
* @returns {Uint8Array|*}
|
|
335
|
+
*/
|
|
336
|
+
export function serialize(responseData, request, responseObject) {
|
|
337
|
+
// TODO: Maybe support other compression encodings; browsers basically universally support brotli, but Node's HTTP
|
|
338
|
+
// client itself actually (just) supports gzip/deflate
|
|
339
|
+
let canCompress = COMPRESSION_THRESHOLD && request.headers.asObject?.['accept-encoding']?.includes('br');
|
|
340
|
+
let responseBody;
|
|
341
|
+
if (responseData?.contentType != null && responseData.data != null) {
|
|
342
|
+
// we use this as a special marker for blobs of data that are explicitly one content type
|
|
343
|
+
responseObject.headers.set('Content-Type', responseData.contentType);
|
|
344
|
+
responseObject.headers.set('Vary', 'Accept-Encoding');
|
|
345
|
+
responseBody = responseData.data;
|
|
346
|
+
} else if (responseData instanceof Uint8Array || responseData instanceof Blob) {
|
|
347
|
+
// If a user function or property returns a direct Buffer of binary data, this is the most appropriate content
|
|
348
|
+
// type for it.
|
|
349
|
+
responseObject.headers.set('Content-Type', 'application/octet-stream');
|
|
350
|
+
responseObject.headers.set('Vary', 'Accept-Encoding');
|
|
351
|
+
responseBody = responseData;
|
|
352
|
+
} else {
|
|
353
|
+
const serializer = findBestSerializer(request);
|
|
354
|
+
if (serializer.serializer.compressible === false) canCompress = false;
|
|
355
|
+
// TODO: If a different content type is preferred, look through resources to see if there is one
|
|
356
|
+
// specifically for that content type (most useful for html).
|
|
357
|
+
responseObject.headers.set('Vary', 'Accept, Accept-Encoding');
|
|
358
|
+
responseObject.headers.set('Content-Type', serializer.type);
|
|
359
|
+
if (
|
|
360
|
+
typeof responseData === 'object' &&
|
|
361
|
+
responseData &&
|
|
362
|
+
(responseData[Symbol.iterator] || responseData[Symbol.asyncIterator]) &&
|
|
363
|
+
serializer.serializer.serializeStream
|
|
364
|
+
) {
|
|
365
|
+
if (responseData.mapError) {
|
|
366
|
+
// indicate that we want iterator errors to be returned so we can serialize them in a meaningful way, if possible
|
|
367
|
+
const getColumns = responseData.getColumns;
|
|
368
|
+
responseData = responseData.mapError((error) => {
|
|
369
|
+
// make errors serializable in a descriptive way
|
|
370
|
+
error.toJSON = () => ({ error: error.name, message: error.message, ...error.partialObject });
|
|
371
|
+
logger.warn?.(`Error serializing error ${request?.url || request}: ${error}`);
|
|
372
|
+
return error;
|
|
373
|
+
});
|
|
374
|
+
responseData.getColumns = getColumns;
|
|
375
|
+
}
|
|
376
|
+
let stream = serializer.serializer.serializeStream(responseData, responseObject);
|
|
377
|
+
if (canCompress) {
|
|
378
|
+
responseObject.headers.set('Content-Encoding', 'br');
|
|
379
|
+
stream = stream.pipe(
|
|
380
|
+
createBrotliCompress({
|
|
381
|
+
params: {
|
|
382
|
+
[constants.BROTLI_PARAM_MODE]:
|
|
383
|
+
serializer.type.includes('json') || serializer.type.includes('text')
|
|
384
|
+
? constants.BROTLI_MODE_TEXT
|
|
385
|
+
: constants.BROTLI_MODE_GENERIC,
|
|
386
|
+
[constants.BROTLI_PARAM_QUALITY]: 2, // go fast
|
|
387
|
+
},
|
|
388
|
+
})
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
return stream;
|
|
392
|
+
}
|
|
393
|
+
responseBody = serializer.serializer.serialize(responseData, responseObject);
|
|
394
|
+
}
|
|
395
|
+
if (canCompress && responseBody?.length > COMPRESSION_THRESHOLD) {
|
|
396
|
+
// TODO: Only do this if the size is large and we can cache the result (otherwise use logic above)
|
|
397
|
+
responseObject.headers.set('Content-Encoding', 'br');
|
|
398
|
+
// if we have a single buffer (or string) we compress in a single async call
|
|
399
|
+
return new Promise((resolve, reject) =>
|
|
400
|
+
brotliCompress(responseBody, (err, data) => {
|
|
401
|
+
if (err) reject(err);
|
|
402
|
+
else resolve(data);
|
|
403
|
+
})
|
|
404
|
+
);
|
|
405
|
+
}
|
|
406
|
+
return responseBody;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
let asyncSerializations: Promise<void>[];
|
|
410
|
+
/**
|
|
411
|
+
* Serialize a message, may be use multiple times (like with WebSockets)
|
|
412
|
+
* @param message
|
|
413
|
+
* @param request
|
|
414
|
+
* @returns {*}
|
|
415
|
+
*/
|
|
416
|
+
export function serializeMessage(
|
|
417
|
+
message: any,
|
|
418
|
+
request?: Request,
|
|
419
|
+
inAsyncContinuation?: boolean
|
|
420
|
+
): Buffer | string | Promise<Buffer | string> {
|
|
421
|
+
if (message?.contentType != null && message.data != null) return message.data;
|
|
422
|
+
asyncSerializations = inAsyncContinuation ? undefined : [];
|
|
423
|
+
try {
|
|
424
|
+
let serialized: Buffer | string;
|
|
425
|
+
if (request) {
|
|
426
|
+
let serialize = request.serialize;
|
|
427
|
+
if (serialize) serialized = serialize(message);
|
|
428
|
+
else {
|
|
429
|
+
const serializer = findBestSerializer(request);
|
|
430
|
+
serialize = request.serialize = serializer.serializer.serialize;
|
|
431
|
+
serialized = serialize(message);
|
|
432
|
+
}
|
|
433
|
+
} else {
|
|
434
|
+
serialized = JSONStringify(message);
|
|
435
|
+
}
|
|
436
|
+
if (asyncSerializations?.length > 0)
|
|
437
|
+
// if there were any serialization attempts that must wait for async work to be done, we wait now and then retry the serialization
|
|
438
|
+
return (asyncSerializations.length === 1 ? asyncSerializations[0] : Promise.all(asyncSerializations)).then(() =>
|
|
439
|
+
serializeMessage(message, request, true)
|
|
440
|
+
);
|
|
441
|
+
return serialized;
|
|
442
|
+
} finally {
|
|
443
|
+
asyncSerializations = undefined;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* This can be called during serialization indicating that an object requires asynchronous serialization (or async completion of a task prior to serialization) to be properly serialized.
|
|
449
|
+
* A promise for when the object is ready to be serialized. Typically serialization will be re-executed and this object should be ready to be synchronously serialized
|
|
450
|
+
* @param promiseToSerialize
|
|
451
|
+
*/
|
|
452
|
+
export function asyncSerialization(promiseToSerialize: Promise<any>) {
|
|
453
|
+
if (asyncSerializations) asyncSerializations.push(promiseToSerialize);
|
|
454
|
+
else throw new Error('Unable to serialize asynchronously');
|
|
455
|
+
}
|
|
456
|
+
export function hasAsyncSerialization() {
|
|
457
|
+
return !!asyncSerializations;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function streamToBuffer(stream: Readable): Promise<Buffer> {
|
|
461
|
+
const MAX_REQUEST_BODY_SIZE = envMgr.get(CONFIG_PARAMS.HTTP_MAXREQUESTBODYSIZE) ?? 10_000_000;
|
|
462
|
+
return new Promise((resolve, reject) => {
|
|
463
|
+
const buffers = [];
|
|
464
|
+
let size = 0;
|
|
465
|
+
stream.on('data', (data) => {
|
|
466
|
+
size += data.length;
|
|
467
|
+
if (size > MAX_REQUEST_BODY_SIZE) {
|
|
468
|
+
const error = new ClientError(`Request body too large, maximum size is ${MAX_REQUEST_BODY_SIZE} bytes`, 413);
|
|
469
|
+
buffers.length = 0; // free up memory
|
|
470
|
+
reject(error);
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
473
|
+
buffers.push(data);
|
|
474
|
+
});
|
|
475
|
+
stream.on('end', () => resolve(Buffer.concat(buffers)));
|
|
476
|
+
stream.on('error', reject);
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/**
|
|
481
|
+
* An object based representation of a content-type header string.
|
|
482
|
+
* The parameters `charset` and `boundary` have been added to the type as
|
|
483
|
+
* they are common parameters for the `Content-Type` header, but HTTP specifies
|
|
484
|
+
* that any parameter can be included (hence the `[k: string]: string`).
|
|
485
|
+
*
|
|
486
|
+
* Use `parseContentType(contentType: string)` to create this object.
|
|
487
|
+
*/
|
|
488
|
+
type ContentType = {
|
|
489
|
+
type: string;
|
|
490
|
+
parameters?: { charset?: string; boundary?: string; [k: string]: string };
|
|
491
|
+
};
|
|
492
|
+
|
|
493
|
+
const BUFFER_ENCODINGS = [
|
|
494
|
+
'ascii',
|
|
495
|
+
'utf8',
|
|
496
|
+
'utf-8',
|
|
497
|
+
'utf16le',
|
|
498
|
+
'utf-16le',
|
|
499
|
+
'ucs2',
|
|
500
|
+
'ucs-2',
|
|
501
|
+
'base64',
|
|
502
|
+
'base64url',
|
|
503
|
+
'latin1',
|
|
504
|
+
'binary',
|
|
505
|
+
'hex',
|
|
506
|
+
];
|
|
507
|
+
function isBufferEncoding(value: string): value is NodeJS.BufferEncoding {
|
|
508
|
+
return BUFFER_ENCODINGS.includes(value);
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Parse the content-type header for the type and parameters.
|
|
513
|
+
* @param contentType
|
|
514
|
+
*/
|
|
515
|
+
function parseContentType(contentType: string): ContentType {
|
|
516
|
+
// Get the first `;` character to separate the type from the parameters
|
|
517
|
+
const parametersStart = contentType.indexOf(';');
|
|
518
|
+
let parameters: ContentType['parameters'];
|
|
519
|
+
|
|
520
|
+
// If the `;` exists, then parse the parameters
|
|
521
|
+
if (parametersStart > -1) {
|
|
522
|
+
parameters = {};
|
|
523
|
+
// Parameters are separated by `;` and key-value pairs are separated by `=`
|
|
524
|
+
// i.e. `multipart/form-data; charset=UTF-8; boundary=---123`
|
|
525
|
+
const parts = contentType.slice(parametersStart + 1).split(';');
|
|
526
|
+
for (const part of parts) {
|
|
527
|
+
const [key, value] = part.split('=');
|
|
528
|
+
parameters[key.trim()] = value.trim();
|
|
529
|
+
}
|
|
530
|
+
contentType = contentType.slice(0, parametersStart);
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
return { type: contentType, parameters };
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
/**
|
|
537
|
+
* Given a content-type header string, get a deserializer function that can be used to parse the body.
|
|
538
|
+
*/
|
|
539
|
+
export function getDeserializer(contentTypeString: string, streaming: false): Deserialize;
|
|
540
|
+
export function getDeserializer(
|
|
541
|
+
contentTypeString: string,
|
|
542
|
+
streaming: true
|
|
543
|
+
): (stream: Readable) => Promise<ReturnType<Deserialize>>;
|
|
544
|
+
export function getDeserializer(
|
|
545
|
+
contentTypeString: string = '',
|
|
546
|
+
streaming: boolean = false
|
|
547
|
+
): Deserialize | ((stream: Readable) => Promise<ReturnType<Deserialize>>) {
|
|
548
|
+
const contentType = parseContentType(contentTypeString);
|
|
549
|
+
|
|
550
|
+
const deserialize =
|
|
551
|
+
(contentType.type && mediaTypes.get(contentType.type)?.deserialize) || deserializerUnknownType(contentType);
|
|
552
|
+
|
|
553
|
+
return streaming ? (stream: Readable) => streamToBuffer(stream).then(deserialize) : deserialize;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
function deserializerUnknownType(contentType: ContentType): Deserialize {
|
|
557
|
+
// TODO: store the content-disposition too
|
|
558
|
+
|
|
559
|
+
if (contentType.type.startsWith('text/')) {
|
|
560
|
+
// convert the data to a string since it is text (using the provided charset if specified)
|
|
561
|
+
if (contentType.parameters?.charset && !isBufferEncoding(contentType.parameters.charset)) {
|
|
562
|
+
logger.info?.(`Unknown Buffer encoding ${contentType.parameters.charset} in content-type. Proceeding anyways.`);
|
|
563
|
+
}
|
|
564
|
+
return (data) => ({
|
|
565
|
+
contentType: contentType.type,
|
|
566
|
+
// @ts-expect-error We are okay with passing whatever the user has specified as the encoding to the `toString` method
|
|
567
|
+
data: data.toString(contentType.parameters?.charset || 'utf-8'),
|
|
568
|
+
});
|
|
569
|
+
} else if (contentType.type === 'application/octet-stream') {
|
|
570
|
+
// use this type as a way of directly transferring binary data (since that is what it means)
|
|
571
|
+
return (data) => data;
|
|
572
|
+
} else {
|
|
573
|
+
return (data) => {
|
|
574
|
+
if (contentType.type === '') {
|
|
575
|
+
// try to parse as JSON if no content type
|
|
576
|
+
try {
|
|
577
|
+
// if the first byte is `{` then it is likely JSON
|
|
578
|
+
if (data?.[0] === 123) return JSONParse(data);
|
|
579
|
+
} catch {
|
|
580
|
+
// continue if cannot parse as JSON
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
// else record the type and binary data as a pair
|
|
584
|
+
return { contentType: contentType.type || 'application/octet-stream', data };
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
function transformIterable(iterable, transform) {
|
|
590
|
+
return {
|
|
591
|
+
[Symbol.asyncIterator]() {
|
|
592
|
+
const iterator = iterable[Symbol.asyncIterator] ? iterable[Symbol.asyncIterator]() : iterable[Symbol.iterator]();
|
|
593
|
+
return {
|
|
594
|
+
next() {
|
|
595
|
+
const step = iterator.next();
|
|
596
|
+
if (step.then) {
|
|
597
|
+
return step.then((step) => ({
|
|
598
|
+
value: transform(step.value),
|
|
599
|
+
done: step.done,
|
|
600
|
+
}));
|
|
601
|
+
}
|
|
602
|
+
return {
|
|
603
|
+
value: transform(step.value),
|
|
604
|
+
done: step.done,
|
|
605
|
+
};
|
|
606
|
+
},
|
|
607
|
+
return(value) {
|
|
608
|
+
return iterator.return(value);
|
|
609
|
+
},
|
|
610
|
+
throw(error) {
|
|
611
|
+
return iterator.throw(error);
|
|
612
|
+
},
|
|
613
|
+
};
|
|
614
|
+
},
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* Converts JS objects/arrays/iterators to a CSV stream. Should support iterators with full backpressure handling
|
|
620
|
+
* @param data
|
|
621
|
+
* @returns stream
|
|
622
|
+
*/
|
|
623
|
+
export function toCsvStream(data, columns) {
|
|
624
|
+
// ensure that we pass it an iterable
|
|
625
|
+
const readStream = stream.Readable.from(data?.[Symbol.iterator] || data?.[Symbol.asyncIterator] ? data : [data]);
|
|
626
|
+
const options = {};
|
|
627
|
+
if (columns)
|
|
628
|
+
options.fields = columns.map((column) => ({
|
|
629
|
+
label: column,
|
|
630
|
+
value: column,
|
|
631
|
+
}));
|
|
632
|
+
const transformOptions = { objectMode: true };
|
|
633
|
+
// Create a json2csv stream transform.
|
|
634
|
+
const json2csv = new Transform(options, transformOptions);
|
|
635
|
+
// Pipe the data read stream through json2csv which converts it to CSV
|
|
636
|
+
return readStream.pipe(json2csv);
|
|
637
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
const { recordAction, recordActionBinary } = require('../../resources/analytics/write.ts');
|
|
2
|
+
const fp = require('fastify-plugin');
|
|
3
|
+
|
|
4
|
+
const ESTIMATED_HEADER_SIZE = 200; // it is very expensive to actually measure HTTP response header size (we change it
|
|
5
|
+
// ourselves) with an unacceptable performance penalty, so we estimate this part
|
|
6
|
+
|
|
7
|
+
module.exports = fp(
|
|
8
|
+
function (fastify, opts, done) {
|
|
9
|
+
// eslint-disable-next-line require-await
|
|
10
|
+
fastify.addHook('onResponse', async (request, reply) => {
|
|
11
|
+
// elapsedTime has to be accessed in onResponse or it won't work
|
|
12
|
+
let _time = reply.elapsedTime;
|
|
13
|
+
});
|
|
14
|
+
// eslint-disable-next-line require-await
|
|
15
|
+
fastify.addHook('onSend', async (request, reply, payload) => {
|
|
16
|
+
let responseTime = reply.elapsedTime;
|
|
17
|
+
let startTransfer = performance.now();
|
|
18
|
+
let config = reply.request.routeOptions;
|
|
19
|
+
let action;
|
|
20
|
+
let type;
|
|
21
|
+
let method;
|
|
22
|
+
if (config.config?.isOperation) {
|
|
23
|
+
action = request.body?.operation;
|
|
24
|
+
type = 'operation';
|
|
25
|
+
} else {
|
|
26
|
+
action = config.url;
|
|
27
|
+
type = 'fastify-route';
|
|
28
|
+
method = config.method;
|
|
29
|
+
}
|
|
30
|
+
recordAction(responseTime, 'duration', action, method, type);
|
|
31
|
+
// TODO: Remove the "success" metric, since we have switch to using recording responses by status code
|
|
32
|
+
recordActionBinary(reply.raw.statusCode < 400, 'success', action, method, type);
|
|
33
|
+
recordActionBinary(1, 'response_' + reply.raw.statusCode, action, method, type);
|
|
34
|
+
let bytesSent = ESTIMATED_HEADER_SIZE;
|
|
35
|
+
if (payload?.pipe) {
|
|
36
|
+
// if we are sending a stream, track the bytes sent and wait for when it completes
|
|
37
|
+
payload.on('data', (data) => {
|
|
38
|
+
bytesSent += data.length;
|
|
39
|
+
});
|
|
40
|
+
payload.on('end', () => {
|
|
41
|
+
recordAction(performance.now() - startTransfer, 'transfer', action, method, type);
|
|
42
|
+
recordAction(bytesSent, 'bytes-sent', action, method, type);
|
|
43
|
+
});
|
|
44
|
+
} else {
|
|
45
|
+
// otherwise just record bytes sent
|
|
46
|
+
bytesSent += payload?.length || 0;
|
|
47
|
+
recordAction(bytesSent, 'bytes-sent', action, method, type);
|
|
48
|
+
}
|
|
49
|
+
let roundedTime = responseTime.toFixed(3);
|
|
50
|
+
let appServerTiming = reply.getHeader('Server-Timing');
|
|
51
|
+
let serverTiming = `db;dur=${roundedTime}`;
|
|
52
|
+
reply.header('Server-Timing', appServerTiming ? `${appServerTiming}, ${serverTiming}` : serverTiming);
|
|
53
|
+
});
|
|
54
|
+
done();
|
|
55
|
+
},
|
|
56
|
+
{ name: 'hdb-request-time' }
|
|
57
|
+
);
|