@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/BinObjects.js +17 -0
- package/bin/cliOperations.js +157 -0
- package/bin/copyDb.ts +280 -0
- package/bin/harper.js +156 -0
- package/bin/install.js +15 -0
- package/bin/lite.js +5 -0
- package/bin/restart.js +201 -0
- package/bin/run.js +409 -0
- package/bin/status.js +65 -0
- package/bin/stop.js +22 -0
- package/bin/upgrade.js +134 -0
- package/components/Application.ts +646 -0
- package/components/ApplicationScope.ts +49 -0
- package/components/Component.ts +53 -0
- package/components/ComponentV1.ts +342 -0
- package/components/DEFAULT_CONFIG.ts +18 -0
- package/components/EntryHandler.ts +227 -0
- package/components/Logger.ts +14 -0
- package/components/OptionsWatcher.ts +354 -0
- package/components/PluginModule.ts +6 -0
- package/components/Scope.ts +329 -0
- package/components/componentLoader.ts +529 -0
- package/components/deriveCommonPatternBase.ts +31 -0
- package/components/deriveGlobOptions.ts +44 -0
- package/components/deriveURLPath.ts +57 -0
- package/components/operations.js +658 -0
- package/components/operationsValidation.js +246 -0
- package/components/packageComponent.ts +39 -0
- package/components/requestRestart.ts +26 -0
- package/components/resolveBaseURLPath.ts +38 -0
- package/components/status/ComponentStatus.ts +110 -0
- package/components/status/ComponentStatusRegistry.ts +251 -0
- package/components/status/api.ts +153 -0
- package/components/status/crossThread.ts +405 -0
- package/components/status/errors.ts +152 -0
- package/components/status/index.ts +44 -0
- package/components/status/internal.ts +65 -0
- package/components/status/registry.ts +12 -0
- package/components/status/types.ts +96 -0
- package/config/RootConfigWatcher.ts +59 -0
- package/config/configHelpers.ts +11 -0
- package/config/configUtils.js +967 -0
- package/config/harperConfigEnvVars.ts +641 -0
- package/dataLayer/CreateAttributeObject.js +25 -0
- package/dataLayer/CreateTableObject.js +11 -0
- package/dataLayer/DataLayerObjects.js +43 -0
- package/dataLayer/DeleteBeforeObject.js +22 -0
- package/dataLayer/DeleteObject.js +25 -0
- package/dataLayer/DropAttributeObject.js +11 -0
- package/dataLayer/GetBackupObject.js +22 -0
- package/dataLayer/InsertObject.js +24 -0
- package/dataLayer/ReadAuditLogObject.js +24 -0
- package/dataLayer/SQLSearch.js +1335 -0
- package/dataLayer/SearchByConditionsObject.js +61 -0
- package/dataLayer/SearchByHashObject.js +21 -0
- package/dataLayer/SearchObject.js +45 -0
- package/dataLayer/SqlSearchObject.js +14 -0
- package/dataLayer/UpdateObject.js +23 -0
- package/dataLayer/UpsertObject.js +23 -0
- package/dataLayer/bulkLoad.js +813 -0
- package/dataLayer/dataObjects/BulkLoadObjects.js +27 -0
- package/dataLayer/dataObjects/UpsertObject.js +23 -0
- package/dataLayer/delete.js +164 -0
- package/dataLayer/export.js +381 -0
- package/dataLayer/getBackup.js +40 -0
- package/dataLayer/harperBridge/BridgeMethods.js +81 -0
- package/dataLayer/harperBridge/ResourceBridge.ts +633 -0
- package/dataLayer/harperBridge/bridgeUtility/insertUpdateReturnObj.js +28 -0
- package/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +88 -0
- package/dataLayer/harperBridge/harperBridge.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/LMDBBridge.js +119 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +19 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +112 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +67 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +31 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +94 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +98 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +89 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +109 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +107 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +137 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +35 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +111 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +28 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +29 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +207 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +156 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +30 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +19 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +64 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +70 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +22 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +23 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +22 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBTransactionObject.js +23 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +24 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +24 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +25 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +157 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +94 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +39 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +34 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +100 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +371 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +109 -0
- package/dataLayer/hdbInfoController.js +254 -0
- package/dataLayer/insert.js +266 -0
- package/dataLayer/readAuditLog.js +59 -0
- package/dataLayer/schema.js +366 -0
- package/dataLayer/schemaDescribe.js +289 -0
- package/dataLayer/search.js +60 -0
- package/dataLayer/transaction.js +17 -0
- package/dataLayer/update.js +124 -0
- package/dist/components/Logger.d.ts +12 -0
- package/dist/components/Logger.js +3 -0
- package/dist/components/Logger.js.map +1 -0
- package/dist/components/Scope.d.ts +14 -4
- package/dist/components/Scope.js +18 -10
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/componentLoader.js +16 -9
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/operations.js +2 -2
- package/dist/components/operations.js.map +1 -1
- package/dist/config/configUtils.d.ts +1 -1
- package/dist/config/configUtils.js +1 -1
- package/dist/config/configUtils.js.map +1 -1
- package/dist/dataLayer/CreateTableObject.d.ts +2 -2
- package/dist/dataLayer/CreateTableObject.js +2 -2
- package/dist/dataLayer/CreateTableObject.js.map +1 -1
- package/dist/dataLayer/delete.d.ts +1 -1
- package/dist/dataLayer/schema.js +6 -5
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +1 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/resources/DatabaseTransaction.d.ts +1 -1
- package/dist/resources/IterableEventQueue.d.ts +1 -1
- package/dist/resources/LMDBTransaction.d.ts +5 -1
- package/dist/resources/Resource.d.ts +1 -1
- package/dist/resources/RocksIndexStore.d.ts +3 -3
- package/dist/resources/RocksTransactionLogStore.d.ts +6 -3
- package/dist/resources/Table.d.ts +15 -6
- package/dist/resources/Table.js +4 -1
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/read.js +32 -22
- package/dist/resources/analytics/read.js.map +1 -1
- package/dist/resources/analytics/write.js +3 -6
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +3 -3
- package/dist/resources/blob.d.ts +25 -2
- package/dist/resources/databases.d.ts +12 -2
- package/dist/resources/databases.js +22 -19
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/search.js +11 -5
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transaction.d.ts +2 -1
- package/dist/security/auth.js +1 -1
- package/dist/security/auth.js.map +1 -1
- package/dist/security/cryptoHash.d.ts +2 -2
- package/dist/security/jsLoader.js +243 -66
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +4 -5
- package/dist/security/keys.js.map +1 -1
- package/dist/security/user.js +3 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/REST.js +16 -2
- package/dist/server/REST.js.map +1 -1
- package/dist/server/Server.d.ts +2 -1
- package/dist/server/Server.js.map +1 -1
- package/dist/server/fastifyRoutes/plugins/hdbCore.d.ts +6 -1
- package/dist/server/fastifyRoutes.js +2 -0
- package/dist/server/fastifyRoutes.js.map +1 -1
- package/dist/server/http.js +12 -6
- package/dist/server/http.js.map +1 -1
- package/dist/server/jobs/JobObject.d.ts +3 -3
- package/dist/server/loadRootComponents.js +1 -0
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/operationsServer.js +3 -1
- package/dist/server/operationsServer.js.map +1 -1
- package/dist/server/serverHelpers/JSONStream.d.ts +3 -3
- package/dist/server/serverHelpers/Request.d.ts +5 -5
- package/dist/server/serverHelpers/requestTimePlugin.d.ts +1 -1
- package/dist/server/threads/manageThreads.d.ts +2 -2
- package/dist/server/threads/manageThreads.js +50 -35
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.d.ts +1 -1
- package/dist/sqlTranslator/deleteTranslator.d.ts +1 -1
- package/dist/utility/AWS/AWSConnector.d.ts +3 -2
- package/dist/utility/common_utils.d.ts +3 -3
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/functions/date/dateFunctions.d.ts +11 -11
- package/dist/utility/globalSchema.d.ts +1 -1
- package/dist/utility/hdbTerms.d.ts +3 -0
- package/dist/utility/hdbTerms.js +3 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/installation.d.ts +2 -4
- package/dist/utility/installation.js.map +1 -1
- package/dist/utility/lmdb/commonUtility.d.ts +1 -0
- package/dist/utility/lmdb/deleteUtility.d.ts +1 -0
- package/dist/utility/lmdb/environmentUtility.d.ts +1 -0
- package/dist/utility/lmdb/searchUtility.d.ts +2 -1
- package/dist/utility/lmdb/writeUtility.d.ts +1 -0
- package/dist/utility/logging/harper_logger.d.ts +6 -6
- package/dist/utility/processManagement/processManagement.d.ts +1 -1
- package/dist/utility/processManagement/servicesConfig.d.ts +12 -6
- package/dist/validation/common_validators.d.ts +4 -3
- package/dist/validation/configValidator.d.ts +3 -2
- package/index.d.ts +56 -0
- package/index.js +41 -0
- package/json/systemSchema.json +373 -0
- package/launchServiceScripts/launchHarperDB.js +3 -0
- package/launchServiceScripts/utility/checkNodeVersion.js +15 -0
- package/package.json +21 -3
- package/resources/DatabaseTransaction.ts +378 -0
- package/resources/ErrorResource.ts +57 -0
- package/resources/IterableEventQueue.ts +94 -0
- package/resources/LMDBTransaction.ts +349 -0
- package/resources/RecordEncoder.ts +702 -0
- package/resources/RequestTarget.ts +134 -0
- package/resources/Resource.ts +789 -0
- package/resources/ResourceInterface.ts +221 -0
- package/resources/ResourceInterfaceV2.ts +53 -0
- package/resources/ResourceV2.ts +67 -0
- package/resources/Resources.ts +162 -0
- package/resources/RocksIndexStore.ts +70 -0
- package/resources/RocksTransactionLogStore.ts +352 -0
- package/resources/Table.ts +4527 -0
- package/resources/analytics/hostnames.ts +72 -0
- package/resources/analytics/metadata.ts +10 -0
- package/resources/analytics/read.ts +252 -0
- package/resources/analytics/write.ts +803 -0
- package/resources/auditStore.ts +556 -0
- package/resources/blob.ts +1268 -0
- package/resources/crdt.ts +125 -0
- package/resources/dataLoader.ts +527 -0
- package/resources/databases.ts +1290 -0
- package/resources/graphql.ts +221 -0
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +638 -0
- package/resources/indexes/customIndexes.ts +7 -0
- package/resources/indexes/vector.ts +38 -0
- package/resources/jsResource.ts +86 -0
- package/resources/loadEnv.ts +22 -0
- package/resources/login.ts +18 -0
- package/resources/openApi.ts +409 -0
- package/resources/registrationDeprecated.ts +8 -0
- package/resources/replayLogs.ts +136 -0
- package/resources/roles.ts +98 -0
- package/resources/search.ts +1301 -0
- package/resources/tracked.ts +584 -0
- package/resources/transaction.ts +89 -0
- package/resources/transactionBroadcast.ts +258 -0
- package/security/auth.ts +376 -0
- package/security/certificateVerification/certificateVerificationSource.ts +84 -0
- package/security/certificateVerification/configValidation.ts +107 -0
- package/security/certificateVerification/crlVerification.ts +623 -0
- package/security/certificateVerification/index.ts +121 -0
- package/security/certificateVerification/ocspVerification.ts +148 -0
- package/security/certificateVerification/pkijs-ed25519-patch.ts +188 -0
- package/security/certificateVerification/types.ts +128 -0
- package/security/certificateVerification/verificationConfig.ts +138 -0
- package/security/certificateVerification/verificationUtils.ts +447 -0
- package/security/cryptoHash.js +42 -0
- package/security/data_objects/PermissionAttributeResponseObject.js +15 -0
- package/security/data_objects/PermissionResponseObject.js +115 -0
- package/security/data_objects/PermissionTableResponseObject.js +20 -0
- package/security/fastifyAuth.js +169 -0
- package/security/impersonation.ts +160 -0
- package/security/jsLoader.ts +716 -0
- package/security/keys.js +948 -0
- package/security/permissionsTranslator.js +300 -0
- package/security/role.js +218 -0
- package/security/tokenAuthentication.ts +228 -0
- package/security/user.ts +449 -0
- package/server/DurableSubscriptionsSession.ts +503 -0
- package/server/REST.ts +407 -0
- package/server/Server.ts +89 -0
- package/server/fastifyRoutes/helpers/getCORSOptions.js +36 -0
- package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +15 -0
- package/server/fastifyRoutes/helpers/getServerOptions.js +33 -0
- package/server/fastifyRoutes/plugins/hdbCore.js +39 -0
- package/server/fastifyRoutes.ts +205 -0
- package/server/graphqlQuerying.ts +700 -0
- package/server/http.ts +640 -0
- package/server/itc/serverHandlers.js +161 -0
- package/server/itc/utility/ITCEventObject.js +10 -0
- package/server/jobs/JobObject.js +24 -0
- package/server/jobs/jobProcess.js +69 -0
- package/server/jobs/jobRunner.js +162 -0
- package/server/jobs/jobs.js +304 -0
- package/server/loadRootComponents.js +44 -0
- package/server/mqtt.ts +485 -0
- package/server/nodeName.ts +75 -0
- package/server/operationsServer.ts +313 -0
- package/server/serverHelpers/Headers.ts +108 -0
- package/server/serverHelpers/JSONStream.ts +269 -0
- package/server/serverHelpers/OperationFunctionObject.ts +13 -0
- package/server/serverHelpers/Request.ts +158 -0
- package/server/serverHelpers/contentTypes.ts +637 -0
- package/server/serverHelpers/requestTimePlugin.js +57 -0
- package/server/serverHelpers/serverHandlers.js +148 -0
- package/server/serverHelpers/serverUtilities.ts +473 -0
- package/server/serverRegistry.ts +8 -0
- package/server/static.ts +187 -0
- package/server/status/definitions.ts +37 -0
- package/server/status/index.ts +125 -0
- package/server/storageReclamation.ts +93 -0
- package/server/threads/itc.js +89 -0
- package/server/threads/manageThreads.js +594 -0
- package/server/threads/socketRouter.ts +360 -0
- package/server/threads/threadServer.js +279 -0
- package/server/throttle.ts +73 -0
- package/sqlTranslator/SelectValidator.js +330 -0
- package/sqlTranslator/alasqlFunctionImporter.js +62 -0
- package/sqlTranslator/deleteTranslator.js +67 -0
- package/sqlTranslator/index.js +242 -0
- package/sqlTranslator/sql_statement_bucket.js +472 -0
- package/static/defaultConfig.yaml +3 -0
- package/studio/web/HDBDogOnly.svg +78 -0
- package/studio/web/assets/PPRadioGrotesk-Bold-DDaUYG8E.woff +0 -0
- package/studio/web/assets/fa-brands-400-CEJbCg16.woff +0 -0
- package/studio/web/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
- package/studio/web/assets/fa-brands-400-DnkPfk3o.eot +0 -0
- package/studio/web/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
- package/studio/web/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
- package/studio/web/assets/fa-regular-400-BhTwtT8w.eot +0 -0
- package/studio/web/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
- package/studio/web/assets/fa-regular-400-DFnMcJPd.woff +0 -0
- package/studio/web/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
- package/studio/web/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
- package/studio/web/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
- package/studio/web/assets/fa-solid-900-B6Axprfb.eot +0 -0
- package/studio/web/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
- package/studio/web/assets/fa-solid-900-DOXgCApm.woff +0 -0
- package/studio/web/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
- package/studio/web/assets/index-BTgXJX9d.js +235 -0
- package/studio/web/assets/index-BTgXJX9d.js.map +1 -0
- package/studio/web/assets/index-C-GXfcup.js +37 -0
- package/studio/web/assets/index-C-GXfcup.js.map +1 -0
- package/studio/web/assets/index-PFlNdimM.js +2 -0
- package/studio/web/assets/index-PFlNdimM.js.map +1 -0
- package/studio/web/assets/index-Y2g_iFpU.css +1 -0
- package/studio/web/assets/index-jiPwkrsB.css +1 -0
- package/studio/web/assets/index.lazy-C3TJZJ4o.js +266 -0
- package/studio/web/assets/index.lazy-C3TJZJ4o.js.map +1 -0
- package/studio/web/assets/profiler-DotzgiCJ.js +2 -0
- package/studio/web/assets/profiler-DotzgiCJ.js.map +1 -0
- package/studio/web/assets/react-redux-VxUEx_mU.js +6 -0
- package/studio/web/assets/react-redux-VxUEx_mU.js.map +1 -0
- package/studio/web/assets/startRecording-B_9J9Csd.js +3 -0
- package/studio/web/assets/startRecording-B_9J9Csd.js.map +1 -0
- package/studio/web/fabric-signup-background.webp +0 -0
- package/studio/web/fabric-signup-text.png +0 -0
- package/studio/web/favicon_purple.png +0 -0
- package/studio/web/github-icon.svg +15 -0
- package/studio/web/harper-fabric_black.png +0 -0
- package/studio/web/harper-fabric_white.png +0 -0
- package/studio/web/harper-studio_white.png +0 -0
- package/studio/web/index.html +16 -0
- package/studio/web/running.css +148 -0
- package/studio/web/running.html +147 -0
- package/studio/web/running.js +111 -0
- package/upgrade/UpgradeObjects.js +13 -0
- package/upgrade/directives/directivesController.js +90 -0
- package/upgrade/directivesManager.js +139 -0
- package/upgrade/upgradePrompt.js +124 -0
- package/upgrade/upgradeUtilities.js +28 -0
- package/utility/AWS/AWSConnector.js +29 -0
- package/utility/OperationFunctionCaller.js +63 -0
- package/utility/assignCmdEnvVariables.js +62 -0
- package/utility/common_utils.js +867 -0
- package/utility/environment/environmentManager.js +208 -0
- package/utility/environment/systemInformation.js +355 -0
- package/utility/errors/commonErrors.js +267 -0
- package/utility/errors/hdbError.js +146 -0
- package/utility/functions/date/dateFunctions.js +65 -0
- package/utility/functions/geo.js +355 -0
- package/utility/functions/sql/alaSQLExtension.js +104 -0
- package/utility/globalSchema.js +35 -0
- package/utility/hdbTerms.ts +819 -0
- package/utility/install/checkJWTTokensExist.js +62 -0
- package/utility/install/harperdb.conf +15 -0
- package/utility/install/harperdb.service +14 -0
- package/utility/install/installer.js +635 -0
- package/utility/installation.ts +30 -0
- package/utility/lmdb/DBIDefinition.js +20 -0
- package/utility/lmdb/DeleteRecordsResponseObject.js +25 -0
- package/utility/lmdb/InsertRecordsResponseObject.js +22 -0
- package/utility/lmdb/OpenDBIObject.js +31 -0
- package/utility/lmdb/OpenEnvironmentObject.js +41 -0
- package/utility/lmdb/UpdateRecordsResponseObject.js +25 -0
- package/utility/lmdb/UpsertRecordsResponseObject.js +22 -0
- package/utility/lmdb/cleanLMDBMap.js +65 -0
- package/utility/lmdb/commonUtility.js +119 -0
- package/utility/lmdb/deleteUtility.js +128 -0
- package/utility/lmdb/environmentUtility.js +477 -0
- package/utility/lmdb/searchCursorFunctions.js +187 -0
- package/utility/lmdb/searchUtility.js +918 -0
- package/utility/lmdb/terms.js +57 -0
- package/utility/lmdb/writeUtility.js +407 -0
- package/utility/logging/harper_logger.js +876 -0
- package/utility/logging/logRotator.js +157 -0
- package/utility/logging/logger.ts +24 -0
- package/utility/logging/readLog.js +355 -0
- package/utility/logging/transactionLog.js +57 -0
- package/utility/mount_hdb.js +59 -0
- package/utility/npmUtilities.js +102 -0
- package/utility/operationPermissions.ts +112 -0
- package/utility/operation_authorization.js +836 -0
- package/utility/packageUtils.js +55 -0
- package/utility/password.ts +99 -0
- package/utility/processManagement/processManagement.js +187 -0
- package/utility/processManagement/servicesConfig.js +56 -0
- package/utility/scripts/restartHdb.js +24 -0
- package/utility/scripts/user_data.sh +13 -0
- package/utility/signalling.js +36 -0
- package/utility/terms/certificates.js +81 -0
- package/utility/when.ts +20 -0
- package/v1.d.ts +39 -0
- package/v1.js +41 -0
- package/v2.d.ts +39 -0
- package/v2.js +41 -0
- package/validation/bulkDeleteValidator.js +24 -0
- package/validation/check_permissions.js +19 -0
- package/validation/common_validators.js +95 -0
- package/validation/configValidator.js +331 -0
- package/validation/deleteValidator.js +15 -0
- package/validation/fileLoadValidator.js +153 -0
- package/validation/insertValidator.js +40 -0
- package/validation/installValidator.js +37 -0
- package/validation/readLogValidator.js +64 -0
- package/validation/role_validation.js +320 -0
- package/validation/schemaMetadataValidator.js +42 -0
- package/validation/searchValidator.js +166 -0
- package/validation/statusValidator.ts +66 -0
- package/validation/transactionLogValidator.js +33 -0
- package/validation/user_validation.js +55 -0
- package/validation/validationWrapper.js +105 -0
- package/dist/resources/analytics/profile.d.ts +0 -2
- package/dist/resources/analytics/profile.js +0 -144
- package/dist/resources/analytics/profile.js.map +0 -1
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
const _ = require('lodash'),
|
|
2
|
+
validator = require('./validationWrapper.js');
|
|
3
|
+
const Joi = require('joi');
|
|
4
|
+
const hdbUtils = require('../utility/common_utils.js');
|
|
5
|
+
const { hdbSchemaTable, checkValidTable, hdbTable, hdbDatabase } = require('./common_validators.js');
|
|
6
|
+
const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.js');
|
|
7
|
+
const { getDatabases } = require('../resources/databases.ts');
|
|
8
|
+
const { HTTP_STATUS_CODES } = hdbErrors;
|
|
9
|
+
|
|
10
|
+
const searchByValueSchema = Joi.object({
|
|
11
|
+
database: hdbDatabase,
|
|
12
|
+
schema: hdbDatabase,
|
|
13
|
+
table: hdbTable,
|
|
14
|
+
attribute: hdbSchemaTable,
|
|
15
|
+
value: Joi.any().required(),
|
|
16
|
+
get_attributes: Joi.array().min(1).items(Joi.alternatives(hdbSchemaTable, Joi.object())).optional(),
|
|
17
|
+
desc: Joi.bool(),
|
|
18
|
+
limit: Joi.number().integer().min(1),
|
|
19
|
+
offset: Joi.number().integer().min(0),
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
const searchByConditionsSchema = Joi.object({
|
|
23
|
+
database: hdbDatabase,
|
|
24
|
+
schema: hdbDatabase,
|
|
25
|
+
table: hdbTable,
|
|
26
|
+
operator: Joi.string().valid('and', 'or').default('and').lowercase(),
|
|
27
|
+
offset: Joi.number().integer().min(0),
|
|
28
|
+
limit: Joi.number().integer().min(1),
|
|
29
|
+
get_attributes: Joi.array().min(1).items(Joi.alternatives(hdbSchemaTable, Joi.object())).optional(),
|
|
30
|
+
sort: Joi.object({
|
|
31
|
+
attribute: Joi.alternatives(hdbSchemaTable, Joi.array().min(1)),
|
|
32
|
+
descending: Joi.bool().optional(),
|
|
33
|
+
}).optional(),
|
|
34
|
+
conditions: Joi.array()
|
|
35
|
+
.min(1)
|
|
36
|
+
.items(
|
|
37
|
+
Joi.alternatives(
|
|
38
|
+
Joi.object({ operator: Joi.string().valid('and', 'or').default('and').lowercase(), conditions: Joi.array() }),
|
|
39
|
+
Joi.object({
|
|
40
|
+
attribute: Joi.alternatives(hdbSchemaTable, Joi.array().min(1)),
|
|
41
|
+
comparator: Joi.string()
|
|
42
|
+
.valid(
|
|
43
|
+
'equals',
|
|
44
|
+
'contains',
|
|
45
|
+
'starts_with',
|
|
46
|
+
'ends_with',
|
|
47
|
+
'greater_than',
|
|
48
|
+
'greater_than_equal',
|
|
49
|
+
'less_than',
|
|
50
|
+
'less_than_equal',
|
|
51
|
+
'between',
|
|
52
|
+
'not_equal'
|
|
53
|
+
)
|
|
54
|
+
.optional(),
|
|
55
|
+
value: Joi.when('comparator', {
|
|
56
|
+
switch: [
|
|
57
|
+
{ is: 'equals', then: Joi.any() },
|
|
58
|
+
{
|
|
59
|
+
is: 'between',
|
|
60
|
+
then: Joi.array()
|
|
61
|
+
.items(Joi.alternatives([Joi.string(), Joi.number()]))
|
|
62
|
+
.length(2),
|
|
63
|
+
},
|
|
64
|
+
],
|
|
65
|
+
otherwise: Joi.alternatives(Joi.string(), Joi.number()),
|
|
66
|
+
}).required(),
|
|
67
|
+
})
|
|
68
|
+
)
|
|
69
|
+
)
|
|
70
|
+
.required(),
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
module.exports = function (searchObject, type) {
|
|
74
|
+
let validationError = null;
|
|
75
|
+
switch (type) {
|
|
76
|
+
case 'value':
|
|
77
|
+
validationError = validator.validateBySchema(searchObject, searchByValueSchema);
|
|
78
|
+
break;
|
|
79
|
+
case 'hashes':
|
|
80
|
+
let errors;
|
|
81
|
+
addError(checkValidTable('database', searchObject.schema));
|
|
82
|
+
addError(checkValidTable('table', searchObject.table));
|
|
83
|
+
if (!searchObject.hash_values) addError(`'hash_values' is required`);
|
|
84
|
+
else if (!Array.isArray(searchObject.hash_values)) addError(`'hash_values' must be an array`);
|
|
85
|
+
else if (!searchObject.hash_values.every((value) => typeof value === 'string' || typeof value === 'number'))
|
|
86
|
+
addError(`'hash_values' must be strings or numbers`);
|
|
87
|
+
if (!searchObject.get_attributes) addError(`'get_attributes' is required`);
|
|
88
|
+
else if (!Array.isArray(searchObject.get_attributes)) addError(`'get_attributes' must be an array`);
|
|
89
|
+
else if (searchObject.get_attributes.length === 0) addError(`'get_attributes' must contain at least 1 item`);
|
|
90
|
+
else if (!searchObject.get_attributes.every((value) => typeof value === 'string' || typeof value === 'number'))
|
|
91
|
+
addError(`'get_attributes' must be strings or numbers`);
|
|
92
|
+
function addError(error) {
|
|
93
|
+
if (errors) errors += '. ' + error;
|
|
94
|
+
else errors = error;
|
|
95
|
+
}
|
|
96
|
+
if (errors) validationError = new Error(errors.trim());
|
|
97
|
+
break;
|
|
98
|
+
case 'conditions':
|
|
99
|
+
validationError = validator.validateBySchema(searchObject, searchByConditionsSchema);
|
|
100
|
+
break;
|
|
101
|
+
default:
|
|
102
|
+
throw new Error(`Error validating search, unknown type: ${type}`);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// validate table and attribute if format validation is valid
|
|
106
|
+
if (!validationError && searchObject.schema !== 'system') {
|
|
107
|
+
// skip validation for system schema
|
|
108
|
+
//check if schema.table does not exist throw error
|
|
109
|
+
let checkSchemaTable = hdbUtils.checkGlobalSchemaTable(searchObject.schema, searchObject.table);
|
|
110
|
+
if (checkSchemaTable) {
|
|
111
|
+
return handleHDBError(new Error(), checkSchemaTable, HTTP_STATUS_CODES.NOT_FOUND);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
let tableSchema = getDatabases()[searchObject.schema][searchObject.table];
|
|
115
|
+
let allTableAttributes = tableSchema.attributes;
|
|
116
|
+
|
|
117
|
+
//this clones the get_attributes array
|
|
118
|
+
let checkAttributes = searchObject.get_attributes ? [...searchObject.get_attributes] : [];
|
|
119
|
+
|
|
120
|
+
if (type === 'value') {
|
|
121
|
+
checkAttributes.push(searchObject.attribute);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
//if search type is conditions add conditions fields to see if the fields exist
|
|
125
|
+
const addConditions = (searchObject) => {
|
|
126
|
+
//this is used to validate condition attributes exist in the schema
|
|
127
|
+
for (const condition of searchObject.conditions) {
|
|
128
|
+
if (condition.conditions) addConditions(condition);
|
|
129
|
+
else checkAttributes.push(condition.attribute);
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
if (type === 'conditions') {
|
|
133
|
+
addConditions(searchObject);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
let unknownAttributes = _.filter(
|
|
137
|
+
checkAttributes,
|
|
138
|
+
(attribute) =>
|
|
139
|
+
attribute !== '*' &&
|
|
140
|
+
!attribute.startsWith?.('$') && // meta attributes
|
|
141
|
+
attribute.attribute !== '*' && // skip check for asterisk attribute
|
|
142
|
+
!Array.isArray(attribute) &&
|
|
143
|
+
!attribute.name && // nested attribute
|
|
144
|
+
!_.some(
|
|
145
|
+
allTableAttributes,
|
|
146
|
+
(
|
|
147
|
+
tableAttribute // attribute should match one of the attribute in global
|
|
148
|
+
) =>
|
|
149
|
+
tableAttribute === attribute ||
|
|
150
|
+
tableAttribute.attribute === attribute ||
|
|
151
|
+
tableAttribute.attribute === attribute.attribute
|
|
152
|
+
)
|
|
153
|
+
);
|
|
154
|
+
|
|
155
|
+
// if any unknown attributes present in the search request then list all indicated as unknown attribute to error message at once split in well format
|
|
156
|
+
// for instance "unknown attribute a, b and c" or "unknown attribute a"
|
|
157
|
+
if (unknownAttributes && unknownAttributes.length > 0) {
|
|
158
|
+
// return error with proper message - replace last comma with and
|
|
159
|
+
let errorMsg = unknownAttributes.join(', ');
|
|
160
|
+
errorMsg = errorMsg.replace(/,([^,]*)$/, ' and$1');
|
|
161
|
+
return new Error(`unknown attribute '${errorMsg}'`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
return validationError;
|
|
166
|
+
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import Joi from 'joi';
|
|
2
|
+
import * as validator from './validationWrapper.js';
|
|
3
|
+
import { STATUS_DEFINITIONS, STATUS_IDS, DEFAULT_STATUS_ID, type StatusId } from '../server/status/definitions.ts';
|
|
4
|
+
|
|
5
|
+
// Re-export constants for backward compatibility
|
|
6
|
+
export const STATUS_SCHEMAS = STATUS_DEFINITIONS;
|
|
7
|
+
export const STATUS_ALLOWED = STATUS_IDS;
|
|
8
|
+
export const STATUS_DEFAULT = DEFAULT_STATUS_ID;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Pregenerate error messages to avoid repeated string concatenation
|
|
12
|
+
*/
|
|
13
|
+
const ERROR_MESSAGES = Object.entries(STATUS_DEFINITIONS).reduce(
|
|
14
|
+
(messages, [id, definition]) => {
|
|
15
|
+
if (definition.allowedValues) {
|
|
16
|
+
messages[id] = `Status "${id}" only accepts these values: ${definition.allowedValues.join(', ')}`;
|
|
17
|
+
}
|
|
18
|
+
return messages;
|
|
19
|
+
},
|
|
20
|
+
{} as Record<string, string>
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Creates the status validation schema using the STATUS_DEFINITIONS
|
|
25
|
+
*/
|
|
26
|
+
const createStatusValidationSchema = () => {
|
|
27
|
+
// Start with base schema
|
|
28
|
+
let statusSchema = Joi.string().min(1).max(512);
|
|
29
|
+
|
|
30
|
+
// Add conditional validations for each status type that has allowedValues
|
|
31
|
+
(Object.entries(STATUS_DEFINITIONS) as [StatusId, (typeof STATUS_DEFINITIONS)[StatusId]][]).forEach(
|
|
32
|
+
([id, definition]) => {
|
|
33
|
+
if (definition.allowedValues) {
|
|
34
|
+
statusSchema = statusSchema.when('id', {
|
|
35
|
+
is: id,
|
|
36
|
+
then: Joi.string()
|
|
37
|
+
.valid(...definition.allowedValues)
|
|
38
|
+
.messages({
|
|
39
|
+
'any.only': ERROR_MESSAGES[id],
|
|
40
|
+
}),
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
return statusSchema.required();
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Joi schema for validating status operations
|
|
51
|
+
*/
|
|
52
|
+
const setStatusSchema = Joi.object({
|
|
53
|
+
id: Joi.string()
|
|
54
|
+
.valid(...STATUS_ALLOWED)
|
|
55
|
+
.required(),
|
|
56
|
+
status: createStatusValidationSchema(),
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Validates the status operation parameters
|
|
61
|
+
* @param obj The status operation parameters to validate
|
|
62
|
+
* @returns Error if validation fails, null otherwise
|
|
63
|
+
*/
|
|
64
|
+
export function validateStatus(obj: any) {
|
|
65
|
+
return validator.validateBySchema(obj, setStatusSchema);
|
|
66
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const Joi = require('joi');
|
|
4
|
+
const validator = require('./validationWrapper.js');
|
|
5
|
+
|
|
6
|
+
module.exports = {
|
|
7
|
+
readTransactionLogValidator,
|
|
8
|
+
deleteTransactionLogsBeforeValidator,
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
function readTransactionLogValidator(req) {
|
|
12
|
+
const schema = Joi.object({
|
|
13
|
+
schema: Joi.string(),
|
|
14
|
+
database: Joi.string(),
|
|
15
|
+
table: Joi.string().required(),
|
|
16
|
+
from: Joi.date().timestamp(),
|
|
17
|
+
to: Joi.date().timestamp(),
|
|
18
|
+
limit: Joi.number().min(1),
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
return validator.validateBySchema(req, schema);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function deleteTransactionLogsBeforeValidator(req) {
|
|
25
|
+
const schema = Joi.object({
|
|
26
|
+
schema: Joi.string(),
|
|
27
|
+
database: Joi.string(),
|
|
28
|
+
table: Joi.string().required(),
|
|
29
|
+
timestamp: Joi.date().timestamp().required(),
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
return validator.validateBySchema(req, schema);
|
|
33
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
const validator = require('./validationWrapper.js');
|
|
2
|
+
|
|
3
|
+
const constraints = {
|
|
4
|
+
username: {
|
|
5
|
+
presence: true,
|
|
6
|
+
exclusion: {
|
|
7
|
+
within: ['system'],
|
|
8
|
+
message: 'You cannot create tables within the system schema',
|
|
9
|
+
},
|
|
10
|
+
},
|
|
11
|
+
password: {
|
|
12
|
+
presence: true,
|
|
13
|
+
},
|
|
14
|
+
role: {
|
|
15
|
+
presence: true,
|
|
16
|
+
format: '[\\w\\-\\_]+',
|
|
17
|
+
},
|
|
18
|
+
active: {
|
|
19
|
+
presence: true,
|
|
20
|
+
inclusion: {
|
|
21
|
+
within: [true, false],
|
|
22
|
+
message: 'must be a boolean',
|
|
23
|
+
},
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
function addUserValidation(object) {
|
|
28
|
+
constraints.password.presence = true;
|
|
29
|
+
constraints.username.presence = true;
|
|
30
|
+
constraints.role.presence = true;
|
|
31
|
+
constraints.active.presence = true;
|
|
32
|
+
return validator.validateObject(object, constraints);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function alterUserValidation(object) {
|
|
36
|
+
constraints.password.presence = false;
|
|
37
|
+
constraints.username.presence = true;
|
|
38
|
+
constraints.role.presence = false;
|
|
39
|
+
constraints.active.presence = false;
|
|
40
|
+
return validator.validateObject(object, constraints);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function dropUserValidation(object) {
|
|
44
|
+
constraints.password.presence = false;
|
|
45
|
+
constraints.username.presence = true;
|
|
46
|
+
constraints.role.presence = false;
|
|
47
|
+
constraints.active.presence = false;
|
|
48
|
+
return validator.validateObject(object, constraints);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
module.exports = {
|
|
52
|
+
addUserValidation,
|
|
53
|
+
alterUserValidation,
|
|
54
|
+
dropUserValidation,
|
|
55
|
+
};
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* validationWrapper.js
|
|
5
|
+
*
|
|
6
|
+
* This module is meant as a wrapper for calls to the validate module. This wrapper serves 2 purposes:
|
|
7
|
+
* it decouples the validate() call from the validate module, and it ensures a consistent "Error" return object
|
|
8
|
+
* so we can rely on it for logging and reporting.
|
|
9
|
+
*
|
|
10
|
+
* There are a few cases where the validate module is called directly for functions like isBoolean.
|
|
11
|
+
* These are rare enough for it not to be worth creating wrapper functions for those as well.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const validate = require('validate.js');
|
|
15
|
+
|
|
16
|
+
//This validator is added here b/c we are still on version 0.11.1 that does not include this build in functionality. When
|
|
17
|
+
// we do update, we can remove. The reason we have not is related to a breaking change on the "presence" validator rule
|
|
18
|
+
// that will require a lot of fixes on our end. More here - https://validatejs.org/#changelog-0-12-0
|
|
19
|
+
// TODO: We are on version 0.13.x+ now, so we should be able to get rid of this. - WSM 2026-01-14
|
|
20
|
+
validate.validators.type = function (value, options, _key, _attributes) {
|
|
21
|
+
// allow empty values by default (needs to be checked by "presence" check)
|
|
22
|
+
if (value === null || typeof value === 'undefined') {
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
return validate.validators.type.checks[options](value) ? null : ' must be a ' + `'${options}' value`;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
validate.validators.type.checks = {
|
|
30
|
+
Object: function (value) {
|
|
31
|
+
return validate.isObject(value) && !validate.isArray(value);
|
|
32
|
+
},
|
|
33
|
+
Array: validate.isArray,
|
|
34
|
+
Integer: validate.isInteger,
|
|
35
|
+
Number: validate.isNumber,
|
|
36
|
+
String: validate.isString,
|
|
37
|
+
Date: validate.isDate,
|
|
38
|
+
Boolean: function (value) {
|
|
39
|
+
return typeof value === 'boolean';
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
validate.validators.hasValidFileExt = function (value, options) {
|
|
44
|
+
// allow non-string values by default (needs to be checked by "presence" and "type" checks)
|
|
45
|
+
if (!validate.isString(value)) {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
if (value === '') return `can't be blank`;
|
|
49
|
+
|
|
50
|
+
return options.filter((ext) => value.endsWith(ext)).length > 0
|
|
51
|
+
? null
|
|
52
|
+
: `must include one of the following valid file extensions - '${options.join("', '")}'`;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
module.exports = {
|
|
56
|
+
validateObject,
|
|
57
|
+
validateObjectAsync,
|
|
58
|
+
validateBySchema,
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
function validateObject(object, fileConstraints) {
|
|
62
|
+
if (!object || !fileConstraints) {
|
|
63
|
+
return new Error('validateObject parameters were null');
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
let validateResult = validate(object, fileConstraints, { format: 'flat' });
|
|
67
|
+
if (!validateResult) return null;
|
|
68
|
+
return new Error(validateResult);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Use this function for calls that support async/await
|
|
73
|
+
* @param object - the json object being validated
|
|
74
|
+
* @param fileConstraints - validation rules for the json object
|
|
75
|
+
* @returns {Promise<Error|null>}
|
|
76
|
+
*/
|
|
77
|
+
async function validateObjectAsync(object, fileConstraints) {
|
|
78
|
+
if (!object || !fileConstraints) {
|
|
79
|
+
return new Error('validateObject parameters were null');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
try {
|
|
83
|
+
await validate.async(object, fileConstraints, { format: 'flat' });
|
|
84
|
+
} catch (err) {
|
|
85
|
+
// unroll the array and make a full error message.
|
|
86
|
+
let msg = err.join(`,`);
|
|
87
|
+
return new Error(msg);
|
|
88
|
+
}
|
|
89
|
+
// If no error, just return null so this will behave as the non async version.
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
*
|
|
95
|
+
* @param {{}} object
|
|
96
|
+
* @param {Joi.ObjectSchema} schema
|
|
97
|
+
* @returns {*}
|
|
98
|
+
*/
|
|
99
|
+
function validateBySchema(object, schema) {
|
|
100
|
+
let result = schema.validate(object, { allowUnknown: true, abortEarly: false, errors: { wrap: { label: "'" } } });
|
|
101
|
+
|
|
102
|
+
if (result.error) {
|
|
103
|
+
return new Error(result.error.message);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.userCodeFolders = void 0;
|
|
37
|
-
exports.captureProfile = captureProfile;
|
|
38
|
-
/**
|
|
39
|
-
* This module is responsible for profiling threads so we can determine how much CPU usage can be attributed
|
|
40
|
-
* to user code, harper code, and individual "hot" functions
|
|
41
|
-
*/
|
|
42
|
-
const write_ts_1 = require("./write.js");
|
|
43
|
-
const environmentManager_js_1 = require("../../utility/environment/environmentManager.js");
|
|
44
|
-
const hdbTerms_js_1 = require("../../utility/hdbTerms.js");
|
|
45
|
-
const packageUtils_js_1 = require("../../utility/packageUtils.js");
|
|
46
|
-
const node_fs_1 = require("node:fs");
|
|
47
|
-
const pprof_1 = require("@datadog/pprof");
|
|
48
|
-
const log = __importStar(require("../../utility/logging/harper_logger.js"));
|
|
49
|
-
const basePath = (0, environmentManager_js_1.getHdbBasePath)();
|
|
50
|
-
exports.userCodeFolders = basePath ? [basePath] : [];
|
|
51
|
-
if (process.env.RUN_HDB_APP)
|
|
52
|
-
exports.userCodeFolders.push((0, node_fs_1.realpathSync)(process.env.RUN_HDB_APP));
|
|
53
|
-
let profilerTimer;
|
|
54
|
-
const SAMPLING_INTERVAL_IN_MICROSECONDS = 50000;
|
|
55
|
-
// TODO: Running this on the thread itself can be a problematic because the profiler snapshots are somewhat expensive
|
|
56
|
-
// (calling timeProfiler.stop and getting the large block of JSON and parsing it). This can take a 5ms or more
|
|
57
|
-
// which can have some impact on latency for users. However, the datadog profiler is much better than the node
|
|
58
|
-
// profiler, so we'll keep this for now.
|
|
59
|
-
(async () => {
|
|
60
|
-
if (exports.userCodeFolders.length === 0)
|
|
61
|
-
return;
|
|
62
|
-
// start the profiler
|
|
63
|
-
pprof_1.time.start({ intervalMicros: SAMPLING_INTERVAL_IN_MICROSECONDS });
|
|
64
|
-
const PROFILE_PERIOD = ((0, environmentManager_js_1.get)(hdbTerms_js_1.CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) ?? 60) * 1000;
|
|
65
|
-
if (PROFILE_PERIOD > 0) {
|
|
66
|
-
profilerTimer = setTimeout(() => {
|
|
67
|
-
captureProfile(PROFILE_PERIOD);
|
|
68
|
-
}, PROFILE_PERIOD).unref();
|
|
69
|
-
}
|
|
70
|
-
})();
|
|
71
|
-
async function captureProfile(delayToNextCapture = ((0, environmentManager_js_1.get)(hdbTerms_js_1.CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) ?? 60) * 1000) {
|
|
72
|
-
clearTimeout(profilerTimer);
|
|
73
|
-
const hitCountThreshold = 100;
|
|
74
|
-
const secondsPerHit = SAMPLING_INTERVAL_IN_MICROSECONDS / 1_000_000;
|
|
75
|
-
const locationById = new Map();
|
|
76
|
-
const fileNameById = new Map();
|
|
77
|
-
const samplesByLocationId = new Map();
|
|
78
|
-
let totalUserCount = 0;
|
|
79
|
-
let totalHarperCount = 0;
|
|
80
|
-
try {
|
|
81
|
-
const profile = pprof_1.time.stop(true);
|
|
82
|
-
const strings = profile.stringTable.strings;
|
|
83
|
-
for (let func of profile.function) {
|
|
84
|
-
fileNameById.set(func.id, strings[func.filename]);
|
|
85
|
-
}
|
|
86
|
-
for (let location of profile.location) {
|
|
87
|
-
locationById.set(location.id, location.line[0]);
|
|
88
|
-
}
|
|
89
|
-
for (const sample of profile.sample) {
|
|
90
|
-
getUserHitCount(sample);
|
|
91
|
-
}
|
|
92
|
-
(0, write_ts_1.recordAction)(totalHarperCount * secondsPerHit, 'cpu-usage', 'harper');
|
|
93
|
-
(0, write_ts_1.recordAction)(totalUserCount * secondsPerHit, 'cpu-usage', 'user');
|
|
94
|
-
for (let [locationId, sampleCount] of samplesByLocationId) {
|
|
95
|
-
if (sampleCount > hitCountThreshold) {
|
|
96
|
-
const location = locationById.get(locationId);
|
|
97
|
-
const locationName = fileNameById.get(location.functionId) + ':' + location.line;
|
|
98
|
-
(0, write_ts_1.recordAction)(sampleCount * secondsPerHit, 'cpu-usage', locationName);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
catch (error) {
|
|
103
|
-
log.error?.('analytics profiler error:', error);
|
|
104
|
-
}
|
|
105
|
-
finally {
|
|
106
|
-
// and start the profiler again
|
|
107
|
-
if (delayToNextCapture > 0) {
|
|
108
|
-
profilerTimer = setTimeout(() => {
|
|
109
|
-
captureProfile();
|
|
110
|
-
}, delayToNextCapture).unref();
|
|
111
|
-
}
|
|
112
|
-
else {
|
|
113
|
-
// somehow this can later get set to a negative number which causes big problems (high-frequency restarts of the profiler)
|
|
114
|
-
log.info?.('Profiling disabled');
|
|
115
|
-
pprof_1.time.stop();
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
// this traverses the nodes and returns the number of sampling hits for the sample and attributes it
|
|
119
|
-
// to harper or user code (as opposed to execution of things like node internal modules or native code)
|
|
120
|
-
function getUserHitCount(sample) {
|
|
121
|
-
// if we can assign to user code or harper code, do so
|
|
122
|
-
let recordedTopSample = false;
|
|
123
|
-
for (let locationId of sample.locationId) {
|
|
124
|
-
let fileName = fileNameById.get(locationById.get(locationId).functionId);
|
|
125
|
-
if (exports.userCodeFolders.some((userCodeFolder) => fileName.startsWith(userCodeFolder))) {
|
|
126
|
-
// the call frame location is in user code
|
|
127
|
-
const sampleCount = sample.value[0];
|
|
128
|
-
totalUserCount += sampleCount;
|
|
129
|
-
if (!recordedTopSample)
|
|
130
|
-
samplesByLocationId.set(locationId, (samplesByLocationId.get(locationId) ?? 0) + sampleCount);
|
|
131
|
-
return; // if the highest point in the call stack is in user code, we don't need to check the rest of the call stack, this "counts" as user execution
|
|
132
|
-
}
|
|
133
|
-
if (fileName.startsWith(packageUtils_js_1.PACKAGE_ROOT)) {
|
|
134
|
-
const sampleCount = sample.value[0];
|
|
135
|
-
totalHarperCount += sampleCount;
|
|
136
|
-
if (!recordedTopSample) {
|
|
137
|
-
samplesByLocationId.set(locationId, (samplesByLocationId.get(locationId) ?? 0) + sampleCount);
|
|
138
|
-
recordedTopSample = true;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
//# sourceMappingURL=profile.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"profile.js","sourceRoot":"","sources":["../../../resources/analytics/profile.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCA,wCAwEC;AA5GD;;;GAGG;AACH,yCAA0C;AAC1C,2FAAgG;AAChG,2DAA0D;AAC1D,mEAA6D;AAC7D,qCAAuC;AACvC,0CAAsD;AACtD,4EAA8D;AAI9D,MAAM,QAAQ,GAAG,IAAA,sCAAc,GAAE,CAAC;AACrB,QAAA,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW;IAAE,uBAAe,CAAC,IAAI,CAAC,IAAA,sBAAY,EAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;AAEzF,IAAI,aAAyC,CAAC;AAC9C,MAAM,iCAAiC,GAAG,KAAK,CAAC;AAChD,qHAAqH;AACrH,+GAA+G;AAC/G,+GAA+G;AAC/G,yCAAyC;AACzC,CAAC,KAAK,IAAI,EAAE;IACX,IAAI,uBAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACzC,qBAAqB;IACrB,YAAY,CAAC,KAAK,CAAC,EAAE,cAAc,EAAE,iCAAiC,EAAE,CAAC,CAAC;IAC1E,MAAM,cAAc,GAAG,CAAC,IAAA,2BAAM,EAAC,2BAAa,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;IACtF,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACxB,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;YAC/B,cAAc,CAAC,cAAc,CAAC,CAAC;QAChC,CAAC,EAAE,cAAc,CAAC,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;AACF,CAAC,CAAC,EAAE,CAAC;AAEE,KAAK,UAAU,cAAc,CACnC,kBAAkB,GAAG,CAAC,IAAA,2BAAM,EAAC,2BAAa,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI;IAEnF,YAAY,CAAC,aAAa,CAAC,CAAC;IAC5B,MAAM,iBAAiB,GAAG,GAAG,CAAC;IAC9B,MAAM,aAAa,GAAG,iCAAiC,GAAG,SAAS,CAAC;IACpE,MAAM,YAAY,GAAG,IAAI,GAAG,EAAe,CAAC;IAC5C,MAAM,YAAY,GAAG,IAAI,GAAG,EAAe,CAAC;IAC5C,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtD,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;QAC5C,KAAK,IAAI,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACnC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAY,EAAE,OAAO,CAAC,IAAI,CAAC,QAAkB,CAAC,CAAC,CAAC;QACvE,CAAC;QACD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACrC,eAAe,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QACD,IAAA,uBAAY,EAAC,gBAAgB,GAAG,aAAa,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;QACtE,IAAA,uBAAY,EAAC,cAAc,GAAG,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QAClE,KAAK,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,IAAI,mBAAmB,EAAE,CAAC;YAC3D,IAAI,WAAW,GAAG,iBAAiB,EAAE,CAAC;gBACrC,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACjF,IAAA,uBAAY,EAAC,WAAW,GAAG,aAAa,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;YACtE,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,GAAG,CAAC,KAAK,EAAE,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;YAAS,CAAC;QACV,+BAA+B;QAC/B,IAAI,kBAAkB,GAAG,CAAC,EAAE,CAAC;YAC5B,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC/B,cAAc,EAAE,CAAC;YAClB,CAAC,EAAE,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;aAAM,CAAC;YACP,0HAA0H;YAC1H,GAAG,CAAC,IAAI,EAAE,CAAC,oBAAoB,CAAC,CAAC;YACjC,YAAY,CAAC,IAAI,EAAE,CAAC;QACrB,CAAC;IACF,CAAC;IACD,oGAAoG;IACpG,uGAAuG;IACvG,SAAS,eAAe,CAAC,MAAc;QACtC,sDAAsD;QACtD,IAAI,iBAAiB,GAAG,KAAK,CAAC;QAC9B,KAAK,IAAI,UAAU,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAC1C,IAAI,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC,CAAC;YACzE,IAAI,uBAAe,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;gBACnF,0CAA0C;gBAC1C,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpC,cAAc,IAAI,WAAW,CAAC;gBAC9B,IAAI,CAAC,iBAAiB;oBACrB,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;gBAC/F,OAAO,CAAC,6IAA6I;YACtJ,CAAC;YACD,IAAI,QAAQ,CAAC,UAAU,CAAC,8BAAY,CAAC,EAAE,CAAC;gBACvC,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpC,gBAAgB,IAAI,WAAW,CAAC;gBAChC,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBACxB,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;oBAC9F,iBAAiB,GAAG,IAAI,CAAC;gBAC1B,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC"}
|