@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/BinObjects.js +17 -0
- package/bin/cliOperations.js +157 -0
- package/bin/copyDb.ts +280 -0
- package/bin/harper.js +156 -0
- package/bin/install.js +15 -0
- package/bin/lite.js +5 -0
- package/bin/restart.js +201 -0
- package/bin/run.js +409 -0
- package/bin/status.js +65 -0
- package/bin/stop.js +22 -0
- package/bin/upgrade.js +134 -0
- package/components/Application.ts +646 -0
- package/components/ApplicationScope.ts +49 -0
- package/components/Component.ts +53 -0
- package/components/ComponentV1.ts +342 -0
- package/components/DEFAULT_CONFIG.ts +18 -0
- package/components/EntryHandler.ts +227 -0
- package/components/Logger.ts +14 -0
- package/components/OptionsWatcher.ts +354 -0
- package/components/PluginModule.ts +6 -0
- package/components/Scope.ts +329 -0
- package/components/componentLoader.ts +529 -0
- package/components/deriveCommonPatternBase.ts +31 -0
- package/components/deriveGlobOptions.ts +44 -0
- package/components/deriveURLPath.ts +57 -0
- package/components/operations.js +658 -0
- package/components/operationsValidation.js +246 -0
- package/components/packageComponent.ts +39 -0
- package/components/requestRestart.ts +26 -0
- package/components/resolveBaseURLPath.ts +38 -0
- package/components/status/ComponentStatus.ts +110 -0
- package/components/status/ComponentStatusRegistry.ts +251 -0
- package/components/status/api.ts +153 -0
- package/components/status/crossThread.ts +405 -0
- package/components/status/errors.ts +152 -0
- package/components/status/index.ts +44 -0
- package/components/status/internal.ts +65 -0
- package/components/status/registry.ts +12 -0
- package/components/status/types.ts +96 -0
- package/config/RootConfigWatcher.ts +59 -0
- package/config/configHelpers.ts +11 -0
- package/config/configUtils.js +967 -0
- package/config/harperConfigEnvVars.ts +641 -0
- package/dataLayer/CreateAttributeObject.js +25 -0
- package/dataLayer/CreateTableObject.js +11 -0
- package/dataLayer/DataLayerObjects.js +43 -0
- package/dataLayer/DeleteBeforeObject.js +22 -0
- package/dataLayer/DeleteObject.js +25 -0
- package/dataLayer/DropAttributeObject.js +11 -0
- package/dataLayer/GetBackupObject.js +22 -0
- package/dataLayer/InsertObject.js +24 -0
- package/dataLayer/ReadAuditLogObject.js +24 -0
- package/dataLayer/SQLSearch.js +1335 -0
- package/dataLayer/SearchByConditionsObject.js +61 -0
- package/dataLayer/SearchByHashObject.js +21 -0
- package/dataLayer/SearchObject.js +45 -0
- package/dataLayer/SqlSearchObject.js +14 -0
- package/dataLayer/UpdateObject.js +23 -0
- package/dataLayer/UpsertObject.js +23 -0
- package/dataLayer/bulkLoad.js +813 -0
- package/dataLayer/dataObjects/BulkLoadObjects.js +27 -0
- package/dataLayer/dataObjects/UpsertObject.js +23 -0
- package/dataLayer/delete.js +164 -0
- package/dataLayer/export.js +381 -0
- package/dataLayer/getBackup.js +40 -0
- package/dataLayer/harperBridge/BridgeMethods.js +81 -0
- package/dataLayer/harperBridge/ResourceBridge.ts +633 -0
- package/dataLayer/harperBridge/bridgeUtility/insertUpdateReturnObj.js +28 -0
- package/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +88 -0
- package/dataLayer/harperBridge/harperBridge.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/LMDBBridge.js +119 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +19 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +112 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +67 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +31 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +94 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +98 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +89 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +109 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +107 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +137 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +35 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +111 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +28 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +29 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +207 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +156 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +30 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +19 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +64 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +70 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +22 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +23 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +22 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBTransactionObject.js +23 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +24 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +24 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +25 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +157 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +94 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +39 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +34 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +100 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +371 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +109 -0
- package/dataLayer/hdbInfoController.js +254 -0
- package/dataLayer/insert.js +266 -0
- package/dataLayer/readAuditLog.js +59 -0
- package/dataLayer/schema.js +366 -0
- package/dataLayer/schemaDescribe.js +289 -0
- package/dataLayer/search.js +60 -0
- package/dataLayer/transaction.js +17 -0
- package/dataLayer/update.js +124 -0
- package/dist/components/Logger.d.ts +12 -0
- package/dist/components/Logger.js +3 -0
- package/dist/components/Logger.js.map +1 -0
- package/dist/components/Scope.d.ts +14 -4
- package/dist/components/Scope.js +18 -10
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/componentLoader.js +16 -9
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/operations.js +2 -2
- package/dist/components/operations.js.map +1 -1
- package/dist/config/configUtils.d.ts +1 -1
- package/dist/config/configUtils.js +1 -1
- package/dist/config/configUtils.js.map +1 -1
- package/dist/dataLayer/CreateTableObject.d.ts +2 -2
- package/dist/dataLayer/CreateTableObject.js +2 -2
- package/dist/dataLayer/CreateTableObject.js.map +1 -1
- package/dist/dataLayer/delete.d.ts +1 -1
- package/dist/dataLayer/schema.js +6 -5
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +1 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/resources/DatabaseTransaction.d.ts +1 -1
- package/dist/resources/IterableEventQueue.d.ts +1 -1
- package/dist/resources/LMDBTransaction.d.ts +5 -1
- package/dist/resources/Resource.d.ts +1 -1
- package/dist/resources/RocksIndexStore.d.ts +3 -3
- package/dist/resources/RocksTransactionLogStore.d.ts +6 -3
- package/dist/resources/Table.d.ts +15 -6
- package/dist/resources/Table.js +4 -1
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/read.js +32 -22
- package/dist/resources/analytics/read.js.map +1 -1
- package/dist/resources/analytics/write.js +3 -6
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +3 -3
- package/dist/resources/blob.d.ts +25 -2
- package/dist/resources/databases.d.ts +12 -2
- package/dist/resources/databases.js +22 -19
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/search.js +11 -5
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transaction.d.ts +2 -1
- package/dist/security/auth.js +1 -1
- package/dist/security/auth.js.map +1 -1
- package/dist/security/cryptoHash.d.ts +2 -2
- package/dist/security/jsLoader.js +243 -66
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +4 -5
- package/dist/security/keys.js.map +1 -1
- package/dist/security/user.js +3 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/REST.js +16 -2
- package/dist/server/REST.js.map +1 -1
- package/dist/server/Server.d.ts +2 -1
- package/dist/server/Server.js.map +1 -1
- package/dist/server/fastifyRoutes/plugins/hdbCore.d.ts +6 -1
- package/dist/server/fastifyRoutes.js +2 -0
- package/dist/server/fastifyRoutes.js.map +1 -1
- package/dist/server/http.js +12 -6
- package/dist/server/http.js.map +1 -1
- package/dist/server/jobs/JobObject.d.ts +3 -3
- package/dist/server/loadRootComponents.js +1 -0
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/operationsServer.js +3 -1
- package/dist/server/operationsServer.js.map +1 -1
- package/dist/server/serverHelpers/JSONStream.d.ts +3 -3
- package/dist/server/serverHelpers/Request.d.ts +5 -5
- package/dist/server/serverHelpers/requestTimePlugin.d.ts +1 -1
- package/dist/server/threads/manageThreads.d.ts +2 -2
- package/dist/server/threads/manageThreads.js +50 -35
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.d.ts +1 -1
- package/dist/sqlTranslator/deleteTranslator.d.ts +1 -1
- package/dist/utility/AWS/AWSConnector.d.ts +3 -2
- package/dist/utility/common_utils.d.ts +3 -3
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/functions/date/dateFunctions.d.ts +11 -11
- package/dist/utility/globalSchema.d.ts +1 -1
- package/dist/utility/hdbTerms.d.ts +3 -0
- package/dist/utility/hdbTerms.js +3 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/installation.d.ts +2 -4
- package/dist/utility/installation.js.map +1 -1
- package/dist/utility/lmdb/commonUtility.d.ts +1 -0
- package/dist/utility/lmdb/deleteUtility.d.ts +1 -0
- package/dist/utility/lmdb/environmentUtility.d.ts +1 -0
- package/dist/utility/lmdb/searchUtility.d.ts +2 -1
- package/dist/utility/lmdb/writeUtility.d.ts +1 -0
- package/dist/utility/logging/harper_logger.d.ts +6 -6
- package/dist/utility/processManagement/processManagement.d.ts +1 -1
- package/dist/utility/processManagement/servicesConfig.d.ts +12 -6
- package/dist/validation/common_validators.d.ts +4 -3
- package/dist/validation/configValidator.d.ts +3 -2
- package/index.d.ts +56 -0
- package/index.js +41 -0
- package/json/systemSchema.json +373 -0
- package/launchServiceScripts/launchHarperDB.js +3 -0
- package/launchServiceScripts/utility/checkNodeVersion.js +15 -0
- package/package.json +21 -3
- package/resources/DatabaseTransaction.ts +378 -0
- package/resources/ErrorResource.ts +57 -0
- package/resources/IterableEventQueue.ts +94 -0
- package/resources/LMDBTransaction.ts +349 -0
- package/resources/RecordEncoder.ts +702 -0
- package/resources/RequestTarget.ts +134 -0
- package/resources/Resource.ts +789 -0
- package/resources/ResourceInterface.ts +221 -0
- package/resources/ResourceInterfaceV2.ts +53 -0
- package/resources/ResourceV2.ts +67 -0
- package/resources/Resources.ts +162 -0
- package/resources/RocksIndexStore.ts +70 -0
- package/resources/RocksTransactionLogStore.ts +352 -0
- package/resources/Table.ts +4527 -0
- package/resources/analytics/hostnames.ts +72 -0
- package/resources/analytics/metadata.ts +10 -0
- package/resources/analytics/read.ts +252 -0
- package/resources/analytics/write.ts +803 -0
- package/resources/auditStore.ts +556 -0
- package/resources/blob.ts +1268 -0
- package/resources/crdt.ts +125 -0
- package/resources/dataLoader.ts +527 -0
- package/resources/databases.ts +1290 -0
- package/resources/graphql.ts +221 -0
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +638 -0
- package/resources/indexes/customIndexes.ts +7 -0
- package/resources/indexes/vector.ts +38 -0
- package/resources/jsResource.ts +86 -0
- package/resources/loadEnv.ts +22 -0
- package/resources/login.ts +18 -0
- package/resources/openApi.ts +409 -0
- package/resources/registrationDeprecated.ts +8 -0
- package/resources/replayLogs.ts +136 -0
- package/resources/roles.ts +98 -0
- package/resources/search.ts +1301 -0
- package/resources/tracked.ts +584 -0
- package/resources/transaction.ts +89 -0
- package/resources/transactionBroadcast.ts +258 -0
- package/security/auth.ts +376 -0
- package/security/certificateVerification/certificateVerificationSource.ts +84 -0
- package/security/certificateVerification/configValidation.ts +107 -0
- package/security/certificateVerification/crlVerification.ts +623 -0
- package/security/certificateVerification/index.ts +121 -0
- package/security/certificateVerification/ocspVerification.ts +148 -0
- package/security/certificateVerification/pkijs-ed25519-patch.ts +188 -0
- package/security/certificateVerification/types.ts +128 -0
- package/security/certificateVerification/verificationConfig.ts +138 -0
- package/security/certificateVerification/verificationUtils.ts +447 -0
- package/security/cryptoHash.js +42 -0
- package/security/data_objects/PermissionAttributeResponseObject.js +15 -0
- package/security/data_objects/PermissionResponseObject.js +115 -0
- package/security/data_objects/PermissionTableResponseObject.js +20 -0
- package/security/fastifyAuth.js +169 -0
- package/security/impersonation.ts +160 -0
- package/security/jsLoader.ts +716 -0
- package/security/keys.js +948 -0
- package/security/permissionsTranslator.js +300 -0
- package/security/role.js +218 -0
- package/security/tokenAuthentication.ts +228 -0
- package/security/user.ts +449 -0
- package/server/DurableSubscriptionsSession.ts +503 -0
- package/server/REST.ts +407 -0
- package/server/Server.ts +89 -0
- package/server/fastifyRoutes/helpers/getCORSOptions.js +36 -0
- package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +15 -0
- package/server/fastifyRoutes/helpers/getServerOptions.js +33 -0
- package/server/fastifyRoutes/plugins/hdbCore.js +39 -0
- package/server/fastifyRoutes.ts +205 -0
- package/server/graphqlQuerying.ts +700 -0
- package/server/http.ts +640 -0
- package/server/itc/serverHandlers.js +161 -0
- package/server/itc/utility/ITCEventObject.js +10 -0
- package/server/jobs/JobObject.js +24 -0
- package/server/jobs/jobProcess.js +69 -0
- package/server/jobs/jobRunner.js +162 -0
- package/server/jobs/jobs.js +304 -0
- package/server/loadRootComponents.js +44 -0
- package/server/mqtt.ts +485 -0
- package/server/nodeName.ts +75 -0
- package/server/operationsServer.ts +313 -0
- package/server/serverHelpers/Headers.ts +108 -0
- package/server/serverHelpers/JSONStream.ts +269 -0
- package/server/serverHelpers/OperationFunctionObject.ts +13 -0
- package/server/serverHelpers/Request.ts +158 -0
- package/server/serverHelpers/contentTypes.ts +637 -0
- package/server/serverHelpers/requestTimePlugin.js +57 -0
- package/server/serverHelpers/serverHandlers.js +148 -0
- package/server/serverHelpers/serverUtilities.ts +473 -0
- package/server/serverRegistry.ts +8 -0
- package/server/static.ts +187 -0
- package/server/status/definitions.ts +37 -0
- package/server/status/index.ts +125 -0
- package/server/storageReclamation.ts +93 -0
- package/server/threads/itc.js +89 -0
- package/server/threads/manageThreads.js +594 -0
- package/server/threads/socketRouter.ts +360 -0
- package/server/threads/threadServer.js +279 -0
- package/server/throttle.ts +73 -0
- package/sqlTranslator/SelectValidator.js +330 -0
- package/sqlTranslator/alasqlFunctionImporter.js +62 -0
- package/sqlTranslator/deleteTranslator.js +67 -0
- package/sqlTranslator/index.js +242 -0
- package/sqlTranslator/sql_statement_bucket.js +472 -0
- package/static/defaultConfig.yaml +3 -0
- package/studio/web/HDBDogOnly.svg +78 -0
- package/studio/web/assets/PPRadioGrotesk-Bold-DDaUYG8E.woff +0 -0
- package/studio/web/assets/fa-brands-400-CEJbCg16.woff +0 -0
- package/studio/web/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
- package/studio/web/assets/fa-brands-400-DnkPfk3o.eot +0 -0
- package/studio/web/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
- package/studio/web/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
- package/studio/web/assets/fa-regular-400-BhTwtT8w.eot +0 -0
- package/studio/web/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
- package/studio/web/assets/fa-regular-400-DFnMcJPd.woff +0 -0
- package/studio/web/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
- package/studio/web/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
- package/studio/web/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
- package/studio/web/assets/fa-solid-900-B6Axprfb.eot +0 -0
- package/studio/web/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
- package/studio/web/assets/fa-solid-900-DOXgCApm.woff +0 -0
- package/studio/web/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
- package/studio/web/assets/index-BTgXJX9d.js +235 -0
- package/studio/web/assets/index-BTgXJX9d.js.map +1 -0
- package/studio/web/assets/index-C-GXfcup.js +37 -0
- package/studio/web/assets/index-C-GXfcup.js.map +1 -0
- package/studio/web/assets/index-PFlNdimM.js +2 -0
- package/studio/web/assets/index-PFlNdimM.js.map +1 -0
- package/studio/web/assets/index-Y2g_iFpU.css +1 -0
- package/studio/web/assets/index-jiPwkrsB.css +1 -0
- package/studio/web/assets/index.lazy-C3TJZJ4o.js +266 -0
- package/studio/web/assets/index.lazy-C3TJZJ4o.js.map +1 -0
- package/studio/web/assets/profiler-DotzgiCJ.js +2 -0
- package/studio/web/assets/profiler-DotzgiCJ.js.map +1 -0
- package/studio/web/assets/react-redux-VxUEx_mU.js +6 -0
- package/studio/web/assets/react-redux-VxUEx_mU.js.map +1 -0
- package/studio/web/assets/startRecording-B_9J9Csd.js +3 -0
- package/studio/web/assets/startRecording-B_9J9Csd.js.map +1 -0
- package/studio/web/fabric-signup-background.webp +0 -0
- package/studio/web/fabric-signup-text.png +0 -0
- package/studio/web/favicon_purple.png +0 -0
- package/studio/web/github-icon.svg +15 -0
- package/studio/web/harper-fabric_black.png +0 -0
- package/studio/web/harper-fabric_white.png +0 -0
- package/studio/web/harper-studio_white.png +0 -0
- package/studio/web/index.html +16 -0
- package/studio/web/running.css +148 -0
- package/studio/web/running.html +147 -0
- package/studio/web/running.js +111 -0
- package/upgrade/UpgradeObjects.js +13 -0
- package/upgrade/directives/directivesController.js +90 -0
- package/upgrade/directivesManager.js +139 -0
- package/upgrade/upgradePrompt.js +124 -0
- package/upgrade/upgradeUtilities.js +28 -0
- package/utility/AWS/AWSConnector.js +29 -0
- package/utility/OperationFunctionCaller.js +63 -0
- package/utility/assignCmdEnvVariables.js +62 -0
- package/utility/common_utils.js +867 -0
- package/utility/environment/environmentManager.js +208 -0
- package/utility/environment/systemInformation.js +355 -0
- package/utility/errors/commonErrors.js +267 -0
- package/utility/errors/hdbError.js +146 -0
- package/utility/functions/date/dateFunctions.js +65 -0
- package/utility/functions/geo.js +355 -0
- package/utility/functions/sql/alaSQLExtension.js +104 -0
- package/utility/globalSchema.js +35 -0
- package/utility/hdbTerms.ts +819 -0
- package/utility/install/checkJWTTokensExist.js +62 -0
- package/utility/install/harperdb.conf +15 -0
- package/utility/install/harperdb.service +14 -0
- package/utility/install/installer.js +635 -0
- package/utility/installation.ts +30 -0
- package/utility/lmdb/DBIDefinition.js +20 -0
- package/utility/lmdb/DeleteRecordsResponseObject.js +25 -0
- package/utility/lmdb/InsertRecordsResponseObject.js +22 -0
- package/utility/lmdb/OpenDBIObject.js +31 -0
- package/utility/lmdb/OpenEnvironmentObject.js +41 -0
- package/utility/lmdb/UpdateRecordsResponseObject.js +25 -0
- package/utility/lmdb/UpsertRecordsResponseObject.js +22 -0
- package/utility/lmdb/cleanLMDBMap.js +65 -0
- package/utility/lmdb/commonUtility.js +119 -0
- package/utility/lmdb/deleteUtility.js +128 -0
- package/utility/lmdb/environmentUtility.js +477 -0
- package/utility/lmdb/searchCursorFunctions.js +187 -0
- package/utility/lmdb/searchUtility.js +918 -0
- package/utility/lmdb/terms.js +57 -0
- package/utility/lmdb/writeUtility.js +407 -0
- package/utility/logging/harper_logger.js +876 -0
- package/utility/logging/logRotator.js +157 -0
- package/utility/logging/logger.ts +24 -0
- package/utility/logging/readLog.js +355 -0
- package/utility/logging/transactionLog.js +57 -0
- package/utility/mount_hdb.js +59 -0
- package/utility/npmUtilities.js +102 -0
- package/utility/operationPermissions.ts +112 -0
- package/utility/operation_authorization.js +836 -0
- package/utility/packageUtils.js +55 -0
- package/utility/password.ts +99 -0
- package/utility/processManagement/processManagement.js +187 -0
- package/utility/processManagement/servicesConfig.js +56 -0
- package/utility/scripts/restartHdb.js +24 -0
- package/utility/scripts/user_data.sh +13 -0
- package/utility/signalling.js +36 -0
- package/utility/terms/certificates.js +81 -0
- package/utility/when.ts +20 -0
- package/v1.d.ts +39 -0
- package/v1.js +41 -0
- package/v2.d.ts +39 -0
- package/v2.js +41 -0
- package/validation/bulkDeleteValidator.js +24 -0
- package/validation/check_permissions.js +19 -0
- package/validation/common_validators.js +95 -0
- package/validation/configValidator.js +331 -0
- package/validation/deleteValidator.js +15 -0
- package/validation/fileLoadValidator.js +153 -0
- package/validation/insertValidator.js +40 -0
- package/validation/installValidator.js +37 -0
- package/validation/readLogValidator.js +64 -0
- package/validation/role_validation.js +320 -0
- package/validation/schemaMetadataValidator.js +42 -0
- package/validation/searchValidator.js +166 -0
- package/validation/statusValidator.ts +66 -0
- package/validation/transactionLogValidator.js +33 -0
- package/validation/user_validation.js +55 -0
- package/validation/validationWrapper.js +105 -0
- package/dist/resources/analytics/profile.d.ts +0 -2
- package/dist/resources/analytics/profile.js +0 -144
- package/dist/resources/analytics/profile.js.map +0 -1
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
import cluster from 'cluster';
|
|
2
|
+
import zlib from 'node:zlib';
|
|
3
|
+
import env from '../utility/environment/environmentManager.js';
|
|
4
|
+
env.initSync();
|
|
5
|
+
import * as terms from '../utility/hdbTerms.ts';
|
|
6
|
+
import harperLogger from '../utility/logging/harper_logger.js';
|
|
7
|
+
import fastify, { FastifyInstance, FastifyReply, FastifyRequest, FastifyServerOptions } from 'fastify';
|
|
8
|
+
import fastifyCors, { type FastifyCorsOptions } from '@fastify/cors';
|
|
9
|
+
import fastifyCompress from '@fastify/compress';
|
|
10
|
+
import fastifyStatic from '@fastify/static';
|
|
11
|
+
import requestTimePlugin from './serverHelpers/requestTimePlugin.js';
|
|
12
|
+
import guidePath from 'path';
|
|
13
|
+
import { PACKAGE_ROOT } from '../utility/packageUtils.js';
|
|
14
|
+
import globalSchema from '../utility/globalSchema.js';
|
|
15
|
+
import commonUtils from '../utility/common_utils.js';
|
|
16
|
+
import * as userSchema from '../security/user.ts';
|
|
17
|
+
import { server as serverRegistration, type ServerOptions } from '../server/Server.ts';
|
|
18
|
+
import {
|
|
19
|
+
authHandler,
|
|
20
|
+
authAndEnsureUserOnRequest,
|
|
21
|
+
handlePostRequest,
|
|
22
|
+
serverErrorHandler,
|
|
23
|
+
reqBodyValidationHandler,
|
|
24
|
+
} from './serverHelpers/serverHandlers.js';
|
|
25
|
+
import { registerContentHandlers } from './serverHelpers/contentTypes.ts';
|
|
26
|
+
import type { OperationFunctionName } from './serverHelpers/serverUtilities.ts';
|
|
27
|
+
import type { ParsedSqlObject } from '../sqlTranslator/index.js';
|
|
28
|
+
import { generateJsonApi } from '../resources/openApi.ts';
|
|
29
|
+
import { Resources } from '../resources/Resources.ts';
|
|
30
|
+
import { ServerError } from '../utility/errors/hdbError.js';
|
|
31
|
+
|
|
32
|
+
const DEFAULT_HEADERS_TIMEOUT = 60000;
|
|
33
|
+
const REQ_MAX_BODY_SIZE = env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_MAXREQUESTBODYSIZE) ?? 1024 * 1024 * 1024; //this defaults to 1GB in bytes
|
|
34
|
+
const TRUE_COMPARE_VAL = 'TRUE';
|
|
35
|
+
|
|
36
|
+
const { CONFIG_PARAMS } = terms;
|
|
37
|
+
let server;
|
|
38
|
+
|
|
39
|
+
export { operationsServer as hdbServer };
|
|
40
|
+
export { operationsServer as start };
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Builds a Harper server.
|
|
44
|
+
*/
|
|
45
|
+
async function operationsServer(options: ServerOptions & { resources?: Resources }) {
|
|
46
|
+
try {
|
|
47
|
+
harperLogger.debug('In Fastify server' + process.cwd());
|
|
48
|
+
harperLogger.debug(`Running with NODE_ENV set as: ${process.env.NODE_ENV}`);
|
|
49
|
+
harperLogger.debug(`Harper server process ${process.pid} starting up.`);
|
|
50
|
+
|
|
51
|
+
global.clustering_on = false;
|
|
52
|
+
global.isMaster = cluster.isMaster;
|
|
53
|
+
|
|
54
|
+
await setUp();
|
|
55
|
+
// if we have a secure port, need to use the secure HTTP server for fastify (it can be used for HTTP as well)
|
|
56
|
+
const isHttps = options.securePort > 0;
|
|
57
|
+
|
|
58
|
+
//generate a Fastify server instance
|
|
59
|
+
server = buildServer(isHttps, options.resources);
|
|
60
|
+
|
|
61
|
+
//make sure the process waits for the server to be fully instantiated before moving forward
|
|
62
|
+
await server.ready();
|
|
63
|
+
if (!options) options = {};
|
|
64
|
+
options.usageType = 'operations-api';
|
|
65
|
+
// fastify can't clean up properly
|
|
66
|
+
try {
|
|
67
|
+
// now that server is fully loaded/ready, start listening on port provided in config settings or just use
|
|
68
|
+
// zero to wait for sockets from the main thread
|
|
69
|
+
serverRegistration.http(server.server, options);
|
|
70
|
+
if (!server.server.closeIdleConnections) {
|
|
71
|
+
// before Node v18, closeIdleConnections is not available, and we have to setup a listener for fastify
|
|
72
|
+
// to handle closing by setting up the dynamic port
|
|
73
|
+
await server.listen({ port: 0, host: '::' });
|
|
74
|
+
}
|
|
75
|
+
} catch (err) {
|
|
76
|
+
server.close();
|
|
77
|
+
harperLogger.error(err);
|
|
78
|
+
harperLogger.error(`Error configuring operations server`);
|
|
79
|
+
throw err;
|
|
80
|
+
}
|
|
81
|
+
} catch (err) {
|
|
82
|
+
console.error(`Failed to build server on ${process.pid}`, err);
|
|
83
|
+
harperLogger.fatal(err);
|
|
84
|
+
process.exit(1);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Makes sure global values are set and that clustering connections are set/ready before server starts.
|
|
90
|
+
*/
|
|
91
|
+
async function setUp() {
|
|
92
|
+
harperLogger.trace('Configuring Harper process.');
|
|
93
|
+
globalSchema.setSchemaDataToGlobal();
|
|
94
|
+
return userSchema.setUsersWithRolesCache();
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export interface ImpersonatePayload {
|
|
98
|
+
username?: string;
|
|
99
|
+
role?: {
|
|
100
|
+
permission: Partial<userSchema.UserRoleNamedPermissions & userSchema.UserRoleDatabasePermissions>;
|
|
101
|
+
};
|
|
102
|
+
role_name?: string;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
interface BaseOperationRequestBody {
|
|
106
|
+
operation: OperationFunctionName;
|
|
107
|
+
bypassAuth: boolean;
|
|
108
|
+
hdb_user?: userSchema.User;
|
|
109
|
+
hdbAuthHeader?: unknown;
|
|
110
|
+
bypass_auth?: boolean;
|
|
111
|
+
impersonate?: ImpersonatePayload;
|
|
112
|
+
password?: string;
|
|
113
|
+
payload?: string;
|
|
114
|
+
sql?: string;
|
|
115
|
+
parsedSqlObject?: ParsedSqlObject;
|
|
116
|
+
[key: string]: unknown;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
type SearchOperation = BaseOperationRequestBody;
|
|
120
|
+
|
|
121
|
+
interface SearchOperationRequestBody {
|
|
122
|
+
search_operation: SearchOperation;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export type OperationRequestBody = BaseOperationRequestBody & Partial<SearchOperationRequestBody>;
|
|
126
|
+
|
|
127
|
+
export interface OperationRequest {
|
|
128
|
+
body: OperationRequestBody;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export interface OperationResult {
|
|
132
|
+
message?: any;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* This method configures and returns a Fastify server - for either HTTP or HTTPS - based on the provided config settings
|
|
137
|
+
*/
|
|
138
|
+
function buildServer(isHttps: boolean, resources: Resources): FastifyInstance {
|
|
139
|
+
harperLogger.debug(`Harper process starting to build ${isHttps ? 'HTTPS' : 'HTTP'} server.`);
|
|
140
|
+
const serverOpts = getServerOptions(isHttps);
|
|
141
|
+
|
|
142
|
+
const app = fastify(serverOpts);
|
|
143
|
+
|
|
144
|
+
// Fastify does not set this property in the initial app construction
|
|
145
|
+
app.server.headersTimeout = getHeaderTimeoutConfig();
|
|
146
|
+
|
|
147
|
+
// Set a top-level error handler for the server - all errors caught/thrown within the API will bubble up to this
|
|
148
|
+
// handler so that they can be handled in a coordinated way
|
|
149
|
+
app.setErrorHandler(serverErrorHandler);
|
|
150
|
+
|
|
151
|
+
const corsOptions = getCORSOpts();
|
|
152
|
+
if (corsOptions) {
|
|
153
|
+
app.register(fastifyCors, corsOptions);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
app.register(function (instance, options, done) {
|
|
157
|
+
instance.setNotFoundHandler(function (request, reply) {
|
|
158
|
+
if (reply.sent || reply.raw.headersSent || reply.raw.writableEnded) return;
|
|
159
|
+
app.server.emit('unhandled', request.raw, reply.raw);
|
|
160
|
+
});
|
|
161
|
+
done();
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
app.register(requestTimePlugin);
|
|
165
|
+
|
|
166
|
+
// This handles all get requests for the studio
|
|
167
|
+
app.register(fastifyCompress, {
|
|
168
|
+
brotliOptions: {
|
|
169
|
+
params: {
|
|
170
|
+
[zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT, // useful for APIs that primarily return text
|
|
171
|
+
[zlib.constants.BROTLI_PARAM_QUALITY]: 2, // default is 4, max is 11, min is 0
|
|
172
|
+
},
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
registerContentHandlers(app);
|
|
176
|
+
|
|
177
|
+
// Add a simple health check
|
|
178
|
+
app.get('/health', () => 'Harper is running.');
|
|
179
|
+
|
|
180
|
+
// Add a top-level GET handler for browsers.
|
|
181
|
+
app.register(fastifyStatic, { root: guidePath.join(PACKAGE_ROOT, 'studio/web') });
|
|
182
|
+
const studioOn = env.get(terms.HDB_SETTINGS_NAMES.LOCAL_STUDIO_ON);
|
|
183
|
+
if (!commonUtils.isEmpty(studioOn) && studioOn.toString().toLowerCase() === 'true') {
|
|
184
|
+
app.get('/', (req, res) => res.sendFile('index.html'));
|
|
185
|
+
} else {
|
|
186
|
+
app.get('/', (req, res) => res.sendFile('running.html'));
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// Describe the APIs.
|
|
190
|
+
app.get('/api/openapi/rest', { preValidation: [authAndEnsureUserOnRequest] }, restOpenAPIHandler(resources));
|
|
191
|
+
|
|
192
|
+
// Add the top-level POST handler.
|
|
193
|
+
app.post<{ Body: OperationRequestBody }, { isOperation?: boolean }>(
|
|
194
|
+
'/',
|
|
195
|
+
{
|
|
196
|
+
preValidation: [reqBodyValidationHandler, authHandler],
|
|
197
|
+
config: { isOperation: true },
|
|
198
|
+
},
|
|
199
|
+
handler
|
|
200
|
+
);
|
|
201
|
+
|
|
202
|
+
harperLogger.debug(`Harper process starting up ${isHttps ? 'HTTPS' : 'HTTP'} server listener.`);
|
|
203
|
+
|
|
204
|
+
return app;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function restOpenAPIHandler(resources: Resources) {
|
|
208
|
+
const httpPort = env.get(terms.CONFIG_PARAMS.HTTP_PORT);
|
|
209
|
+
const httpSecurePort = env.get(terms.CONFIG_PARAMS.HTTP_SECUREPORT);
|
|
210
|
+
return (req: FastifyRequest & { hdb_user?: { role?: { permission?: { super_user: boolean } } } }) => {
|
|
211
|
+
if (req.hdb_user?.role?.permission?.super_user) {
|
|
212
|
+
return generateJsonApi(resources, calculateRestHttpURL(httpPort, httpSecurePort, req));
|
|
213
|
+
} else {
|
|
214
|
+
harperLogger.warn(
|
|
215
|
+
`{"ip":"${req.socket.remoteAddress}", "error":"attempt to access /api/openapi/rest without being super_user"`
|
|
216
|
+
);
|
|
217
|
+
return new ServerError(`Forbidden`, 403);
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export function calculateRestHttpURL(
|
|
223
|
+
httpPort: string | undefined,
|
|
224
|
+
httpSecurePort: string | undefined,
|
|
225
|
+
req: { hostname: string; protocol: string }
|
|
226
|
+
): string {
|
|
227
|
+
const httpURL = new URL(`${req.protocol}://${req.hostname}`);
|
|
228
|
+
// note that the request is from fastify, which doesn't seem to have a correct hostname property (includes port), so the URL needs to be used for hostname
|
|
229
|
+
if (httpURL.hostname.toLowerCase() === 'localhost' || httpURL.hostname.match(/^[\d.:]+$/)) {
|
|
230
|
+
// Only use ports when running against localhost, or an ip address.
|
|
231
|
+
if (httpSecurePort) {
|
|
232
|
+
httpURL.port = httpSecurePort;
|
|
233
|
+
httpURL.protocol = 'https:';
|
|
234
|
+
} else if (httpPort) {
|
|
235
|
+
httpURL.port = httpPort;
|
|
236
|
+
httpURL.protocol = 'http:';
|
|
237
|
+
}
|
|
238
|
+
} else {
|
|
239
|
+
// Otherwise, assume that port forwarding is happening, and possibly SSL termination.
|
|
240
|
+
httpURL.port = '443';
|
|
241
|
+
httpURL.protocol = 'https:';
|
|
242
|
+
}
|
|
243
|
+
return httpURL.toString();
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function handler(req: FastifyRequest<{ Body?: OperationRequestBody }>, reply: FastifyReply) {
|
|
247
|
+
// if the operation is a restart, we have to tell the client not to use keep alive on this connection
|
|
248
|
+
// anymore; it needs to be closed because this thread is going to be terminated
|
|
249
|
+
if (req.body?.operation?.startsWith('restart')) {
|
|
250
|
+
reply.header('Connection', 'close');
|
|
251
|
+
}
|
|
252
|
+
//if no error is thrown below, the response 'data' returned from the handler will be returned with 200/OK code
|
|
253
|
+
return handlePostRequest(req, reply);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
interface HttpServerOptions extends FastifyServerOptions {
|
|
257
|
+
https?: boolean;
|
|
258
|
+
http2?: boolean;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Builds server options object to pass to Fastify when using server factory.
|
|
263
|
+
*/
|
|
264
|
+
function getServerOptions(isHttps: boolean): HttpServerOptions {
|
|
265
|
+
const server_timeout = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_TIMEOUT);
|
|
266
|
+
const keep_alive_timeout = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_KEEPALIVETIMEOUT);
|
|
267
|
+
return {
|
|
268
|
+
bodyLimit: REQ_MAX_BODY_SIZE,
|
|
269
|
+
connectionTimeout: server_timeout,
|
|
270
|
+
keepAliveTimeout: keep_alive_timeout,
|
|
271
|
+
forceCloseConnections: true,
|
|
272
|
+
return503OnClosing: false,
|
|
273
|
+
http2: env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_HTTP2),
|
|
274
|
+
https: isHttps /* && {
|
|
275
|
+
allowHTTP1: true,
|
|
276
|
+
},*/,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Builds CORS options object to pass to cors plugin when/if it needs to be registered with Fastify
|
|
282
|
+
*/
|
|
283
|
+
function getCORSOpts(): FastifyCorsOptions {
|
|
284
|
+
const propsCors = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORS);
|
|
285
|
+
const propsCorsAccesslist = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST);
|
|
286
|
+
let corsOptions: FastifyCorsOptions;
|
|
287
|
+
|
|
288
|
+
if (propsCors && (propsCors === true || propsCors.toUpperCase() === TRUE_COMPARE_VAL)) {
|
|
289
|
+
corsOptions = {
|
|
290
|
+
origin: true,
|
|
291
|
+
allowedHeaders: ['Content-Type', 'Authorization', 'Accept'],
|
|
292
|
+
credentials: false,
|
|
293
|
+
};
|
|
294
|
+
if (
|
|
295
|
+
propsCorsAccesslist &&
|
|
296
|
+
propsCorsAccesslist.length > 0 &&
|
|
297
|
+
propsCorsAccesslist[0] !== null &&
|
|
298
|
+
propsCorsAccesslist[0] !== '*'
|
|
299
|
+
) {
|
|
300
|
+
corsOptions.origin = (origin, callback) => {
|
|
301
|
+
return callback(null, propsCorsAccesslist.indexOf(origin) !== -1);
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
return corsOptions;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* Returns header timeout value from config file or, if not entered, the default value
|
|
310
|
+
*/
|
|
311
|
+
function getHeaderTimeoutConfig(): number {
|
|
312
|
+
return env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_HEADERSTIMEOUT) ?? DEFAULT_HEADERS_TIMEOUT;
|
|
313
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fast implementation of standard Headers
|
|
3
|
+
*/
|
|
4
|
+
export class Headers extends Map<string, string | string[]> {
|
|
5
|
+
constructor(init?: Headers | HeadersInit) {
|
|
6
|
+
if (init) {
|
|
7
|
+
if (init[Symbol.iterator]) {
|
|
8
|
+
super(init);
|
|
9
|
+
} else {
|
|
10
|
+
super();
|
|
11
|
+
for (const name in init) this.set(name, init[name]);
|
|
12
|
+
}
|
|
13
|
+
} else super();
|
|
14
|
+
}
|
|
15
|
+
set(name, value) {
|
|
16
|
+
if (typeof name !== 'string') name = '' + name;
|
|
17
|
+
if (Array.isArray(value)) {
|
|
18
|
+
// Ensure all array elements are strings
|
|
19
|
+
value = value.map((v) => (typeof v === 'string' ? v : '' + v));
|
|
20
|
+
} else if (typeof value !== 'string') {
|
|
21
|
+
value = '' + value;
|
|
22
|
+
}
|
|
23
|
+
return super.set(name.toLowerCase(), [name, value]);
|
|
24
|
+
}
|
|
25
|
+
get(name) {
|
|
26
|
+
if (typeof name !== 'string') name = '' + name;
|
|
27
|
+
return super.get(name.toLowerCase())?.[1];
|
|
28
|
+
}
|
|
29
|
+
has(name) {
|
|
30
|
+
if (typeof name !== 'string') name = '' + name;
|
|
31
|
+
return super.has(name.toLowerCase());
|
|
32
|
+
}
|
|
33
|
+
setIfNone(name, value) {
|
|
34
|
+
if (typeof name !== 'string') name = '' + name;
|
|
35
|
+
if (typeof value !== 'string') value = '' + value;
|
|
36
|
+
const lowerName = name.toLowerCase();
|
|
37
|
+
if (!super.has(lowerName)) return super.set(lowerName, [name, value]);
|
|
38
|
+
}
|
|
39
|
+
append(name, value, commaDelimited) {
|
|
40
|
+
if (typeof name !== 'string') name = '' + name;
|
|
41
|
+
if (typeof value !== 'string') value = '' + value;
|
|
42
|
+
const lowerName = name.toLowerCase();
|
|
43
|
+
const existing = super.get(lowerName);
|
|
44
|
+
if (existing) {
|
|
45
|
+
const existingValue = existing[1];
|
|
46
|
+
if (commaDelimited)
|
|
47
|
+
value = (typeof existingValue === 'string' ? existingValue : existingValue.join(', ')) + ', ' + value;
|
|
48
|
+
else if (typeof existingValue === 'string') value = [existingValue, value];
|
|
49
|
+
else {
|
|
50
|
+
existingValue.push(value);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return super.set(lowerName, [name, value]);
|
|
55
|
+
}
|
|
56
|
+
[Symbol.iterator]() {
|
|
57
|
+
return super.values()[Symbol.iterator]();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function appendHeader(headers, name, value, commaDelimited) {
|
|
62
|
+
if (headers.append) {
|
|
63
|
+
headers.append(name, value, commaDelimited);
|
|
64
|
+
} else if (headers.set) {
|
|
65
|
+
const existingValue = headers.get(name);
|
|
66
|
+
if (existingValue) {
|
|
67
|
+
if (commaDelimited)
|
|
68
|
+
value = (typeof existingValue === 'string' ? existingValue : existingValue.join(', ')) + ', ' + value;
|
|
69
|
+
else if (typeof existingValue === 'string') value = [existingValue, value];
|
|
70
|
+
else {
|
|
71
|
+
existingValue.push(value);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return headers.set(name, value);
|
|
76
|
+
} else {
|
|
77
|
+
headers[name] = (headers[name] ? headers[name] + ', ' : '') + value;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Merge headers from source into target, ensuring that target is a Headers object, and avoiding any overwrite
|
|
83
|
+
* of existing headers in target.
|
|
84
|
+
* @param target
|
|
85
|
+
* @param source
|
|
86
|
+
*/
|
|
87
|
+
export function mergeHeaders(target: any, source: Headers) {
|
|
88
|
+
// ensure target is a Headers object, which could be this Headers class, the global.Headers, or even a Map, which is ok
|
|
89
|
+
if (typeof target.set !== 'function' || typeof target.has !== 'function') target = new Headers(target);
|
|
90
|
+
for (const [name, value] of source) {
|
|
91
|
+
if (!target.has(name)) target.set(name, value);
|
|
92
|
+
else if (name.toLowerCase() === 'set-cookie') {
|
|
93
|
+
// Set-Cookie headers must NEVER be comma-delimited
|
|
94
|
+
// If value is an array, append each one separately; otherwise append the single value
|
|
95
|
+
const values = Array.isArray(value) ? value : [value];
|
|
96
|
+
if (target.append) {
|
|
97
|
+
for (const v of values) target.append(name, v);
|
|
98
|
+
} else {
|
|
99
|
+
// Fallback for Map or objects without append method
|
|
100
|
+
// We know existing exists because we're in the else-if branch (target.has(name) is true)
|
|
101
|
+
const existing = target.get(name);
|
|
102
|
+
const newValue = Array.isArray(existing) ? [...existing, ...values] : [existing, ...values];
|
|
103
|
+
target.set(name, newValue);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return target;
|
|
108
|
+
}
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import { Readable } from 'stream';
|
|
2
|
+
import * as harperLogger from '../../utility/logging/harper_logger.js';
|
|
3
|
+
import { when } from '../../utility/when.ts';
|
|
4
|
+
import JSONbig from 'json-bigint-fixes';
|
|
5
|
+
const JSONbigint = JSONbig({ useNativeBigInt: true });
|
|
6
|
+
const BUFFER_SIZE = 10000;
|
|
7
|
+
const BIGINT_SERIALIZATION = { message: 'Cannot serialize BigInt to JSON' };
|
|
8
|
+
BigInt.prototype.toJSON = function () {
|
|
9
|
+
throw BIGINT_SERIALIZATION;
|
|
10
|
+
};
|
|
11
|
+
const { errorToString } = harperLogger;
|
|
12
|
+
export function streamAsJSON(value) {
|
|
13
|
+
return new JSONStream({ value });
|
|
14
|
+
}
|
|
15
|
+
// a readable stream for serializing a set of variables to a JSON stream
|
|
16
|
+
class JSONStream extends Readable {
|
|
17
|
+
constructor(options) {
|
|
18
|
+
// Calls the stream.Readable(options) constructor
|
|
19
|
+
super(options);
|
|
20
|
+
this.buffer = [];
|
|
21
|
+
this.bufferSize = 0;
|
|
22
|
+
this.iterator = this.serialize(options.value, true);
|
|
23
|
+
this.activeIterators = [];
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
*serialize(object) {
|
|
27
|
+
// using a generator to serialize JSON for convenience of recursive pause and resume functionality
|
|
28
|
+
// serialize a value to an iterator that can be consumed by streaming API
|
|
29
|
+
if (object && typeof object === 'object') {
|
|
30
|
+
const hasAsyncIterator = object[Symbol.asyncIterator];
|
|
31
|
+
const hasIterator = object[Symbol.iterator];
|
|
32
|
+
if ((hasIterator || hasAsyncIterator) && !object.then) {
|
|
33
|
+
yield '[';
|
|
34
|
+
let first = true;
|
|
35
|
+
if ((hasAsyncIterator || hasIterator) && !(object instanceof Array)) {
|
|
36
|
+
let iterator = hasAsyncIterator ? object[Symbol.asyncIterator]() : object[Symbol.iterator]();
|
|
37
|
+
this.activeIterators.push(iterator);
|
|
38
|
+
let iteratorResult;
|
|
39
|
+
while (true) {
|
|
40
|
+
try {
|
|
41
|
+
iteratorResult = iterator.next();
|
|
42
|
+
if (iteratorResult.then) {
|
|
43
|
+
yield iteratorResult.then(
|
|
44
|
+
(result) => {
|
|
45
|
+
// don't send anything here, assign the iteratorResult and continue
|
|
46
|
+
// with the serialization loop
|
|
47
|
+
iteratorResult = result;
|
|
48
|
+
return ''; // return nothing now and serialization will continue below
|
|
49
|
+
},
|
|
50
|
+
(error) => {
|
|
51
|
+
// try to properly serialize the error, but then finish the iterator
|
|
52
|
+
harperLogger.warn('Error serializing in stream', error);
|
|
53
|
+
iteratorResult = { done: false, value: { error: errorToString(error) } };
|
|
54
|
+
iterator = {
|
|
55
|
+
next: () => ({ done: true }),
|
|
56
|
+
};
|
|
57
|
+
return '';
|
|
58
|
+
}
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
} catch (error) {
|
|
62
|
+
iteratorResult = { done: false, value: { error: errorToString(error) } };
|
|
63
|
+
iterator = {
|
|
64
|
+
next: () => ({ done: true }),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
if (iteratorResult.done) {
|
|
68
|
+
this.activeIterators.splice(this.activeIterators.indexOf(iterator), 1);
|
|
69
|
+
yield ']';
|
|
70
|
+
return;
|
|
71
|
+
} else {
|
|
72
|
+
if (first) {
|
|
73
|
+
first = false;
|
|
74
|
+
} else {
|
|
75
|
+
yield ',';
|
|
76
|
+
}
|
|
77
|
+
yield* this.serialize(iteratorResult.value);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
for (const element of object) {
|
|
82
|
+
if (first) first = false;
|
|
83
|
+
else {
|
|
84
|
+
yield ',';
|
|
85
|
+
}
|
|
86
|
+
yield* this.serialize(element);
|
|
87
|
+
}
|
|
88
|
+
yield ']';
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (object.then) {
|
|
92
|
+
try {
|
|
93
|
+
yield object.then((object) => this.serialize(object), handleError);
|
|
94
|
+
} catch (error) {
|
|
95
|
+
yield handleError(error);
|
|
96
|
+
}
|
|
97
|
+
} else {
|
|
98
|
+
yield stringify(object);
|
|
99
|
+
}
|
|
100
|
+
} else {
|
|
101
|
+
yield stringify(object);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
_read() {
|
|
106
|
+
if (this._amReading) {
|
|
107
|
+
// I don't know why _read is called from within a push call, but if we are already reading, ignore the call
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
this._amReading = true;
|
|
111
|
+
if (this.done) {
|
|
112
|
+
return this.push(null);
|
|
113
|
+
}
|
|
114
|
+
when(
|
|
115
|
+
this.readIterator(this.iterator),
|
|
116
|
+
(done) => {
|
|
117
|
+
if (done) {
|
|
118
|
+
this.done = true;
|
|
119
|
+
this.push(null);
|
|
120
|
+
} else {
|
|
121
|
+
this._amReading = false;
|
|
122
|
+
}
|
|
123
|
+
},
|
|
124
|
+
(error) => {
|
|
125
|
+
console.error(error);
|
|
126
|
+
this.done = true;
|
|
127
|
+
this.push(errorToString(error));
|
|
128
|
+
this.push(null);
|
|
129
|
+
}
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
push(content) {
|
|
134
|
+
if (content === null || content instanceof Buffer) {
|
|
135
|
+
if (this.bufferSize > 0) this.flush();
|
|
136
|
+
return super.push(content);
|
|
137
|
+
}
|
|
138
|
+
this.bufferSize += content.length || content.toString().length;
|
|
139
|
+
this.buffer.push(content);
|
|
140
|
+
if (this.bufferSize > BUFFER_SIZE) {
|
|
141
|
+
return this.flush();
|
|
142
|
+
}
|
|
143
|
+
return true;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
flush() {
|
|
147
|
+
const pushResult = super.push(this.buffer.join(''));
|
|
148
|
+
this.buffer = [];
|
|
149
|
+
this.bufferSize = 0;
|
|
150
|
+
return pushResult;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
readIterator(iterator) {
|
|
154
|
+
try {
|
|
155
|
+
// eventually we should be able to just put this around iterator.next()
|
|
156
|
+
let nextString;
|
|
157
|
+
if (iterator.childIterator) {
|
|
158
|
+
// resuming in a child iterator
|
|
159
|
+
return when(this.readIterator(iterator.childIterator), (done) => {
|
|
160
|
+
if (done) {
|
|
161
|
+
iterator.childIterator = null;
|
|
162
|
+
// continue on with the current iterator
|
|
163
|
+
return this.readIterator(iterator);
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
do {
|
|
168
|
+
const stepReturn = iterator.next();
|
|
169
|
+
if (stepReturn.done) {
|
|
170
|
+
return true;
|
|
171
|
+
}
|
|
172
|
+
nextString = stepReturn.value;
|
|
173
|
+
if (nextString == null) {
|
|
174
|
+
nextString = 'null';
|
|
175
|
+
} else {
|
|
176
|
+
if (nextString.then) {
|
|
177
|
+
this.flush();
|
|
178
|
+
return Promise.resolve(nextString).then((resolved) => {
|
|
179
|
+
if (resolved && typeof resolved.return === 'function') {
|
|
180
|
+
iterator.childIterator = resolved;
|
|
181
|
+
return this.readIterator(iterator);
|
|
182
|
+
} else if (this.push(resolved + '')) {
|
|
183
|
+
return this.readIterator(iterator);
|
|
184
|
+
} // else return false
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
if (typeof nextString.return === 'function') {
|
|
188
|
+
iterator.childIterator = nextString;
|
|
189
|
+
return this.readIterator(iterator);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
} while (this.push(nextString));
|
|
193
|
+
} catch (error) {
|
|
194
|
+
console.error(error);
|
|
195
|
+
this.push(errorToString(error));
|
|
196
|
+
this.push(null);
|
|
197
|
+
return true;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
_destroy(error, callback) {
|
|
202
|
+
for (const iterator of this.activeIterators) {
|
|
203
|
+
if (error) iterator.throw(error);
|
|
204
|
+
else iterator.return();
|
|
205
|
+
}
|
|
206
|
+
callback();
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function handleError(error) {
|
|
211
|
+
console.error(error);
|
|
212
|
+
return JSON.stringify(errorToString(error));
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
export function stringify(value) {
|
|
216
|
+
try {
|
|
217
|
+
return JSON.stringify(value) ?? 'null';
|
|
218
|
+
} catch (error) {
|
|
219
|
+
if (error === BIGINT_SERIALIZATION) {
|
|
220
|
+
return jsStringify(value);
|
|
221
|
+
}
|
|
222
|
+
if (error.resolution) {
|
|
223
|
+
return error.resolution.then(() => stringify(value));
|
|
224
|
+
}
|
|
225
|
+
throw error;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function jsStringify(value) {
|
|
230
|
+
const type = typeof value;
|
|
231
|
+
if (type === 'object') {
|
|
232
|
+
if (value === null) return 'null';
|
|
233
|
+
if (value.toJSON) value = value.toJSON();
|
|
234
|
+
let str;
|
|
235
|
+
if (Array.isArray(value)) {
|
|
236
|
+
str = '[';
|
|
237
|
+
for (let i = 0; i < value.length; i++) {
|
|
238
|
+
if (i > 0) str += ',';
|
|
239
|
+
// we continue to use jsStringify assuming that if one element has a BigInt, they all do
|
|
240
|
+
str += jsStringify(value[i]);
|
|
241
|
+
}
|
|
242
|
+
return str + ']';
|
|
243
|
+
} else {
|
|
244
|
+
str = '{';
|
|
245
|
+
let first = true;
|
|
246
|
+
for (const key in value) {
|
|
247
|
+
if (first) first = false;
|
|
248
|
+
else str += ',';
|
|
249
|
+
// we assume probably only one element has a BigInt, so we can use stringify for the rest
|
|
250
|
+
str += JSON.stringify(key) + ':' + stringify(value[key]);
|
|
251
|
+
}
|
|
252
|
+
return str + '}';
|
|
253
|
+
}
|
|
254
|
+
} else if (type === 'string') {
|
|
255
|
+
return JSON.stringify(value);
|
|
256
|
+
} else if (type === 'undefined') {
|
|
257
|
+
return 'null';
|
|
258
|
+
}
|
|
259
|
+
return value.toString(); // this handles bigint, number, boolean, undefined, symbol
|
|
260
|
+
}
|
|
261
|
+
const HAS_BIG_NUMBER = /[[,:]\s*-?\d{16,}/;
|
|
262
|
+
export function parse(json) {
|
|
263
|
+
if (!(json?.length > 0)) return null; // empty string or null
|
|
264
|
+
if (typeof json !== 'string') json = json.toString(); // make sure we just do this once
|
|
265
|
+
// we use JSONbig if there is a big number in the JSON, otherwise we use the native JSON parser
|
|
266
|
+
// because JSONbig is much slower (about 4x slower)
|
|
267
|
+
if (HAS_BIG_NUMBER.test(json)) return JSONbigint.parse(json);
|
|
268
|
+
else return JSON.parse(json);
|
|
269
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* represents the operation function object used for get operation
|
|
3
|
+
*/
|
|
4
|
+
export class OperationFunctionObject {
|
|
5
|
+
operation_function: Function;
|
|
6
|
+
job_operation_function: Function | undefined;
|
|
7
|
+
httpMethod?: string;
|
|
8
|
+
|
|
9
|
+
constructor(operation_function: Function, job_operation_function: Function = undefined) {
|
|
10
|
+
this.operation_function = operation_function;
|
|
11
|
+
this.job_operation_function = job_operation_function;
|
|
12
|
+
}
|
|
13
|
+
}
|