@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/BinObjects.js +17 -0
- package/bin/cliOperations.js +157 -0
- package/bin/copyDb.ts +280 -0
- package/bin/harper.js +156 -0
- package/bin/install.js +15 -0
- package/bin/lite.js +5 -0
- package/bin/restart.js +201 -0
- package/bin/run.js +409 -0
- package/bin/status.js +65 -0
- package/bin/stop.js +22 -0
- package/bin/upgrade.js +134 -0
- package/components/Application.ts +646 -0
- package/components/ApplicationScope.ts +49 -0
- package/components/Component.ts +53 -0
- package/components/ComponentV1.ts +342 -0
- package/components/DEFAULT_CONFIG.ts +18 -0
- package/components/EntryHandler.ts +227 -0
- package/components/Logger.ts +14 -0
- package/components/OptionsWatcher.ts +354 -0
- package/components/PluginModule.ts +6 -0
- package/components/Scope.ts +329 -0
- package/components/componentLoader.ts +529 -0
- package/components/deriveCommonPatternBase.ts +31 -0
- package/components/deriveGlobOptions.ts +44 -0
- package/components/deriveURLPath.ts +57 -0
- package/components/operations.js +658 -0
- package/components/operationsValidation.js +246 -0
- package/components/packageComponent.ts +39 -0
- package/components/requestRestart.ts +26 -0
- package/components/resolveBaseURLPath.ts +38 -0
- package/components/status/ComponentStatus.ts +110 -0
- package/components/status/ComponentStatusRegistry.ts +251 -0
- package/components/status/api.ts +153 -0
- package/components/status/crossThread.ts +405 -0
- package/components/status/errors.ts +152 -0
- package/components/status/index.ts +44 -0
- package/components/status/internal.ts +65 -0
- package/components/status/registry.ts +12 -0
- package/components/status/types.ts +96 -0
- package/config/RootConfigWatcher.ts +59 -0
- package/config/configHelpers.ts +11 -0
- package/config/configUtils.js +967 -0
- package/config/harperConfigEnvVars.ts +641 -0
- package/dataLayer/CreateAttributeObject.js +25 -0
- package/dataLayer/CreateTableObject.js +11 -0
- package/dataLayer/DataLayerObjects.js +43 -0
- package/dataLayer/DeleteBeforeObject.js +22 -0
- package/dataLayer/DeleteObject.js +25 -0
- package/dataLayer/DropAttributeObject.js +11 -0
- package/dataLayer/GetBackupObject.js +22 -0
- package/dataLayer/InsertObject.js +24 -0
- package/dataLayer/ReadAuditLogObject.js +24 -0
- package/dataLayer/SQLSearch.js +1335 -0
- package/dataLayer/SearchByConditionsObject.js +61 -0
- package/dataLayer/SearchByHashObject.js +21 -0
- package/dataLayer/SearchObject.js +45 -0
- package/dataLayer/SqlSearchObject.js +14 -0
- package/dataLayer/UpdateObject.js +23 -0
- package/dataLayer/UpsertObject.js +23 -0
- package/dataLayer/bulkLoad.js +813 -0
- package/dataLayer/dataObjects/BulkLoadObjects.js +27 -0
- package/dataLayer/dataObjects/UpsertObject.js +23 -0
- package/dataLayer/delete.js +164 -0
- package/dataLayer/export.js +381 -0
- package/dataLayer/getBackup.js +40 -0
- package/dataLayer/harperBridge/BridgeMethods.js +81 -0
- package/dataLayer/harperBridge/ResourceBridge.ts +633 -0
- package/dataLayer/harperBridge/bridgeUtility/insertUpdateReturnObj.js +28 -0
- package/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +88 -0
- package/dataLayer/harperBridge/harperBridge.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/LMDBBridge.js +119 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +19 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +112 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +67 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +31 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +94 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +98 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +89 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +109 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +107 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +137 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +35 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +111 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +28 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +29 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +207 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +156 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +30 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +19 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +64 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +70 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +22 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +23 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +22 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBTransactionObject.js +23 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +24 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +24 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +25 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +21 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +157 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +94 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +39 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +34 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +100 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +371 -0
- package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +109 -0
- package/dataLayer/hdbInfoController.js +254 -0
- package/dataLayer/insert.js +266 -0
- package/dataLayer/readAuditLog.js +59 -0
- package/dataLayer/schema.js +366 -0
- package/dataLayer/schemaDescribe.js +289 -0
- package/dataLayer/search.js +60 -0
- package/dataLayer/transaction.js +17 -0
- package/dataLayer/update.js +124 -0
- package/dist/components/Logger.d.ts +12 -0
- package/dist/components/Logger.js +3 -0
- package/dist/components/Logger.js.map +1 -0
- package/dist/components/Scope.d.ts +14 -4
- package/dist/components/Scope.js +18 -10
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/componentLoader.js +16 -9
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/operations.js +2 -2
- package/dist/components/operations.js.map +1 -1
- package/dist/config/configUtils.d.ts +1 -1
- package/dist/config/configUtils.js +1 -1
- package/dist/config/configUtils.js.map +1 -1
- package/dist/dataLayer/CreateTableObject.d.ts +2 -2
- package/dist/dataLayer/CreateTableObject.js +2 -2
- package/dist/dataLayer/CreateTableObject.js.map +1 -1
- package/dist/dataLayer/delete.d.ts +1 -1
- package/dist/dataLayer/schema.js +6 -5
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +1 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/resources/DatabaseTransaction.d.ts +1 -1
- package/dist/resources/IterableEventQueue.d.ts +1 -1
- package/dist/resources/LMDBTransaction.d.ts +5 -1
- package/dist/resources/Resource.d.ts +1 -1
- package/dist/resources/RocksIndexStore.d.ts +3 -3
- package/dist/resources/RocksTransactionLogStore.d.ts +6 -3
- package/dist/resources/Table.d.ts +15 -6
- package/dist/resources/Table.js +4 -1
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/read.js +32 -22
- package/dist/resources/analytics/read.js.map +1 -1
- package/dist/resources/analytics/write.js +3 -6
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +3 -3
- package/dist/resources/blob.d.ts +25 -2
- package/dist/resources/databases.d.ts +12 -2
- package/dist/resources/databases.js +22 -19
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/search.js +11 -5
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transaction.d.ts +2 -1
- package/dist/security/auth.js +1 -1
- package/dist/security/auth.js.map +1 -1
- package/dist/security/cryptoHash.d.ts +2 -2
- package/dist/security/jsLoader.js +243 -66
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +4 -5
- package/dist/security/keys.js.map +1 -1
- package/dist/security/user.js +3 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/REST.js +16 -2
- package/dist/server/REST.js.map +1 -1
- package/dist/server/Server.d.ts +2 -1
- package/dist/server/Server.js.map +1 -1
- package/dist/server/fastifyRoutes/plugins/hdbCore.d.ts +6 -1
- package/dist/server/fastifyRoutes.js +2 -0
- package/dist/server/fastifyRoutes.js.map +1 -1
- package/dist/server/http.js +12 -6
- package/dist/server/http.js.map +1 -1
- package/dist/server/jobs/JobObject.d.ts +3 -3
- package/dist/server/loadRootComponents.js +1 -0
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/operationsServer.js +3 -1
- package/dist/server/operationsServer.js.map +1 -1
- package/dist/server/serverHelpers/JSONStream.d.ts +3 -3
- package/dist/server/serverHelpers/Request.d.ts +5 -5
- package/dist/server/serverHelpers/requestTimePlugin.d.ts +1 -1
- package/dist/server/threads/manageThreads.d.ts +2 -2
- package/dist/server/threads/manageThreads.js +50 -35
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.d.ts +1 -1
- package/dist/sqlTranslator/deleteTranslator.d.ts +1 -1
- package/dist/utility/AWS/AWSConnector.d.ts +3 -2
- package/dist/utility/common_utils.d.ts +3 -3
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/functions/date/dateFunctions.d.ts +11 -11
- package/dist/utility/globalSchema.d.ts +1 -1
- package/dist/utility/hdbTerms.d.ts +3 -0
- package/dist/utility/hdbTerms.js +3 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/installation.d.ts +2 -4
- package/dist/utility/installation.js.map +1 -1
- package/dist/utility/lmdb/commonUtility.d.ts +1 -0
- package/dist/utility/lmdb/deleteUtility.d.ts +1 -0
- package/dist/utility/lmdb/environmentUtility.d.ts +1 -0
- package/dist/utility/lmdb/searchUtility.d.ts +2 -1
- package/dist/utility/lmdb/writeUtility.d.ts +1 -0
- package/dist/utility/logging/harper_logger.d.ts +6 -6
- package/dist/utility/processManagement/processManagement.d.ts +1 -1
- package/dist/utility/processManagement/servicesConfig.d.ts +12 -6
- package/dist/validation/common_validators.d.ts +4 -3
- package/dist/validation/configValidator.d.ts +3 -2
- package/index.d.ts +56 -0
- package/index.js +41 -0
- package/json/systemSchema.json +373 -0
- package/launchServiceScripts/launchHarperDB.js +3 -0
- package/launchServiceScripts/utility/checkNodeVersion.js +15 -0
- package/package.json +21 -3
- package/resources/DatabaseTransaction.ts +378 -0
- package/resources/ErrorResource.ts +57 -0
- package/resources/IterableEventQueue.ts +94 -0
- package/resources/LMDBTransaction.ts +349 -0
- package/resources/RecordEncoder.ts +702 -0
- package/resources/RequestTarget.ts +134 -0
- package/resources/Resource.ts +789 -0
- package/resources/ResourceInterface.ts +221 -0
- package/resources/ResourceInterfaceV2.ts +53 -0
- package/resources/ResourceV2.ts +67 -0
- package/resources/Resources.ts +162 -0
- package/resources/RocksIndexStore.ts +70 -0
- package/resources/RocksTransactionLogStore.ts +352 -0
- package/resources/Table.ts +4527 -0
- package/resources/analytics/hostnames.ts +72 -0
- package/resources/analytics/metadata.ts +10 -0
- package/resources/analytics/read.ts +252 -0
- package/resources/analytics/write.ts +803 -0
- package/resources/auditStore.ts +556 -0
- package/resources/blob.ts +1268 -0
- package/resources/crdt.ts +125 -0
- package/resources/dataLoader.ts +527 -0
- package/resources/databases.ts +1290 -0
- package/resources/graphql.ts +221 -0
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +638 -0
- package/resources/indexes/customIndexes.ts +7 -0
- package/resources/indexes/vector.ts +38 -0
- package/resources/jsResource.ts +86 -0
- package/resources/loadEnv.ts +22 -0
- package/resources/login.ts +18 -0
- package/resources/openApi.ts +409 -0
- package/resources/registrationDeprecated.ts +8 -0
- package/resources/replayLogs.ts +136 -0
- package/resources/roles.ts +98 -0
- package/resources/search.ts +1301 -0
- package/resources/tracked.ts +584 -0
- package/resources/transaction.ts +89 -0
- package/resources/transactionBroadcast.ts +258 -0
- package/security/auth.ts +376 -0
- package/security/certificateVerification/certificateVerificationSource.ts +84 -0
- package/security/certificateVerification/configValidation.ts +107 -0
- package/security/certificateVerification/crlVerification.ts +623 -0
- package/security/certificateVerification/index.ts +121 -0
- package/security/certificateVerification/ocspVerification.ts +148 -0
- package/security/certificateVerification/pkijs-ed25519-patch.ts +188 -0
- package/security/certificateVerification/types.ts +128 -0
- package/security/certificateVerification/verificationConfig.ts +138 -0
- package/security/certificateVerification/verificationUtils.ts +447 -0
- package/security/cryptoHash.js +42 -0
- package/security/data_objects/PermissionAttributeResponseObject.js +15 -0
- package/security/data_objects/PermissionResponseObject.js +115 -0
- package/security/data_objects/PermissionTableResponseObject.js +20 -0
- package/security/fastifyAuth.js +169 -0
- package/security/impersonation.ts +160 -0
- package/security/jsLoader.ts +716 -0
- package/security/keys.js +948 -0
- package/security/permissionsTranslator.js +300 -0
- package/security/role.js +218 -0
- package/security/tokenAuthentication.ts +228 -0
- package/security/user.ts +449 -0
- package/server/DurableSubscriptionsSession.ts +503 -0
- package/server/REST.ts +407 -0
- package/server/Server.ts +89 -0
- package/server/fastifyRoutes/helpers/getCORSOptions.js +36 -0
- package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +15 -0
- package/server/fastifyRoutes/helpers/getServerOptions.js +33 -0
- package/server/fastifyRoutes/plugins/hdbCore.js +39 -0
- package/server/fastifyRoutes.ts +205 -0
- package/server/graphqlQuerying.ts +700 -0
- package/server/http.ts +640 -0
- package/server/itc/serverHandlers.js +161 -0
- package/server/itc/utility/ITCEventObject.js +10 -0
- package/server/jobs/JobObject.js +24 -0
- package/server/jobs/jobProcess.js +69 -0
- package/server/jobs/jobRunner.js +162 -0
- package/server/jobs/jobs.js +304 -0
- package/server/loadRootComponents.js +44 -0
- package/server/mqtt.ts +485 -0
- package/server/nodeName.ts +75 -0
- package/server/operationsServer.ts +313 -0
- package/server/serverHelpers/Headers.ts +108 -0
- package/server/serverHelpers/JSONStream.ts +269 -0
- package/server/serverHelpers/OperationFunctionObject.ts +13 -0
- package/server/serverHelpers/Request.ts +158 -0
- package/server/serverHelpers/contentTypes.ts +637 -0
- package/server/serverHelpers/requestTimePlugin.js +57 -0
- package/server/serverHelpers/serverHandlers.js +148 -0
- package/server/serverHelpers/serverUtilities.ts +473 -0
- package/server/serverRegistry.ts +8 -0
- package/server/static.ts +187 -0
- package/server/status/definitions.ts +37 -0
- package/server/status/index.ts +125 -0
- package/server/storageReclamation.ts +93 -0
- package/server/threads/itc.js +89 -0
- package/server/threads/manageThreads.js +594 -0
- package/server/threads/socketRouter.ts +360 -0
- package/server/threads/threadServer.js +279 -0
- package/server/throttle.ts +73 -0
- package/sqlTranslator/SelectValidator.js +330 -0
- package/sqlTranslator/alasqlFunctionImporter.js +62 -0
- package/sqlTranslator/deleteTranslator.js +67 -0
- package/sqlTranslator/index.js +242 -0
- package/sqlTranslator/sql_statement_bucket.js +472 -0
- package/static/defaultConfig.yaml +3 -0
- package/studio/web/HDBDogOnly.svg +78 -0
- package/studio/web/assets/PPRadioGrotesk-Bold-DDaUYG8E.woff +0 -0
- package/studio/web/assets/fa-brands-400-CEJbCg16.woff +0 -0
- package/studio/web/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
- package/studio/web/assets/fa-brands-400-DnkPfk3o.eot +0 -0
- package/studio/web/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
- package/studio/web/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
- package/studio/web/assets/fa-regular-400-BhTwtT8w.eot +0 -0
- package/studio/web/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
- package/studio/web/assets/fa-regular-400-DFnMcJPd.woff +0 -0
- package/studio/web/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
- package/studio/web/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
- package/studio/web/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
- package/studio/web/assets/fa-solid-900-B6Axprfb.eot +0 -0
- package/studio/web/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
- package/studio/web/assets/fa-solid-900-DOXgCApm.woff +0 -0
- package/studio/web/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
- package/studio/web/assets/index-BTgXJX9d.js +235 -0
- package/studio/web/assets/index-BTgXJX9d.js.map +1 -0
- package/studio/web/assets/index-C-GXfcup.js +37 -0
- package/studio/web/assets/index-C-GXfcup.js.map +1 -0
- package/studio/web/assets/index-PFlNdimM.js +2 -0
- package/studio/web/assets/index-PFlNdimM.js.map +1 -0
- package/studio/web/assets/index-Y2g_iFpU.css +1 -0
- package/studio/web/assets/index-jiPwkrsB.css +1 -0
- package/studio/web/assets/index.lazy-C3TJZJ4o.js +266 -0
- package/studio/web/assets/index.lazy-C3TJZJ4o.js.map +1 -0
- package/studio/web/assets/profiler-DotzgiCJ.js +2 -0
- package/studio/web/assets/profiler-DotzgiCJ.js.map +1 -0
- package/studio/web/assets/react-redux-VxUEx_mU.js +6 -0
- package/studio/web/assets/react-redux-VxUEx_mU.js.map +1 -0
- package/studio/web/assets/startRecording-B_9J9Csd.js +3 -0
- package/studio/web/assets/startRecording-B_9J9Csd.js.map +1 -0
- package/studio/web/fabric-signup-background.webp +0 -0
- package/studio/web/fabric-signup-text.png +0 -0
- package/studio/web/favicon_purple.png +0 -0
- package/studio/web/github-icon.svg +15 -0
- package/studio/web/harper-fabric_black.png +0 -0
- package/studio/web/harper-fabric_white.png +0 -0
- package/studio/web/harper-studio_white.png +0 -0
- package/studio/web/index.html +16 -0
- package/studio/web/running.css +148 -0
- package/studio/web/running.html +147 -0
- package/studio/web/running.js +111 -0
- package/upgrade/UpgradeObjects.js +13 -0
- package/upgrade/directives/directivesController.js +90 -0
- package/upgrade/directivesManager.js +139 -0
- package/upgrade/upgradePrompt.js +124 -0
- package/upgrade/upgradeUtilities.js +28 -0
- package/utility/AWS/AWSConnector.js +29 -0
- package/utility/OperationFunctionCaller.js +63 -0
- package/utility/assignCmdEnvVariables.js +62 -0
- package/utility/common_utils.js +867 -0
- package/utility/environment/environmentManager.js +208 -0
- package/utility/environment/systemInformation.js +355 -0
- package/utility/errors/commonErrors.js +267 -0
- package/utility/errors/hdbError.js +146 -0
- package/utility/functions/date/dateFunctions.js +65 -0
- package/utility/functions/geo.js +355 -0
- package/utility/functions/sql/alaSQLExtension.js +104 -0
- package/utility/globalSchema.js +35 -0
- package/utility/hdbTerms.ts +819 -0
- package/utility/install/checkJWTTokensExist.js +62 -0
- package/utility/install/harperdb.conf +15 -0
- package/utility/install/harperdb.service +14 -0
- package/utility/install/installer.js +635 -0
- package/utility/installation.ts +30 -0
- package/utility/lmdb/DBIDefinition.js +20 -0
- package/utility/lmdb/DeleteRecordsResponseObject.js +25 -0
- package/utility/lmdb/InsertRecordsResponseObject.js +22 -0
- package/utility/lmdb/OpenDBIObject.js +31 -0
- package/utility/lmdb/OpenEnvironmentObject.js +41 -0
- package/utility/lmdb/UpdateRecordsResponseObject.js +25 -0
- package/utility/lmdb/UpsertRecordsResponseObject.js +22 -0
- package/utility/lmdb/cleanLMDBMap.js +65 -0
- package/utility/lmdb/commonUtility.js +119 -0
- package/utility/lmdb/deleteUtility.js +128 -0
- package/utility/lmdb/environmentUtility.js +477 -0
- package/utility/lmdb/searchCursorFunctions.js +187 -0
- package/utility/lmdb/searchUtility.js +918 -0
- package/utility/lmdb/terms.js +57 -0
- package/utility/lmdb/writeUtility.js +407 -0
- package/utility/logging/harper_logger.js +876 -0
- package/utility/logging/logRotator.js +157 -0
- package/utility/logging/logger.ts +24 -0
- package/utility/logging/readLog.js +355 -0
- package/utility/logging/transactionLog.js +57 -0
- package/utility/mount_hdb.js +59 -0
- package/utility/npmUtilities.js +102 -0
- package/utility/operationPermissions.ts +112 -0
- package/utility/operation_authorization.js +836 -0
- package/utility/packageUtils.js +55 -0
- package/utility/password.ts +99 -0
- package/utility/processManagement/processManagement.js +187 -0
- package/utility/processManagement/servicesConfig.js +56 -0
- package/utility/scripts/restartHdb.js +24 -0
- package/utility/scripts/user_data.sh +13 -0
- package/utility/signalling.js +36 -0
- package/utility/terms/certificates.js +81 -0
- package/utility/when.ts +20 -0
- package/v1.d.ts +39 -0
- package/v1.js +41 -0
- package/v2.d.ts +39 -0
- package/v2.js +41 -0
- package/validation/bulkDeleteValidator.js +24 -0
- package/validation/check_permissions.js +19 -0
- package/validation/common_validators.js +95 -0
- package/validation/configValidator.js +331 -0
- package/validation/deleteValidator.js +15 -0
- package/validation/fileLoadValidator.js +153 -0
- package/validation/insertValidator.js +40 -0
- package/validation/installValidator.js +37 -0
- package/validation/readLogValidator.js +64 -0
- package/validation/role_validation.js +320 -0
- package/validation/schemaMetadataValidator.js +42 -0
- package/validation/searchValidator.js +166 -0
- package/validation/statusValidator.ts +66 -0
- package/validation/transactionLogValidator.js +33 -0
- package/validation/user_validation.js +55 -0
- package/validation/validationWrapper.js +105 -0
- package/dist/resources/analytics/profile.d.ts +0 -2
- package/dist/resources/analytics/profile.js +0 -144
- package/dist/resources/analytics/profile.js.map +0 -1
|
@@ -0,0 +1,646 @@
|
|
|
1
|
+
import { type Logger } from '../utility/logging/logger.ts';
|
|
2
|
+
import { getConfigObj, getConfigValue } from '../config/configUtils.js';
|
|
3
|
+
import { CONFIG_PARAMS } from '../utility/hdbTerms.js';
|
|
4
|
+
import logger from '../utility/logging/harper_logger.js';
|
|
5
|
+
|
|
6
|
+
import { dirname, extname, join } from 'node:path';
|
|
7
|
+
import {
|
|
8
|
+
access,
|
|
9
|
+
constants,
|
|
10
|
+
cp,
|
|
11
|
+
mkdir,
|
|
12
|
+
mkdtemp,
|
|
13
|
+
readdir,
|
|
14
|
+
readFile,
|
|
15
|
+
rm,
|
|
16
|
+
stat,
|
|
17
|
+
symlink,
|
|
18
|
+
writeFile,
|
|
19
|
+
} from 'node:fs/promises';
|
|
20
|
+
import { spawn } from 'node:child_process';
|
|
21
|
+
import { createReadStream, existsSync, readdirSync } from 'node:fs';
|
|
22
|
+
import { Readable } from 'node:stream';
|
|
23
|
+
import { pipeline } from 'node:stream/promises';
|
|
24
|
+
|
|
25
|
+
import { extract } from 'tar-fs';
|
|
26
|
+
import gunzip from 'gunzip-maybe';
|
|
27
|
+
|
|
28
|
+
interface ApplicationConfig {
|
|
29
|
+
// define known config properties
|
|
30
|
+
package: string;
|
|
31
|
+
install?: {
|
|
32
|
+
command?: string;
|
|
33
|
+
timeout?: number;
|
|
34
|
+
};
|
|
35
|
+
// an application config can have other arbitrary properties
|
|
36
|
+
[key: string]: unknown;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export class InvalidPackageIdentifierError extends TypeError {
|
|
40
|
+
constructor(applicationName: string, packageIdentifier: unknown) {
|
|
41
|
+
super(
|
|
42
|
+
`Invalid 'package' property for application ${applicationName}: expected string, got ${typeof packageIdentifier}`
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export class InvalidInstallPropertyError extends TypeError {
|
|
48
|
+
constructor(applicationName: string, installProperty: unknown) {
|
|
49
|
+
super(
|
|
50
|
+
`Invalid 'install' property for application ${applicationName}: expected object, got ${typeof installProperty}`
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export class InvalidInstallCommandError extends TypeError {
|
|
56
|
+
constructor(applicationName: string, command: unknown) {
|
|
57
|
+
super(
|
|
58
|
+
`Invalid 'install.command' property for application ${applicationName}: expected string, got ${typeof command}`
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export class InvalidInstallTimeoutError extends TypeError {
|
|
64
|
+
constructor(applicationName: string, timeout: unknown) {
|
|
65
|
+
super(
|
|
66
|
+
`Invalid 'install.timeout' property for application ${applicationName}: expected non-negative number, got ${typeof timeout}`
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function assertApplicationConfig(
|
|
72
|
+
applicationName: string,
|
|
73
|
+
applicationConfig: Record<'package', unknown> & Record<string, unknown>
|
|
74
|
+
): asserts applicationConfig is ApplicationConfig {
|
|
75
|
+
if (typeof applicationConfig.package !== 'string') {
|
|
76
|
+
throw new InvalidPackageIdentifierError(applicationName, applicationConfig.package);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if ('install' in applicationConfig) {
|
|
80
|
+
if (
|
|
81
|
+
typeof applicationConfig.install !== 'object' ||
|
|
82
|
+
applicationConfig.install === null ||
|
|
83
|
+
Array.isArray(applicationConfig.install)
|
|
84
|
+
) {
|
|
85
|
+
throw new InvalidInstallPropertyError(applicationName, applicationConfig.install);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if ('command' in applicationConfig.install && typeof applicationConfig.install.command !== 'string') {
|
|
89
|
+
throw new InvalidInstallCommandError(applicationName, applicationConfig.install.command);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (
|
|
93
|
+
'timeout' in applicationConfig.install &&
|
|
94
|
+
(typeof applicationConfig.install.timeout !== 'number' || applicationConfig.install.timeout < 0)
|
|
95
|
+
) {
|
|
96
|
+
throw new InvalidInstallTimeoutError(applicationName, applicationConfig.install.timeout);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Extract an application given payload (content of the application) or package (npm-compatible identifier to the application).
|
|
103
|
+
*
|
|
104
|
+
* Only one of `application.payload` or `application.package` should be specified; otherwise, an error is thrown.
|
|
105
|
+
*
|
|
106
|
+
* Writes the application to the configured components root directory using the `application.name` and overwrites any existing directory.
|
|
107
|
+
*
|
|
108
|
+
* This method should only be called from the main thread
|
|
109
|
+
*/
|
|
110
|
+
export async function extractApplication(application: Application) {
|
|
111
|
+
// Can't specify neither
|
|
112
|
+
if (!application.payload && !application.packageIdentifier) {
|
|
113
|
+
throw new Error('Either payload or package must be provided');
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Can't specify both
|
|
117
|
+
if (application.payload && application.packageIdentifier) {
|
|
118
|
+
throw new Error('Both payload and package cannot be provided');
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Resolve the tarball from the input
|
|
122
|
+
let tarballPath: string;
|
|
123
|
+
let tarball: Readable;
|
|
124
|
+
if (application.payload) {
|
|
125
|
+
// Given a payload, create a Readable from the Buffer or string
|
|
126
|
+
tarball = Readable.from(
|
|
127
|
+
application.payload instanceof Buffer ? application.payload : Buffer.from(application.payload, 'base64')
|
|
128
|
+
);
|
|
129
|
+
} else {
|
|
130
|
+
// Given a package, there are a a couple options
|
|
131
|
+
const parentDirPath = dirname(application.dirPath);
|
|
132
|
+
|
|
133
|
+
// If the package identifier is a file path we need to check if its a tarball or a directory
|
|
134
|
+
if (application.packageIdentifier.startsWith('file:')) {
|
|
135
|
+
const packagePath = application.packageIdentifier.slice(5);
|
|
136
|
+
try {
|
|
137
|
+
// Have to remove the 'file:' prefix in order to use fs methods
|
|
138
|
+
const stats = await stat(packagePath);
|
|
139
|
+
|
|
140
|
+
if (stats.isDirectory()) {
|
|
141
|
+
// If its a directory, symlink
|
|
142
|
+
await symlink(packagePath, application.dirPath, 'dir');
|
|
143
|
+
// And return early since we're done; no extraction needed
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
if (!stats.isFile()) {
|
|
148
|
+
throw new Error(`File path specified in package identifier is not a file or directory: ${packagePath}`);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// If its a file, we assume it can be unzipped and extracted.
|
|
152
|
+
// We are using maybe-gunzip to handle both gzipped and non-gzipped tarballs
|
|
153
|
+
// And then we are happy to let the `tar-fs` library handle the extraction.
|
|
154
|
+
// Maybe worth adding some detection or at least some error handling if that step below fails.
|
|
155
|
+
tarballPath = packagePath;
|
|
156
|
+
tarball = createReadStream(tarballPath);
|
|
157
|
+
} catch (err) {
|
|
158
|
+
if (err.code === 'ENOENT') {
|
|
159
|
+
throw new Error(`File path specified in package identifier does not exist: ${packagePath}`);
|
|
160
|
+
} else {
|
|
161
|
+
throw err;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
} else {
|
|
165
|
+
// Given a package, resolve using `npm pack` (downloads the package as a tarball and writes the path to stdout)
|
|
166
|
+
const {
|
|
167
|
+
stdout: tarballFilePath,
|
|
168
|
+
code,
|
|
169
|
+
stderr,
|
|
170
|
+
} = await nonInteractiveSpawn(application.name, 'npm', ['pack', application.packageIdentifier], parentDirPath);
|
|
171
|
+
if (code !== 0) throw new Error(`Failed to download package ${application.packageIdentifier}: ${stderr}`);
|
|
172
|
+
tarballPath = join(parentDirPath, tarballFilePath.trim());
|
|
173
|
+
// Create a Readable from the tarball
|
|
174
|
+
tarball = createReadStream(tarballPath);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Create the application directory
|
|
179
|
+
try {
|
|
180
|
+
await access(application.dirPath, constants.F_OK);
|
|
181
|
+
// directory already exists; clear it
|
|
182
|
+
await rm(application.dirPath, { recursive: true, force: true });
|
|
183
|
+
} catch (err) {
|
|
184
|
+
// Ignore does not exist error
|
|
185
|
+
if (err.code !== 'ENOENT') {
|
|
186
|
+
throw err;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
// Finally, create the application directory fresh
|
|
190
|
+
await mkdir(application.dirPath, { recursive: true });
|
|
191
|
+
|
|
192
|
+
// Now pipeline the tarball into maybe-gunzip then tar-fs to reliably decompress and extract the contents
|
|
193
|
+
await pipeline(tarball, gunzip(), extract(application.dirPath));
|
|
194
|
+
|
|
195
|
+
// If the extracted directory contains a single folder, move the contents up one level
|
|
196
|
+
// The `npm pack` command does this (the top-level folder is called "package")
|
|
197
|
+
// Other packing tools may have similar behavior, but the directory name is not guaranteed.
|
|
198
|
+
const extracted = await readdir(application.dirPath, { withFileTypes: true });
|
|
199
|
+
if (extracted.length === 1 && extracted[0].isDirectory()) {
|
|
200
|
+
const topLevelDirPath = join(application.dirPath, extracted[0].name);
|
|
201
|
+
|
|
202
|
+
const tempDirPath = await mkdtemp(application.dirPath);
|
|
203
|
+
|
|
204
|
+
// Copy contents of top-level directory to temp directory (in order to avoid collisions of top-level directory name and one of the contents)
|
|
205
|
+
await cp(topLevelDirPath, tempDirPath, { recursive: true });
|
|
206
|
+
// Remove top-level directory
|
|
207
|
+
await rm(topLevelDirPath, { recursive: true, force: true });
|
|
208
|
+
// Copy contents of temp directory to application directory
|
|
209
|
+
await cp(tempDirPath, application.dirPath, { recursive: true });
|
|
210
|
+
// Finally, remove the temp dir
|
|
211
|
+
await rm(tempDirPath, { recursive: true, force: true });
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// Clean up the original tarball
|
|
215
|
+
if (tarballPath) {
|
|
216
|
+
await rm(tarballPath, { force: true });
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Install an application to its relative `application.dirPath` using either a
|
|
222
|
+
* configured `application.install` command, a derived package manager from the
|
|
223
|
+
* application's `package.json#devEngines`, or falling back to the default
|
|
224
|
+
* package manager, `npm`.
|
|
225
|
+
*
|
|
226
|
+
* Will return early if `node_modules` already exists within the `application.dirPath`
|
|
227
|
+
*
|
|
228
|
+
* This method should only be called from the main thread
|
|
229
|
+
*/
|
|
230
|
+
export async function installApplication(application: Application) {
|
|
231
|
+
let packageJSON: any;
|
|
232
|
+
try {
|
|
233
|
+
packageJSON = JSON.parse(await readFile(join(application.dirPath, 'package.json'), 'utf8'));
|
|
234
|
+
} catch (err) {
|
|
235
|
+
if (err.code !== 'ENOENT') throw err;
|
|
236
|
+
// If no package.json, nothing to install
|
|
237
|
+
application.logger.debug(`Application ${application.name} has no package.json; skipping install`);
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
try {
|
|
241
|
+
// Does node_modules exist?
|
|
242
|
+
await access(join(application.dirPath, 'node_modules'), constants.F_OK);
|
|
243
|
+
application.logger.debug(`Application ${application.name} already has node_modules; skipping install`);
|
|
244
|
+
return;
|
|
245
|
+
} catch (err) {
|
|
246
|
+
if (err.code !== 'ENOENT') throw err;
|
|
247
|
+
// If node_modules doesn't exist, we need to install dependencies
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// If custom install command is specified, run it
|
|
251
|
+
if (application.install?.command) {
|
|
252
|
+
const [command, ...args] = application.install.command.split(' ');
|
|
253
|
+
const { stdout, stderr, code } = await nonInteractiveSpawn(
|
|
254
|
+
application.name,
|
|
255
|
+
command,
|
|
256
|
+
args,
|
|
257
|
+
application.dirPath,
|
|
258
|
+
application.install?.timeout
|
|
259
|
+
);
|
|
260
|
+
// if it succeeds, return
|
|
261
|
+
if (code === 0) {
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
if (stdout) {
|
|
265
|
+
printStd(application.name, command, stdout, 'stdout', 'warn');
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
if (stderr) {
|
|
269
|
+
printStd(application.name, command, stderr, 'stderr', 'warn');
|
|
270
|
+
}
|
|
271
|
+
// and throw a descriptive error
|
|
272
|
+
throw new Error(
|
|
273
|
+
`Failed to install dependencies for ${application.name} using custom install command: ${application.install.command}. Exit code: ${code}`
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// Next, try package.json devEngines field
|
|
278
|
+
const { packageManager } = packageJSON.devEngines || {};
|
|
279
|
+
|
|
280
|
+
// Custom package manager specified
|
|
281
|
+
if (packageManager) {
|
|
282
|
+
// On any given system we want to leverage the `name` to match the package manager executable
|
|
283
|
+
let onFail: string | undefined = packageManager.onFail;
|
|
284
|
+
|
|
285
|
+
const validOnFailValues = ['ignore', 'warn', 'error'];
|
|
286
|
+
|
|
287
|
+
if (onFail === 'download') {
|
|
288
|
+
application.logger.warn(
|
|
289
|
+
'Harper currently does not support `devEngines.packageManager.onFail = "download"`. Defaulting to "error"'
|
|
290
|
+
);
|
|
291
|
+
onFail = 'error';
|
|
292
|
+
} else if (onFail && !validOnFailValues.includes(onFail)) {
|
|
293
|
+
application.logger.error(
|
|
294
|
+
`Invalid \`devEngines.packageManager.onFail\` value: "${onFail}". Expected one of ${validOnFailValues.map((v) => `"${v}"`).join(', ')}. Defaulting to "error"`
|
|
295
|
+
);
|
|
296
|
+
onFail = 'error';
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
onFail = onFail || 'error';
|
|
300
|
+
|
|
301
|
+
// TODO: Implement a version check / resolution system
|
|
302
|
+
// For example, say they specify a specific major version for their package manager
|
|
303
|
+
// Maybe on our system, we have all of the supported majors (for a given Node.js major) of any supported package manager.
|
|
304
|
+
// Then we can do something like <name>@<version> for the corresponding executable.
|
|
305
|
+
// `devEngines: { packageManager: { name: 'pnpm', version: '>=7' } }`
|
|
306
|
+
// Would result in `pnpm@7` being used as the executable.
|
|
307
|
+
// Important note: an `npm` version should not be specifiable; the only valid npm version is the one installed alongside Node.js
|
|
308
|
+
|
|
309
|
+
const { stdout, stderr, code } = await nonInteractiveSpawn(
|
|
310
|
+
application.name,
|
|
311
|
+
(application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + packageManager.name,
|
|
312
|
+
['install'], // All of `npm`, `yarn`, and `pnpm` support the `install` command. If we need to configure options here we may have to use some other defaults though
|
|
313
|
+
application.dirPath,
|
|
314
|
+
application.install?.timeout
|
|
315
|
+
);
|
|
316
|
+
|
|
317
|
+
// if it succeeds, return
|
|
318
|
+
if (code === 0) {
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Otherwise handle failure case based on `onFail` value
|
|
323
|
+
if (onFail === 'error') {
|
|
324
|
+
// Log the std outputs using the error log level (in case the user doesn't have debug level set)
|
|
325
|
+
if (stdout) {
|
|
326
|
+
printStd(application.name, packageManager.name, stdout, 'stdout', 'error');
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (stderr) {
|
|
330
|
+
printStd(application.name, packageManager.name, stderr, 'stderr', 'error');
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// And throw an error instead of continuing
|
|
334
|
+
throw new Error(
|
|
335
|
+
`Failed to install dependencies for ${application.name} using ${packageManager.name}. Exit code: ${code}`
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// If onFail is 'warn', print outputs using the warn level, plus an additional message
|
|
340
|
+
if (onFail === 'warn') {
|
|
341
|
+
if (stdout) {
|
|
342
|
+
printStd(application.name, packageManager.name, stdout, 'stdout', 'warn');
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
if (stderr) {
|
|
346
|
+
printStd(application.name, packageManager.name, stderr, 'stderr', 'warn');
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
application.logger.warn(
|
|
350
|
+
`Failed to install dependencies for ${application.name} using ${packageManager.name}. Exit code: ${code}`
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// But then fall through to installing with npm
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// Finally, default to running `npm install`
|
|
358
|
+
const { stdout, stderr, code } = await nonInteractiveSpawn(
|
|
359
|
+
application.name,
|
|
360
|
+
(application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + 'npm',
|
|
361
|
+
['install', '--force'],
|
|
362
|
+
application.dirPath
|
|
363
|
+
);
|
|
364
|
+
|
|
365
|
+
// if it succeeds, return
|
|
366
|
+
if (code === 0) {
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// Otherwise, print the stdout and stderr outputs
|
|
371
|
+
if (stdout) {
|
|
372
|
+
printStd(application.name, 'npm', stdout, 'stdout', 'warn');
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
if (stderr) {
|
|
376
|
+
printStd(application.name, 'npm', stderr, 'stderr', 'error');
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// and throw a descriptive error
|
|
380
|
+
throw new Error(`Failed to install dependencies for ${application.name} using npm default. Exit code: ${code}`);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
interface ApplicationOptions {
|
|
384
|
+
name: string;
|
|
385
|
+
payload?: Buffer | string;
|
|
386
|
+
packageIdentifier?: string;
|
|
387
|
+
install?: { command?: string; timeout?: number };
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
export class Application {
|
|
391
|
+
name: string;
|
|
392
|
+
payload?: Buffer | string;
|
|
393
|
+
packageIdentifier?: string;
|
|
394
|
+
install?: { command?: string; timeout?: number };
|
|
395
|
+
dirPath: string;
|
|
396
|
+
logger: Logger;
|
|
397
|
+
packageManagerPrefix: string; // can be used to configure a package manager prefix, specifically "sfw".
|
|
398
|
+
|
|
399
|
+
constructor({ name, payload, packageIdentifier, install }: ApplicationOptions) {
|
|
400
|
+
this.name = name;
|
|
401
|
+
this.payload = payload;
|
|
402
|
+
this.packageIdentifier = packageIdentifier && derivePackageIdentifier(packageIdentifier);
|
|
403
|
+
this.install = install;
|
|
404
|
+
this.dirPath = join(getConfigValue(CONFIG_PARAMS.COMPONENTSROOT), name);
|
|
405
|
+
this.logger = logger.loggerWithTag(name);
|
|
406
|
+
this.packageManagerPrefix = getConfigValue(CONFIG_PARAMS.APPLICATIONS_PACKAGEMANAGERPREFIX);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* Based on an old implementation for a method called `getPkgPrefix()` that was used
|
|
412
|
+
* during the installation process in order to actually resolve what the user specifies for a
|
|
413
|
+
* component matching some of npm's package resolution rules.
|
|
414
|
+
*/
|
|
415
|
+
export function derivePackageIdentifier(packageIdentifier: string) {
|
|
416
|
+
if (packageIdentifier.includes(':')) {
|
|
417
|
+
return packageIdentifier;
|
|
418
|
+
}
|
|
419
|
+
if (packageIdentifier.startsWith('@') || (!packageIdentifier.startsWith('@') && !packageIdentifier.includes('/'))) {
|
|
420
|
+
return `npm:${packageIdentifier}`;
|
|
421
|
+
}
|
|
422
|
+
if (extname(packageIdentifier) || existsSync(packageIdentifier)) {
|
|
423
|
+
return `file:${packageIdentifier}`;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
return `github:${packageIdentifier}`;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* Extract and install the specified application.
|
|
431
|
+
*
|
|
432
|
+
* This method should only be called from the main thread
|
|
433
|
+
*
|
|
434
|
+
* @param application The application to prepare.
|
|
435
|
+
* @returns A promise that resolves when all preparation steps complete.
|
|
436
|
+
*/
|
|
437
|
+
export function prepareApplication(application: Application) {
|
|
438
|
+
return extractApplication(application).then(() => installApplication(application));
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* Install all applications specified in the root config.
|
|
443
|
+
*
|
|
444
|
+
* This method should only be called from the main thread otherwise certain
|
|
445
|
+
* operations may conflict with each other (such as writing to the same directory).
|
|
446
|
+
*/
|
|
447
|
+
export async function installApplications() {
|
|
448
|
+
const applicationInstallationPromises: Promise<void>[] = [];
|
|
449
|
+
|
|
450
|
+
// first install any built-in components specified from env vars
|
|
451
|
+
for (const { name, packageIdentifier } of getEnvBuiltInComponents()) {
|
|
452
|
+
if (packageIdentifier.startsWith('@/')) {
|
|
453
|
+
// this is a package relative module id, so later we will resolve it, but we don't need to install anything
|
|
454
|
+
continue;
|
|
455
|
+
}
|
|
456
|
+
const application = new Application({
|
|
457
|
+
name,
|
|
458
|
+
packageIdentifier,
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
applicationInstallationPromises.push(prepareApplication(application));
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
const config = getConfigObj();
|
|
465
|
+
|
|
466
|
+
const componentsRootDirPath = getConfigValue(CONFIG_PARAMS.COMPONENTSROOT);
|
|
467
|
+
|
|
468
|
+
// Ensure component directory exists
|
|
469
|
+
await mkdir(componentsRootDirPath, { recursive: true });
|
|
470
|
+
|
|
471
|
+
const harperApplicationLockPath = join(getConfigValue(CONFIG_PARAMS.ROOTPATH), 'harper-application-lock.json');
|
|
472
|
+
|
|
473
|
+
let harperApplicationLock: { applications: Record<string, ApplicationConfig> } = { applications: {} };
|
|
474
|
+
try {
|
|
475
|
+
harperApplicationLock = JSON.parse(await readFile(harperApplicationLockPath, 'utf8'));
|
|
476
|
+
} catch (error) {
|
|
477
|
+
// Ignore file not found error; will create new lock file after installations
|
|
478
|
+
if (error.code !== 'ENOENT') {
|
|
479
|
+
throw error;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
for (const [name, applicationConfig] of Object.entries(config)) {
|
|
484
|
+
// Pre-validation check if the configuration is actually for an application
|
|
485
|
+
// Don't want to throw an error here as the config may contain non-application entries
|
|
486
|
+
if (typeof applicationConfig !== 'object' || applicationConfig === null || !('package' in applicationConfig)) {
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
try {
|
|
491
|
+
// Then do proper error-based validation with TypeScript `asserts` to provide type safety
|
|
492
|
+
// This will throw if the config is invalid
|
|
493
|
+
assertApplicationConfig(name, applicationConfig);
|
|
494
|
+
|
|
495
|
+
const application = new Application({
|
|
496
|
+
name,
|
|
497
|
+
packageIdentifier: applicationConfig.package,
|
|
498
|
+
install: applicationConfig.install,
|
|
499
|
+
});
|
|
500
|
+
|
|
501
|
+
// Lock check: only install if not already installed with matching configuration
|
|
502
|
+
if (
|
|
503
|
+
existsSync(application.dirPath) &&
|
|
504
|
+
harperApplicationLock.applications[name] &&
|
|
505
|
+
JSON.stringify(harperApplicationLock.applications[name]) === JSON.stringify(applicationConfig)
|
|
506
|
+
) {
|
|
507
|
+
logger.info?.(`Application ${name} is already installed with matching configuration; skipping installation`);
|
|
508
|
+
continue;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
applicationInstallationPromises.push(prepareApplication(application));
|
|
512
|
+
|
|
513
|
+
harperApplicationLock.applications[name] = applicationConfig;
|
|
514
|
+
} catch (error) {
|
|
515
|
+
logger.error?.(`Skipping installation of application ${name} due to invalid configuration: ${error.message}`);
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
const applicationInstallationStatuses = await Promise.allSettled(applicationInstallationPromises);
|
|
520
|
+
logger.debug?.(applicationInstallationStatuses);
|
|
521
|
+
logger.info?.('All root applications loaded');
|
|
522
|
+
|
|
523
|
+
// Finally, write the lock file
|
|
524
|
+
await writeFile(harperApplicationLockPath, JSON.stringify(harperApplicationLock, null, 2), 'utf8');
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
function getGitSSHCommand() {
|
|
528
|
+
const rootDir = getConfigValue(CONFIG_PARAMS.ROOTPATH);
|
|
529
|
+
const sshDir = join(rootDir, 'ssh');
|
|
530
|
+
if (existsSync(sshDir)) {
|
|
531
|
+
for (const file of readdirSync(sshDir)) {
|
|
532
|
+
if (file.includes('.key')) {
|
|
533
|
+
return `ssh -F ${join(sshDir, 'config')} -o UserKnownHostsFile=${join(sshDir, 'known_hosts')}`;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* Execute a command (using `spawn`) with stdin ignored.
|
|
541
|
+
*
|
|
542
|
+
* Stdout is logged chunk-by-chunk. Stderr is buffered and then logged line-by-line.
|
|
543
|
+
*
|
|
544
|
+
* Rejects with an error if the command fails or times out.
|
|
545
|
+
*
|
|
546
|
+
* @param command The command to run.
|
|
547
|
+
* @param args The arguments to pass to the command.
|
|
548
|
+
* @param cwd The working directory for the command.
|
|
549
|
+
* @param timeoutMs The timeout for the command in milliseconds. Defaults to 5 minutes.
|
|
550
|
+
* @returns A promise that resolves when the command completes.
|
|
551
|
+
*/
|
|
552
|
+
export function nonInteractiveSpawn(
|
|
553
|
+
applicationName: string,
|
|
554
|
+
command: string,
|
|
555
|
+
args: string[],
|
|
556
|
+
cwd: string,
|
|
557
|
+
timeoutMs: number = 5 * 60 * 1000
|
|
558
|
+
): Promise<{ stdout: string; stderr: string; code: number }> {
|
|
559
|
+
return new Promise((resolve, reject) => {
|
|
560
|
+
logger
|
|
561
|
+
.loggerWithTag(`${applicationName}:spawn:${command}`)
|
|
562
|
+
.debug?.(`Executing \`${command} ${args.join(' ')}\` in ${cwd}`);
|
|
563
|
+
|
|
564
|
+
const env = { ...process.env };
|
|
565
|
+
|
|
566
|
+
const gitSSHCommand = getGitSSHCommand();
|
|
567
|
+
if (gitSSHCommand) {
|
|
568
|
+
env.GIT_SSH_COMMAND = gitSSHCommand;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const childProcess = spawn(command, args, {
|
|
572
|
+
shell: true,
|
|
573
|
+
cwd,
|
|
574
|
+
env,
|
|
575
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
576
|
+
});
|
|
577
|
+
|
|
578
|
+
const timeout = setTimeout(() => {
|
|
579
|
+
childProcess.kill();
|
|
580
|
+
reject(new Error(`Command\`${command} ${args.join(' ')}\` timed out after ${timeoutMs}ms`));
|
|
581
|
+
}, timeoutMs);
|
|
582
|
+
|
|
583
|
+
let stdout = '';
|
|
584
|
+
childProcess.stdout.on('data', (chunk) => {
|
|
585
|
+
// buffer stdout for later resolve
|
|
586
|
+
stdout += chunk.toString();
|
|
587
|
+
// log stdout lines immediately
|
|
588
|
+
// TODO: Technically nothing guarantees that a chunk will be a complete line so need to implement
|
|
589
|
+
// something here to buffer until a newline character, then log the complete line
|
|
590
|
+
logger.loggerWithTag(`${applicationName}:spawn:${command}:stdout`).debug?.(chunk.toString());
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
// buffer stderr
|
|
594
|
+
let stderr = '';
|
|
595
|
+
childProcess.stderr.on('data', (chunk) => {
|
|
596
|
+
stderr += chunk.toString();
|
|
597
|
+
});
|
|
598
|
+
|
|
599
|
+
childProcess.on('error', (error) => {
|
|
600
|
+
clearTimeout(timeout);
|
|
601
|
+
// Print out stderr before rejecting
|
|
602
|
+
if (stderr) {
|
|
603
|
+
printStd(applicationName, command, stderr, 'stderr');
|
|
604
|
+
}
|
|
605
|
+
reject(error);
|
|
606
|
+
});
|
|
607
|
+
|
|
608
|
+
childProcess.on('close', (code) => {
|
|
609
|
+
clearTimeout(timeout);
|
|
610
|
+
if (stderr) {
|
|
611
|
+
printStd(applicationName, command, stderr, 'stderr');
|
|
612
|
+
}
|
|
613
|
+
logger.loggerWithTag(`${applicationName}:spawn:${command}`).debug?.(`Process exited with code ${code}`);
|
|
614
|
+
resolve({
|
|
615
|
+
stdout,
|
|
616
|
+
stderr,
|
|
617
|
+
code,
|
|
618
|
+
});
|
|
619
|
+
});
|
|
620
|
+
});
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
export function getEnvBuiltInComponents() {
|
|
624
|
+
const builtInComponents: { name: string; packageIdentifier: string }[] = [];
|
|
625
|
+
if (process.env.HARPER_BUILTIN_COMPONENTS) {
|
|
626
|
+
for (const componentDefinition of process.env.HARPER_BUILTIN_COMPONENTS.split(',')) {
|
|
627
|
+
const [name, packageIdentifier] = componentDefinition.trim().split('=');
|
|
628
|
+
if (!componentDefinition) continue;
|
|
629
|
+
builtInComponents.push({ name, packageIdentifier });
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
return builtInComponents;
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
function printStd(
|
|
636
|
+
applicationName: string,
|
|
637
|
+
command: string,
|
|
638
|
+
stdString: string,
|
|
639
|
+
stdStreamLabel: 'stdout' | 'stderr',
|
|
640
|
+
level: 'debug' | 'warn' | 'error' = 'debug'
|
|
641
|
+
) {
|
|
642
|
+
const stdLogger = logger.loggerWithTag(`${applicationName}:spawn:${command}:${stdStreamLabel}`);
|
|
643
|
+
for (const line of stdString.split('\n')) {
|
|
644
|
+
stdLogger[level]?.(line);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { Resources } from '../resources/Resources.ts';
|
|
2
|
+
import { type Server } from '../server/Server.ts';
|
|
3
|
+
import { loggerWithTag } from '../utility/logging/harper_logger.js';
|
|
4
|
+
import { scopedImport } from '../security/jsLoader.ts';
|
|
5
|
+
import * as env from '../utility/environment/environmentManager.js';
|
|
6
|
+
import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
|
|
7
|
+
|
|
8
|
+
export class MissingDefaultFilesOptionError extends Error {
|
|
9
|
+
constructor() {
|
|
10
|
+
super('No default files option exists. Ensure `files` is specified in config.yaml');
|
|
11
|
+
this.name = 'MissingDefaultFilesOptionError';
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* This class is used to represent the application scope for the VM context used for loading modules within an application
|
|
17
|
+
*/
|
|
18
|
+
export class ApplicationScope {
|
|
19
|
+
logger: any;
|
|
20
|
+
resources: Resources;
|
|
21
|
+
server: Server;
|
|
22
|
+
mode?: 'none' | 'vm' | 'compartment'; // option to set this from the scope
|
|
23
|
+
dependencyContainment?: boolean; // option to set this from the scope
|
|
24
|
+
verifyPath?: string;
|
|
25
|
+
config: any;
|
|
26
|
+
constructor(name: string, resources: Resources, server: Server, verifyPath?: string) {
|
|
27
|
+
this.logger = loggerWithTag(name);
|
|
28
|
+
|
|
29
|
+
this.resources = resources;
|
|
30
|
+
this.server = server;
|
|
31
|
+
|
|
32
|
+
this.mode = env.get(CONFIG_PARAMS.APPLICATIONS_CONTAINMENT) ?? 'vm';
|
|
33
|
+
this.dependencyContainment = Boolean(env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYCONTAINMENT));
|
|
34
|
+
this.verifyPath = verifyPath;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The compartment that is used for this scope and any imports that it makes
|
|
39
|
+
*/
|
|
40
|
+
compartment?: Promise<any>;
|
|
41
|
+
/**
|
|
42
|
+
* Import a file into the scope's sandbox.
|
|
43
|
+
* @param filePath - The path of the file to import.
|
|
44
|
+
* @returns A promise that resolves with the imported module or value.
|
|
45
|
+
*/
|
|
46
|
+
async import(filePath: string): Promise<unknown> {
|
|
47
|
+
return scopedImport(filePath, this);
|
|
48
|
+
}
|
|
49
|
+
}
|