@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (444) hide show
  1. package/bin/BinObjects.js +17 -0
  2. package/bin/cliOperations.js +157 -0
  3. package/bin/copyDb.ts +280 -0
  4. package/bin/harper.js +156 -0
  5. package/bin/install.js +15 -0
  6. package/bin/lite.js +5 -0
  7. package/bin/restart.js +201 -0
  8. package/bin/run.js +409 -0
  9. package/bin/status.js +65 -0
  10. package/bin/stop.js +22 -0
  11. package/bin/upgrade.js +134 -0
  12. package/components/Application.ts +646 -0
  13. package/components/ApplicationScope.ts +49 -0
  14. package/components/Component.ts +53 -0
  15. package/components/ComponentV1.ts +342 -0
  16. package/components/DEFAULT_CONFIG.ts +18 -0
  17. package/components/EntryHandler.ts +227 -0
  18. package/components/Logger.ts +14 -0
  19. package/components/OptionsWatcher.ts +354 -0
  20. package/components/PluginModule.ts +6 -0
  21. package/components/Scope.ts +329 -0
  22. package/components/componentLoader.ts +529 -0
  23. package/components/deriveCommonPatternBase.ts +31 -0
  24. package/components/deriveGlobOptions.ts +44 -0
  25. package/components/deriveURLPath.ts +57 -0
  26. package/components/operations.js +658 -0
  27. package/components/operationsValidation.js +246 -0
  28. package/components/packageComponent.ts +39 -0
  29. package/components/requestRestart.ts +26 -0
  30. package/components/resolveBaseURLPath.ts +38 -0
  31. package/components/status/ComponentStatus.ts +110 -0
  32. package/components/status/ComponentStatusRegistry.ts +251 -0
  33. package/components/status/api.ts +153 -0
  34. package/components/status/crossThread.ts +405 -0
  35. package/components/status/errors.ts +152 -0
  36. package/components/status/index.ts +44 -0
  37. package/components/status/internal.ts +65 -0
  38. package/components/status/registry.ts +12 -0
  39. package/components/status/types.ts +96 -0
  40. package/config/RootConfigWatcher.ts +59 -0
  41. package/config/configHelpers.ts +11 -0
  42. package/config/configUtils.js +967 -0
  43. package/config/harperConfigEnvVars.ts +641 -0
  44. package/dataLayer/CreateAttributeObject.js +25 -0
  45. package/dataLayer/CreateTableObject.js +11 -0
  46. package/dataLayer/DataLayerObjects.js +43 -0
  47. package/dataLayer/DeleteBeforeObject.js +22 -0
  48. package/dataLayer/DeleteObject.js +25 -0
  49. package/dataLayer/DropAttributeObject.js +11 -0
  50. package/dataLayer/GetBackupObject.js +22 -0
  51. package/dataLayer/InsertObject.js +24 -0
  52. package/dataLayer/ReadAuditLogObject.js +24 -0
  53. package/dataLayer/SQLSearch.js +1335 -0
  54. package/dataLayer/SearchByConditionsObject.js +61 -0
  55. package/dataLayer/SearchByHashObject.js +21 -0
  56. package/dataLayer/SearchObject.js +45 -0
  57. package/dataLayer/SqlSearchObject.js +14 -0
  58. package/dataLayer/UpdateObject.js +23 -0
  59. package/dataLayer/UpsertObject.js +23 -0
  60. package/dataLayer/bulkLoad.js +813 -0
  61. package/dataLayer/dataObjects/BulkLoadObjects.js +27 -0
  62. package/dataLayer/dataObjects/UpsertObject.js +23 -0
  63. package/dataLayer/delete.js +164 -0
  64. package/dataLayer/export.js +381 -0
  65. package/dataLayer/getBackup.js +40 -0
  66. package/dataLayer/harperBridge/BridgeMethods.js +81 -0
  67. package/dataLayer/harperBridge/ResourceBridge.ts +633 -0
  68. package/dataLayer/harperBridge/bridgeUtility/insertUpdateReturnObj.js +28 -0
  69. package/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +88 -0
  70. package/dataLayer/harperBridge/harperBridge.js +21 -0
  71. package/dataLayer/harperBridge/lmdbBridge/LMDBBridge.js +119 -0
  72. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +19 -0
  73. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +112 -0
  74. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +67 -0
  75. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +31 -0
  76. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +94 -0
  77. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +98 -0
  78. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +89 -0
  79. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +109 -0
  80. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +107 -0
  81. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +137 -0
  82. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +35 -0
  83. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +111 -0
  84. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +28 -0
  85. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +29 -0
  86. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +207 -0
  87. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +156 -0
  88. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +21 -0
  89. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +30 -0
  90. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +19 -0
  91. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +64 -0
  92. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +70 -0
  93. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +22 -0
  94. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +23 -0
  95. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +22 -0
  96. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBTransactionObject.js +23 -0
  97. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +24 -0
  98. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +24 -0
  99. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +25 -0
  100. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +21 -0
  101. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +157 -0
  102. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +94 -0
  103. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +39 -0
  104. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +34 -0
  105. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +100 -0
  106. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +371 -0
  107. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +109 -0
  108. package/dataLayer/hdbInfoController.js +254 -0
  109. package/dataLayer/insert.js +266 -0
  110. package/dataLayer/readAuditLog.js +59 -0
  111. package/dataLayer/schema.js +366 -0
  112. package/dataLayer/schemaDescribe.js +289 -0
  113. package/dataLayer/search.js +60 -0
  114. package/dataLayer/transaction.js +17 -0
  115. package/dataLayer/update.js +124 -0
  116. package/dist/components/Logger.d.ts +12 -0
  117. package/dist/components/Logger.js +3 -0
  118. package/dist/components/Logger.js.map +1 -0
  119. package/dist/components/Scope.d.ts +14 -4
  120. package/dist/components/Scope.js +18 -10
  121. package/dist/components/Scope.js.map +1 -1
  122. package/dist/components/componentLoader.js +16 -9
  123. package/dist/components/componentLoader.js.map +1 -1
  124. package/dist/components/operations.js +2 -2
  125. package/dist/components/operations.js.map +1 -1
  126. package/dist/config/configUtils.d.ts +1 -1
  127. package/dist/config/configUtils.js +1 -1
  128. package/dist/config/configUtils.js.map +1 -1
  129. package/dist/dataLayer/CreateTableObject.d.ts +2 -2
  130. package/dist/dataLayer/CreateTableObject.js +2 -2
  131. package/dist/dataLayer/CreateTableObject.js.map +1 -1
  132. package/dist/dataLayer/delete.d.ts +1 -1
  133. package/dist/dataLayer/schema.js +6 -5
  134. package/dist/dataLayer/schema.js.map +1 -1
  135. package/dist/dataLayer/schemaDescribe.js +1 -1
  136. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  137. package/dist/index.d.ts +1 -1
  138. package/dist/index.js +2 -0
  139. package/dist/index.js.map +1 -1
  140. package/dist/resources/DatabaseTransaction.d.ts +1 -1
  141. package/dist/resources/IterableEventQueue.d.ts +1 -1
  142. package/dist/resources/LMDBTransaction.d.ts +5 -1
  143. package/dist/resources/Resource.d.ts +1 -1
  144. package/dist/resources/RocksIndexStore.d.ts +3 -3
  145. package/dist/resources/RocksTransactionLogStore.d.ts +6 -3
  146. package/dist/resources/Table.d.ts +15 -6
  147. package/dist/resources/Table.js +4 -1
  148. package/dist/resources/Table.js.map +1 -1
  149. package/dist/resources/analytics/read.js +32 -22
  150. package/dist/resources/analytics/read.js.map +1 -1
  151. package/dist/resources/analytics/write.js +3 -6
  152. package/dist/resources/analytics/write.js.map +1 -1
  153. package/dist/resources/auditStore.d.ts +3 -3
  154. package/dist/resources/blob.d.ts +25 -2
  155. package/dist/resources/databases.d.ts +12 -2
  156. package/dist/resources/databases.js +22 -19
  157. package/dist/resources/databases.js.map +1 -1
  158. package/dist/resources/search.js +11 -5
  159. package/dist/resources/search.js.map +1 -1
  160. package/dist/resources/transaction.d.ts +2 -1
  161. package/dist/security/auth.js +1 -1
  162. package/dist/security/auth.js.map +1 -1
  163. package/dist/security/cryptoHash.d.ts +2 -2
  164. package/dist/security/jsLoader.js +243 -66
  165. package/dist/security/jsLoader.js.map +1 -1
  166. package/dist/security/keys.js +4 -5
  167. package/dist/security/keys.js.map +1 -1
  168. package/dist/security/user.js +3 -3
  169. package/dist/security/user.js.map +1 -1
  170. package/dist/server/REST.js +16 -2
  171. package/dist/server/REST.js.map +1 -1
  172. package/dist/server/Server.d.ts +2 -1
  173. package/dist/server/Server.js.map +1 -1
  174. package/dist/server/fastifyRoutes/plugins/hdbCore.d.ts +6 -1
  175. package/dist/server/fastifyRoutes.js +2 -0
  176. package/dist/server/fastifyRoutes.js.map +1 -1
  177. package/dist/server/http.js +12 -6
  178. package/dist/server/http.js.map +1 -1
  179. package/dist/server/jobs/JobObject.d.ts +3 -3
  180. package/dist/server/loadRootComponents.js +1 -0
  181. package/dist/server/loadRootComponents.js.map +1 -1
  182. package/dist/server/operationsServer.js +3 -1
  183. package/dist/server/operationsServer.js.map +1 -1
  184. package/dist/server/serverHelpers/JSONStream.d.ts +3 -3
  185. package/dist/server/serverHelpers/Request.d.ts +5 -5
  186. package/dist/server/serverHelpers/requestTimePlugin.d.ts +1 -1
  187. package/dist/server/threads/manageThreads.d.ts +2 -2
  188. package/dist/server/threads/manageThreads.js +50 -35
  189. package/dist/server/threads/manageThreads.js.map +1 -1
  190. package/dist/server/threads/socketRouter.d.ts +1 -1
  191. package/dist/sqlTranslator/deleteTranslator.d.ts +1 -1
  192. package/dist/utility/AWS/AWSConnector.d.ts +3 -2
  193. package/dist/utility/common_utils.d.ts +3 -3
  194. package/dist/utility/environment/systemInformation.d.ts +1 -0
  195. package/dist/utility/functions/date/dateFunctions.d.ts +11 -11
  196. package/dist/utility/globalSchema.d.ts +1 -1
  197. package/dist/utility/hdbTerms.d.ts +3 -0
  198. package/dist/utility/hdbTerms.js +3 -0
  199. package/dist/utility/hdbTerms.js.map +1 -1
  200. package/dist/utility/installation.d.ts +2 -4
  201. package/dist/utility/installation.js.map +1 -1
  202. package/dist/utility/lmdb/commonUtility.d.ts +1 -0
  203. package/dist/utility/lmdb/deleteUtility.d.ts +1 -0
  204. package/dist/utility/lmdb/environmentUtility.d.ts +1 -0
  205. package/dist/utility/lmdb/searchUtility.d.ts +2 -1
  206. package/dist/utility/lmdb/writeUtility.d.ts +1 -0
  207. package/dist/utility/logging/harper_logger.d.ts +6 -6
  208. package/dist/utility/processManagement/processManagement.d.ts +1 -1
  209. package/dist/utility/processManagement/servicesConfig.d.ts +12 -6
  210. package/dist/validation/common_validators.d.ts +4 -3
  211. package/dist/validation/configValidator.d.ts +3 -2
  212. package/index.d.ts +56 -0
  213. package/index.js +41 -0
  214. package/json/systemSchema.json +373 -0
  215. package/launchServiceScripts/launchHarperDB.js +3 -0
  216. package/launchServiceScripts/utility/checkNodeVersion.js +15 -0
  217. package/package.json +21 -3
  218. package/resources/DatabaseTransaction.ts +378 -0
  219. package/resources/ErrorResource.ts +57 -0
  220. package/resources/IterableEventQueue.ts +94 -0
  221. package/resources/LMDBTransaction.ts +349 -0
  222. package/resources/RecordEncoder.ts +702 -0
  223. package/resources/RequestTarget.ts +134 -0
  224. package/resources/Resource.ts +789 -0
  225. package/resources/ResourceInterface.ts +221 -0
  226. package/resources/ResourceInterfaceV2.ts +53 -0
  227. package/resources/ResourceV2.ts +67 -0
  228. package/resources/Resources.ts +162 -0
  229. package/resources/RocksIndexStore.ts +70 -0
  230. package/resources/RocksTransactionLogStore.ts +352 -0
  231. package/resources/Table.ts +4527 -0
  232. package/resources/analytics/hostnames.ts +72 -0
  233. package/resources/analytics/metadata.ts +10 -0
  234. package/resources/analytics/read.ts +252 -0
  235. package/resources/analytics/write.ts +803 -0
  236. package/resources/auditStore.ts +556 -0
  237. package/resources/blob.ts +1268 -0
  238. package/resources/crdt.ts +125 -0
  239. package/resources/dataLoader.ts +527 -0
  240. package/resources/databases.ts +1290 -0
  241. package/resources/graphql.ts +221 -0
  242. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +638 -0
  243. package/resources/indexes/customIndexes.ts +7 -0
  244. package/resources/indexes/vector.ts +38 -0
  245. package/resources/jsResource.ts +86 -0
  246. package/resources/loadEnv.ts +22 -0
  247. package/resources/login.ts +18 -0
  248. package/resources/openApi.ts +409 -0
  249. package/resources/registrationDeprecated.ts +8 -0
  250. package/resources/replayLogs.ts +136 -0
  251. package/resources/roles.ts +98 -0
  252. package/resources/search.ts +1301 -0
  253. package/resources/tracked.ts +584 -0
  254. package/resources/transaction.ts +89 -0
  255. package/resources/transactionBroadcast.ts +258 -0
  256. package/security/auth.ts +376 -0
  257. package/security/certificateVerification/certificateVerificationSource.ts +84 -0
  258. package/security/certificateVerification/configValidation.ts +107 -0
  259. package/security/certificateVerification/crlVerification.ts +623 -0
  260. package/security/certificateVerification/index.ts +121 -0
  261. package/security/certificateVerification/ocspVerification.ts +148 -0
  262. package/security/certificateVerification/pkijs-ed25519-patch.ts +188 -0
  263. package/security/certificateVerification/types.ts +128 -0
  264. package/security/certificateVerification/verificationConfig.ts +138 -0
  265. package/security/certificateVerification/verificationUtils.ts +447 -0
  266. package/security/cryptoHash.js +42 -0
  267. package/security/data_objects/PermissionAttributeResponseObject.js +15 -0
  268. package/security/data_objects/PermissionResponseObject.js +115 -0
  269. package/security/data_objects/PermissionTableResponseObject.js +20 -0
  270. package/security/fastifyAuth.js +169 -0
  271. package/security/impersonation.ts +160 -0
  272. package/security/jsLoader.ts +716 -0
  273. package/security/keys.js +948 -0
  274. package/security/permissionsTranslator.js +300 -0
  275. package/security/role.js +218 -0
  276. package/security/tokenAuthentication.ts +228 -0
  277. package/security/user.ts +449 -0
  278. package/server/DurableSubscriptionsSession.ts +503 -0
  279. package/server/REST.ts +407 -0
  280. package/server/Server.ts +89 -0
  281. package/server/fastifyRoutes/helpers/getCORSOptions.js +36 -0
  282. package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +15 -0
  283. package/server/fastifyRoutes/helpers/getServerOptions.js +33 -0
  284. package/server/fastifyRoutes/plugins/hdbCore.js +39 -0
  285. package/server/fastifyRoutes.ts +205 -0
  286. package/server/graphqlQuerying.ts +700 -0
  287. package/server/http.ts +640 -0
  288. package/server/itc/serverHandlers.js +161 -0
  289. package/server/itc/utility/ITCEventObject.js +10 -0
  290. package/server/jobs/JobObject.js +24 -0
  291. package/server/jobs/jobProcess.js +69 -0
  292. package/server/jobs/jobRunner.js +162 -0
  293. package/server/jobs/jobs.js +304 -0
  294. package/server/loadRootComponents.js +44 -0
  295. package/server/mqtt.ts +485 -0
  296. package/server/nodeName.ts +75 -0
  297. package/server/operationsServer.ts +313 -0
  298. package/server/serverHelpers/Headers.ts +108 -0
  299. package/server/serverHelpers/JSONStream.ts +269 -0
  300. package/server/serverHelpers/OperationFunctionObject.ts +13 -0
  301. package/server/serverHelpers/Request.ts +158 -0
  302. package/server/serverHelpers/contentTypes.ts +637 -0
  303. package/server/serverHelpers/requestTimePlugin.js +57 -0
  304. package/server/serverHelpers/serverHandlers.js +148 -0
  305. package/server/serverHelpers/serverUtilities.ts +473 -0
  306. package/server/serverRegistry.ts +8 -0
  307. package/server/static.ts +187 -0
  308. package/server/status/definitions.ts +37 -0
  309. package/server/status/index.ts +125 -0
  310. package/server/storageReclamation.ts +93 -0
  311. package/server/threads/itc.js +89 -0
  312. package/server/threads/manageThreads.js +594 -0
  313. package/server/threads/socketRouter.ts +360 -0
  314. package/server/threads/threadServer.js +279 -0
  315. package/server/throttle.ts +73 -0
  316. package/sqlTranslator/SelectValidator.js +330 -0
  317. package/sqlTranslator/alasqlFunctionImporter.js +62 -0
  318. package/sqlTranslator/deleteTranslator.js +67 -0
  319. package/sqlTranslator/index.js +242 -0
  320. package/sqlTranslator/sql_statement_bucket.js +472 -0
  321. package/static/defaultConfig.yaml +3 -0
  322. package/studio/web/HDBDogOnly.svg +78 -0
  323. package/studio/web/assets/PPRadioGrotesk-Bold-DDaUYG8E.woff +0 -0
  324. package/studio/web/assets/fa-brands-400-CEJbCg16.woff +0 -0
  325. package/studio/web/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
  326. package/studio/web/assets/fa-brands-400-DnkPfk3o.eot +0 -0
  327. package/studio/web/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
  328. package/studio/web/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
  329. package/studio/web/assets/fa-regular-400-BhTwtT8w.eot +0 -0
  330. package/studio/web/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
  331. package/studio/web/assets/fa-regular-400-DFnMcJPd.woff +0 -0
  332. package/studio/web/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
  333. package/studio/web/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
  334. package/studio/web/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
  335. package/studio/web/assets/fa-solid-900-B6Axprfb.eot +0 -0
  336. package/studio/web/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
  337. package/studio/web/assets/fa-solid-900-DOXgCApm.woff +0 -0
  338. package/studio/web/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
  339. package/studio/web/assets/index-BTgXJX9d.js +235 -0
  340. package/studio/web/assets/index-BTgXJX9d.js.map +1 -0
  341. package/studio/web/assets/index-C-GXfcup.js +37 -0
  342. package/studio/web/assets/index-C-GXfcup.js.map +1 -0
  343. package/studio/web/assets/index-PFlNdimM.js +2 -0
  344. package/studio/web/assets/index-PFlNdimM.js.map +1 -0
  345. package/studio/web/assets/index-Y2g_iFpU.css +1 -0
  346. package/studio/web/assets/index-jiPwkrsB.css +1 -0
  347. package/studio/web/assets/index.lazy-C3TJZJ4o.js +266 -0
  348. package/studio/web/assets/index.lazy-C3TJZJ4o.js.map +1 -0
  349. package/studio/web/assets/profiler-DotzgiCJ.js +2 -0
  350. package/studio/web/assets/profiler-DotzgiCJ.js.map +1 -0
  351. package/studio/web/assets/react-redux-VxUEx_mU.js +6 -0
  352. package/studio/web/assets/react-redux-VxUEx_mU.js.map +1 -0
  353. package/studio/web/assets/startRecording-B_9J9Csd.js +3 -0
  354. package/studio/web/assets/startRecording-B_9J9Csd.js.map +1 -0
  355. package/studio/web/fabric-signup-background.webp +0 -0
  356. package/studio/web/fabric-signup-text.png +0 -0
  357. package/studio/web/favicon_purple.png +0 -0
  358. package/studio/web/github-icon.svg +15 -0
  359. package/studio/web/harper-fabric_black.png +0 -0
  360. package/studio/web/harper-fabric_white.png +0 -0
  361. package/studio/web/harper-studio_white.png +0 -0
  362. package/studio/web/index.html +16 -0
  363. package/studio/web/running.css +148 -0
  364. package/studio/web/running.html +147 -0
  365. package/studio/web/running.js +111 -0
  366. package/upgrade/UpgradeObjects.js +13 -0
  367. package/upgrade/directives/directivesController.js +90 -0
  368. package/upgrade/directivesManager.js +139 -0
  369. package/upgrade/upgradePrompt.js +124 -0
  370. package/upgrade/upgradeUtilities.js +28 -0
  371. package/utility/AWS/AWSConnector.js +29 -0
  372. package/utility/OperationFunctionCaller.js +63 -0
  373. package/utility/assignCmdEnvVariables.js +62 -0
  374. package/utility/common_utils.js +867 -0
  375. package/utility/environment/environmentManager.js +208 -0
  376. package/utility/environment/systemInformation.js +355 -0
  377. package/utility/errors/commonErrors.js +267 -0
  378. package/utility/errors/hdbError.js +146 -0
  379. package/utility/functions/date/dateFunctions.js +65 -0
  380. package/utility/functions/geo.js +355 -0
  381. package/utility/functions/sql/alaSQLExtension.js +104 -0
  382. package/utility/globalSchema.js +35 -0
  383. package/utility/hdbTerms.ts +819 -0
  384. package/utility/install/checkJWTTokensExist.js +62 -0
  385. package/utility/install/harperdb.conf +15 -0
  386. package/utility/install/harperdb.service +14 -0
  387. package/utility/install/installer.js +635 -0
  388. package/utility/installation.ts +30 -0
  389. package/utility/lmdb/DBIDefinition.js +20 -0
  390. package/utility/lmdb/DeleteRecordsResponseObject.js +25 -0
  391. package/utility/lmdb/InsertRecordsResponseObject.js +22 -0
  392. package/utility/lmdb/OpenDBIObject.js +31 -0
  393. package/utility/lmdb/OpenEnvironmentObject.js +41 -0
  394. package/utility/lmdb/UpdateRecordsResponseObject.js +25 -0
  395. package/utility/lmdb/UpsertRecordsResponseObject.js +22 -0
  396. package/utility/lmdb/cleanLMDBMap.js +65 -0
  397. package/utility/lmdb/commonUtility.js +119 -0
  398. package/utility/lmdb/deleteUtility.js +128 -0
  399. package/utility/lmdb/environmentUtility.js +477 -0
  400. package/utility/lmdb/searchCursorFunctions.js +187 -0
  401. package/utility/lmdb/searchUtility.js +918 -0
  402. package/utility/lmdb/terms.js +57 -0
  403. package/utility/lmdb/writeUtility.js +407 -0
  404. package/utility/logging/harper_logger.js +876 -0
  405. package/utility/logging/logRotator.js +157 -0
  406. package/utility/logging/logger.ts +24 -0
  407. package/utility/logging/readLog.js +355 -0
  408. package/utility/logging/transactionLog.js +57 -0
  409. package/utility/mount_hdb.js +59 -0
  410. package/utility/npmUtilities.js +102 -0
  411. package/utility/operationPermissions.ts +112 -0
  412. package/utility/operation_authorization.js +836 -0
  413. package/utility/packageUtils.js +55 -0
  414. package/utility/password.ts +99 -0
  415. package/utility/processManagement/processManagement.js +187 -0
  416. package/utility/processManagement/servicesConfig.js +56 -0
  417. package/utility/scripts/restartHdb.js +24 -0
  418. package/utility/scripts/user_data.sh +13 -0
  419. package/utility/signalling.js +36 -0
  420. package/utility/terms/certificates.js +81 -0
  421. package/utility/when.ts +20 -0
  422. package/v1.d.ts +39 -0
  423. package/v1.js +41 -0
  424. package/v2.d.ts +39 -0
  425. package/v2.js +41 -0
  426. package/validation/bulkDeleteValidator.js +24 -0
  427. package/validation/check_permissions.js +19 -0
  428. package/validation/common_validators.js +95 -0
  429. package/validation/configValidator.js +331 -0
  430. package/validation/deleteValidator.js +15 -0
  431. package/validation/fileLoadValidator.js +153 -0
  432. package/validation/insertValidator.js +40 -0
  433. package/validation/installValidator.js +37 -0
  434. package/validation/readLogValidator.js +64 -0
  435. package/validation/role_validation.js +320 -0
  436. package/validation/schemaMetadataValidator.js +42 -0
  437. package/validation/searchValidator.js +166 -0
  438. package/validation/statusValidator.ts +66 -0
  439. package/validation/transactionLogValidator.js +33 -0
  440. package/validation/user_validation.js +55 -0
  441. package/validation/validationWrapper.js +105 -0
  442. package/dist/resources/analytics/profile.d.ts +0 -2
  443. package/dist/resources/analytics/profile.js +0 -144
  444. package/dist/resources/analytics/profile.js.map +0 -1
@@ -0,0 +1,646 @@
1
+ import { type Logger } from '../utility/logging/logger.ts';
2
+ import { getConfigObj, getConfigValue } from '../config/configUtils.js';
3
+ import { CONFIG_PARAMS } from '../utility/hdbTerms.js';
4
+ import logger from '../utility/logging/harper_logger.js';
5
+
6
+ import { dirname, extname, join } from 'node:path';
7
+ import {
8
+ access,
9
+ constants,
10
+ cp,
11
+ mkdir,
12
+ mkdtemp,
13
+ readdir,
14
+ readFile,
15
+ rm,
16
+ stat,
17
+ symlink,
18
+ writeFile,
19
+ } from 'node:fs/promises';
20
+ import { spawn } from 'node:child_process';
21
+ import { createReadStream, existsSync, readdirSync } from 'node:fs';
22
+ import { Readable } from 'node:stream';
23
+ import { pipeline } from 'node:stream/promises';
24
+
25
+ import { extract } from 'tar-fs';
26
+ import gunzip from 'gunzip-maybe';
27
+
28
+ interface ApplicationConfig {
29
+ // define known config properties
30
+ package: string;
31
+ install?: {
32
+ command?: string;
33
+ timeout?: number;
34
+ };
35
+ // an application config can have other arbitrary properties
36
+ [key: string]: unknown;
37
+ }
38
+
39
+ export class InvalidPackageIdentifierError extends TypeError {
40
+ constructor(applicationName: string, packageIdentifier: unknown) {
41
+ super(
42
+ `Invalid 'package' property for application ${applicationName}: expected string, got ${typeof packageIdentifier}`
43
+ );
44
+ }
45
+ }
46
+
47
+ export class InvalidInstallPropertyError extends TypeError {
48
+ constructor(applicationName: string, installProperty: unknown) {
49
+ super(
50
+ `Invalid 'install' property for application ${applicationName}: expected object, got ${typeof installProperty}`
51
+ );
52
+ }
53
+ }
54
+
55
+ export class InvalidInstallCommandError extends TypeError {
56
+ constructor(applicationName: string, command: unknown) {
57
+ super(
58
+ `Invalid 'install.command' property for application ${applicationName}: expected string, got ${typeof command}`
59
+ );
60
+ }
61
+ }
62
+
63
+ export class InvalidInstallTimeoutError extends TypeError {
64
+ constructor(applicationName: string, timeout: unknown) {
65
+ super(
66
+ `Invalid 'install.timeout' property for application ${applicationName}: expected non-negative number, got ${typeof timeout}`
67
+ );
68
+ }
69
+ }
70
+
71
+ export function assertApplicationConfig(
72
+ applicationName: string,
73
+ applicationConfig: Record<'package', unknown> & Record<string, unknown>
74
+ ): asserts applicationConfig is ApplicationConfig {
75
+ if (typeof applicationConfig.package !== 'string') {
76
+ throw new InvalidPackageIdentifierError(applicationName, applicationConfig.package);
77
+ }
78
+
79
+ if ('install' in applicationConfig) {
80
+ if (
81
+ typeof applicationConfig.install !== 'object' ||
82
+ applicationConfig.install === null ||
83
+ Array.isArray(applicationConfig.install)
84
+ ) {
85
+ throw new InvalidInstallPropertyError(applicationName, applicationConfig.install);
86
+ }
87
+
88
+ if ('command' in applicationConfig.install && typeof applicationConfig.install.command !== 'string') {
89
+ throw new InvalidInstallCommandError(applicationName, applicationConfig.install.command);
90
+ }
91
+
92
+ if (
93
+ 'timeout' in applicationConfig.install &&
94
+ (typeof applicationConfig.install.timeout !== 'number' || applicationConfig.install.timeout < 0)
95
+ ) {
96
+ throw new InvalidInstallTimeoutError(applicationName, applicationConfig.install.timeout);
97
+ }
98
+ }
99
+ }
100
+
101
+ /**
102
+ * Extract an application given payload (content of the application) or package (npm-compatible identifier to the application).
103
+ *
104
+ * Only one of `application.payload` or `application.package` should be specified; otherwise, an error is thrown.
105
+ *
106
+ * Writes the application to the configured components root directory using the `application.name` and overwrites any existing directory.
107
+ *
108
+ * This method should only be called from the main thread
109
+ */
110
+ export async function extractApplication(application: Application) {
111
+ // Can't specify neither
112
+ if (!application.payload && !application.packageIdentifier) {
113
+ throw new Error('Either payload or package must be provided');
114
+ }
115
+
116
+ // Can't specify both
117
+ if (application.payload && application.packageIdentifier) {
118
+ throw new Error('Both payload and package cannot be provided');
119
+ }
120
+
121
+ // Resolve the tarball from the input
122
+ let tarballPath: string;
123
+ let tarball: Readable;
124
+ if (application.payload) {
125
+ // Given a payload, create a Readable from the Buffer or string
126
+ tarball = Readable.from(
127
+ application.payload instanceof Buffer ? application.payload : Buffer.from(application.payload, 'base64')
128
+ );
129
+ } else {
130
+ // Given a package, there are a a couple options
131
+ const parentDirPath = dirname(application.dirPath);
132
+
133
+ // If the package identifier is a file path we need to check if its a tarball or a directory
134
+ if (application.packageIdentifier.startsWith('file:')) {
135
+ const packagePath = application.packageIdentifier.slice(5);
136
+ try {
137
+ // Have to remove the 'file:' prefix in order to use fs methods
138
+ const stats = await stat(packagePath);
139
+
140
+ if (stats.isDirectory()) {
141
+ // If its a directory, symlink
142
+ await symlink(packagePath, application.dirPath, 'dir');
143
+ // And return early since we're done; no extraction needed
144
+ return;
145
+ }
146
+
147
+ if (!stats.isFile()) {
148
+ throw new Error(`File path specified in package identifier is not a file or directory: ${packagePath}`);
149
+ }
150
+
151
+ // If its a file, we assume it can be unzipped and extracted.
152
+ // We are using maybe-gunzip to handle both gzipped and non-gzipped tarballs
153
+ // And then we are happy to let the `tar-fs` library handle the extraction.
154
+ // Maybe worth adding some detection or at least some error handling if that step below fails.
155
+ tarballPath = packagePath;
156
+ tarball = createReadStream(tarballPath);
157
+ } catch (err) {
158
+ if (err.code === 'ENOENT') {
159
+ throw new Error(`File path specified in package identifier does not exist: ${packagePath}`);
160
+ } else {
161
+ throw err;
162
+ }
163
+ }
164
+ } else {
165
+ // Given a package, resolve using `npm pack` (downloads the package as a tarball and writes the path to stdout)
166
+ const {
167
+ stdout: tarballFilePath,
168
+ code,
169
+ stderr,
170
+ } = await nonInteractiveSpawn(application.name, 'npm', ['pack', application.packageIdentifier], parentDirPath);
171
+ if (code !== 0) throw new Error(`Failed to download package ${application.packageIdentifier}: ${stderr}`);
172
+ tarballPath = join(parentDirPath, tarballFilePath.trim());
173
+ // Create a Readable from the tarball
174
+ tarball = createReadStream(tarballPath);
175
+ }
176
+ }
177
+
178
+ // Create the application directory
179
+ try {
180
+ await access(application.dirPath, constants.F_OK);
181
+ // directory already exists; clear it
182
+ await rm(application.dirPath, { recursive: true, force: true });
183
+ } catch (err) {
184
+ // Ignore does not exist error
185
+ if (err.code !== 'ENOENT') {
186
+ throw err;
187
+ }
188
+ }
189
+ // Finally, create the application directory fresh
190
+ await mkdir(application.dirPath, { recursive: true });
191
+
192
+ // Now pipeline the tarball into maybe-gunzip then tar-fs to reliably decompress and extract the contents
193
+ await pipeline(tarball, gunzip(), extract(application.dirPath));
194
+
195
+ // If the extracted directory contains a single folder, move the contents up one level
196
+ // The `npm pack` command does this (the top-level folder is called "package")
197
+ // Other packing tools may have similar behavior, but the directory name is not guaranteed.
198
+ const extracted = await readdir(application.dirPath, { withFileTypes: true });
199
+ if (extracted.length === 1 && extracted[0].isDirectory()) {
200
+ const topLevelDirPath = join(application.dirPath, extracted[0].name);
201
+
202
+ const tempDirPath = await mkdtemp(application.dirPath);
203
+
204
+ // Copy contents of top-level directory to temp directory (in order to avoid collisions of top-level directory name and one of the contents)
205
+ await cp(topLevelDirPath, tempDirPath, { recursive: true });
206
+ // Remove top-level directory
207
+ await rm(topLevelDirPath, { recursive: true, force: true });
208
+ // Copy contents of temp directory to application directory
209
+ await cp(tempDirPath, application.dirPath, { recursive: true });
210
+ // Finally, remove the temp dir
211
+ await rm(tempDirPath, { recursive: true, force: true });
212
+ }
213
+
214
+ // Clean up the original tarball
215
+ if (tarballPath) {
216
+ await rm(tarballPath, { force: true });
217
+ }
218
+ }
219
+
220
+ /**
221
+ * Install an application to its relative `application.dirPath` using either a
222
+ * configured `application.install` command, a derived package manager from the
223
+ * application's `package.json#devEngines`, or falling back to the default
224
+ * package manager, `npm`.
225
+ *
226
+ * Will return early if `node_modules` already exists within the `application.dirPath`
227
+ *
228
+ * This method should only be called from the main thread
229
+ */
230
+ export async function installApplication(application: Application) {
231
+ let packageJSON: any;
232
+ try {
233
+ packageJSON = JSON.parse(await readFile(join(application.dirPath, 'package.json'), 'utf8'));
234
+ } catch (err) {
235
+ if (err.code !== 'ENOENT') throw err;
236
+ // If no package.json, nothing to install
237
+ application.logger.debug(`Application ${application.name} has no package.json; skipping install`);
238
+ return;
239
+ }
240
+ try {
241
+ // Does node_modules exist?
242
+ await access(join(application.dirPath, 'node_modules'), constants.F_OK);
243
+ application.logger.debug(`Application ${application.name} already has node_modules; skipping install`);
244
+ return;
245
+ } catch (err) {
246
+ if (err.code !== 'ENOENT') throw err;
247
+ // If node_modules doesn't exist, we need to install dependencies
248
+ }
249
+
250
+ // If custom install command is specified, run it
251
+ if (application.install?.command) {
252
+ const [command, ...args] = application.install.command.split(' ');
253
+ const { stdout, stderr, code } = await nonInteractiveSpawn(
254
+ application.name,
255
+ command,
256
+ args,
257
+ application.dirPath,
258
+ application.install?.timeout
259
+ );
260
+ // if it succeeds, return
261
+ if (code === 0) {
262
+ return;
263
+ }
264
+ if (stdout) {
265
+ printStd(application.name, command, stdout, 'stdout', 'warn');
266
+ }
267
+
268
+ if (stderr) {
269
+ printStd(application.name, command, stderr, 'stderr', 'warn');
270
+ }
271
+ // and throw a descriptive error
272
+ throw new Error(
273
+ `Failed to install dependencies for ${application.name} using custom install command: ${application.install.command}. Exit code: ${code}`
274
+ );
275
+ }
276
+
277
+ // Next, try package.json devEngines field
278
+ const { packageManager } = packageJSON.devEngines || {};
279
+
280
+ // Custom package manager specified
281
+ if (packageManager) {
282
+ // On any given system we want to leverage the `name` to match the package manager executable
283
+ let onFail: string | undefined = packageManager.onFail;
284
+
285
+ const validOnFailValues = ['ignore', 'warn', 'error'];
286
+
287
+ if (onFail === 'download') {
288
+ application.logger.warn(
289
+ 'Harper currently does not support `devEngines.packageManager.onFail = "download"`. Defaulting to "error"'
290
+ );
291
+ onFail = 'error';
292
+ } else if (onFail && !validOnFailValues.includes(onFail)) {
293
+ application.logger.error(
294
+ `Invalid \`devEngines.packageManager.onFail\` value: "${onFail}". Expected one of ${validOnFailValues.map((v) => `"${v}"`).join(', ')}. Defaulting to "error"`
295
+ );
296
+ onFail = 'error';
297
+ }
298
+
299
+ onFail = onFail || 'error';
300
+
301
+ // TODO: Implement a version check / resolution system
302
+ // For example, say they specify a specific major version for their package manager
303
+ // Maybe on our system, we have all of the supported majors (for a given Node.js major) of any supported package manager.
304
+ // Then we can do something like <name>@<version> for the corresponding executable.
305
+ // `devEngines: { packageManager: { name: 'pnpm', version: '>=7' } }`
306
+ // Would result in `pnpm@7` being used as the executable.
307
+ // Important note: an `npm` version should not be specifiable; the only valid npm version is the one installed alongside Node.js
308
+
309
+ const { stdout, stderr, code } = await nonInteractiveSpawn(
310
+ application.name,
311
+ (application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + packageManager.name,
312
+ ['install'], // All of `npm`, `yarn`, and `pnpm` support the `install` command. If we need to configure options here we may have to use some other defaults though
313
+ application.dirPath,
314
+ application.install?.timeout
315
+ );
316
+
317
+ // if it succeeds, return
318
+ if (code === 0) {
319
+ return;
320
+ }
321
+
322
+ // Otherwise handle failure case based on `onFail` value
323
+ if (onFail === 'error') {
324
+ // Log the std outputs using the error log level (in case the user doesn't have debug level set)
325
+ if (stdout) {
326
+ printStd(application.name, packageManager.name, stdout, 'stdout', 'error');
327
+ }
328
+
329
+ if (stderr) {
330
+ printStd(application.name, packageManager.name, stderr, 'stderr', 'error');
331
+ }
332
+
333
+ // And throw an error instead of continuing
334
+ throw new Error(
335
+ `Failed to install dependencies for ${application.name} using ${packageManager.name}. Exit code: ${code}`
336
+ );
337
+ }
338
+
339
+ // If onFail is 'warn', print outputs using the warn level, plus an additional message
340
+ if (onFail === 'warn') {
341
+ if (stdout) {
342
+ printStd(application.name, packageManager.name, stdout, 'stdout', 'warn');
343
+ }
344
+
345
+ if (stderr) {
346
+ printStd(application.name, packageManager.name, stderr, 'stderr', 'warn');
347
+ }
348
+
349
+ application.logger.warn(
350
+ `Failed to install dependencies for ${application.name} using ${packageManager.name}. Exit code: ${code}`
351
+ );
352
+ }
353
+
354
+ // But then fall through to installing with npm
355
+ }
356
+
357
+ // Finally, default to running `npm install`
358
+ const { stdout, stderr, code } = await nonInteractiveSpawn(
359
+ application.name,
360
+ (application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + 'npm',
361
+ ['install', '--force'],
362
+ application.dirPath
363
+ );
364
+
365
+ // if it succeeds, return
366
+ if (code === 0) {
367
+ return;
368
+ }
369
+
370
+ // Otherwise, print the stdout and stderr outputs
371
+ if (stdout) {
372
+ printStd(application.name, 'npm', stdout, 'stdout', 'warn');
373
+ }
374
+
375
+ if (stderr) {
376
+ printStd(application.name, 'npm', stderr, 'stderr', 'error');
377
+ }
378
+
379
+ // and throw a descriptive error
380
+ throw new Error(`Failed to install dependencies for ${application.name} using npm default. Exit code: ${code}`);
381
+ }
382
+
383
+ interface ApplicationOptions {
384
+ name: string;
385
+ payload?: Buffer | string;
386
+ packageIdentifier?: string;
387
+ install?: { command?: string; timeout?: number };
388
+ }
389
+
390
+ export class Application {
391
+ name: string;
392
+ payload?: Buffer | string;
393
+ packageIdentifier?: string;
394
+ install?: { command?: string; timeout?: number };
395
+ dirPath: string;
396
+ logger: Logger;
397
+ packageManagerPrefix: string; // can be used to configure a package manager prefix, specifically "sfw".
398
+
399
+ constructor({ name, payload, packageIdentifier, install }: ApplicationOptions) {
400
+ this.name = name;
401
+ this.payload = payload;
402
+ this.packageIdentifier = packageIdentifier && derivePackageIdentifier(packageIdentifier);
403
+ this.install = install;
404
+ this.dirPath = join(getConfigValue(CONFIG_PARAMS.COMPONENTSROOT), name);
405
+ this.logger = logger.loggerWithTag(name);
406
+ this.packageManagerPrefix = getConfigValue(CONFIG_PARAMS.APPLICATIONS_PACKAGEMANAGERPREFIX);
407
+ }
408
+ }
409
+
410
+ /**
411
+ * Based on an old implementation for a method called `getPkgPrefix()` that was used
412
+ * during the installation process in order to actually resolve what the user specifies for a
413
+ * component matching some of npm's package resolution rules.
414
+ */
415
+ export function derivePackageIdentifier(packageIdentifier: string) {
416
+ if (packageIdentifier.includes(':')) {
417
+ return packageIdentifier;
418
+ }
419
+ if (packageIdentifier.startsWith('@') || (!packageIdentifier.startsWith('@') && !packageIdentifier.includes('/'))) {
420
+ return `npm:${packageIdentifier}`;
421
+ }
422
+ if (extname(packageIdentifier) || existsSync(packageIdentifier)) {
423
+ return `file:${packageIdentifier}`;
424
+ }
425
+
426
+ return `github:${packageIdentifier}`;
427
+ }
428
+
429
+ /**
430
+ * Extract and install the specified application.
431
+ *
432
+ * This method should only be called from the main thread
433
+ *
434
+ * @param application The application to prepare.
435
+ * @returns A promise that resolves when all preparation steps complete.
436
+ */
437
+ export function prepareApplication(application: Application) {
438
+ return extractApplication(application).then(() => installApplication(application));
439
+ }
440
+
441
+ /**
442
+ * Install all applications specified in the root config.
443
+ *
444
+ * This method should only be called from the main thread otherwise certain
445
+ * operations may conflict with each other (such as writing to the same directory).
446
+ */
447
+ export async function installApplications() {
448
+ const applicationInstallationPromises: Promise<void>[] = [];
449
+
450
+ // first install any built-in components specified from env vars
451
+ for (const { name, packageIdentifier } of getEnvBuiltInComponents()) {
452
+ if (packageIdentifier.startsWith('@/')) {
453
+ // this is a package relative module id, so later we will resolve it, but we don't need to install anything
454
+ continue;
455
+ }
456
+ const application = new Application({
457
+ name,
458
+ packageIdentifier,
459
+ });
460
+
461
+ applicationInstallationPromises.push(prepareApplication(application));
462
+ }
463
+
464
+ const config = getConfigObj();
465
+
466
+ const componentsRootDirPath = getConfigValue(CONFIG_PARAMS.COMPONENTSROOT);
467
+
468
+ // Ensure component directory exists
469
+ await mkdir(componentsRootDirPath, { recursive: true });
470
+
471
+ const harperApplicationLockPath = join(getConfigValue(CONFIG_PARAMS.ROOTPATH), 'harper-application-lock.json');
472
+
473
+ let harperApplicationLock: { applications: Record<string, ApplicationConfig> } = { applications: {} };
474
+ try {
475
+ harperApplicationLock = JSON.parse(await readFile(harperApplicationLockPath, 'utf8'));
476
+ } catch (error) {
477
+ // Ignore file not found error; will create new lock file after installations
478
+ if (error.code !== 'ENOENT') {
479
+ throw error;
480
+ }
481
+ }
482
+
483
+ for (const [name, applicationConfig] of Object.entries(config)) {
484
+ // Pre-validation check if the configuration is actually for an application
485
+ // Don't want to throw an error here as the config may contain non-application entries
486
+ if (typeof applicationConfig !== 'object' || applicationConfig === null || !('package' in applicationConfig)) {
487
+ continue;
488
+ }
489
+
490
+ try {
491
+ // Then do proper error-based validation with TypeScript `asserts` to provide type safety
492
+ // This will throw if the config is invalid
493
+ assertApplicationConfig(name, applicationConfig);
494
+
495
+ const application = new Application({
496
+ name,
497
+ packageIdentifier: applicationConfig.package,
498
+ install: applicationConfig.install,
499
+ });
500
+
501
+ // Lock check: only install if not already installed with matching configuration
502
+ if (
503
+ existsSync(application.dirPath) &&
504
+ harperApplicationLock.applications[name] &&
505
+ JSON.stringify(harperApplicationLock.applications[name]) === JSON.stringify(applicationConfig)
506
+ ) {
507
+ logger.info?.(`Application ${name} is already installed with matching configuration; skipping installation`);
508
+ continue;
509
+ }
510
+
511
+ applicationInstallationPromises.push(prepareApplication(application));
512
+
513
+ harperApplicationLock.applications[name] = applicationConfig;
514
+ } catch (error) {
515
+ logger.error?.(`Skipping installation of application ${name} due to invalid configuration: ${error.message}`);
516
+ }
517
+ }
518
+
519
+ const applicationInstallationStatuses = await Promise.allSettled(applicationInstallationPromises);
520
+ logger.debug?.(applicationInstallationStatuses);
521
+ logger.info?.('All root applications loaded');
522
+
523
+ // Finally, write the lock file
524
+ await writeFile(harperApplicationLockPath, JSON.stringify(harperApplicationLock, null, 2), 'utf8');
525
+ }
526
+
527
+ function getGitSSHCommand() {
528
+ const rootDir = getConfigValue(CONFIG_PARAMS.ROOTPATH);
529
+ const sshDir = join(rootDir, 'ssh');
530
+ if (existsSync(sshDir)) {
531
+ for (const file of readdirSync(sshDir)) {
532
+ if (file.includes('.key')) {
533
+ return `ssh -F ${join(sshDir, 'config')} -o UserKnownHostsFile=${join(sshDir, 'known_hosts')}`;
534
+ }
535
+ }
536
+ }
537
+ }
538
+
539
+ /**
540
+ * Execute a command (using `spawn`) with stdin ignored.
541
+ *
542
+ * Stdout is logged chunk-by-chunk. Stderr is buffered and then logged line-by-line.
543
+ *
544
+ * Rejects with an error if the command fails or times out.
545
+ *
546
+ * @param command The command to run.
547
+ * @param args The arguments to pass to the command.
548
+ * @param cwd The working directory for the command.
549
+ * @param timeoutMs The timeout for the command in milliseconds. Defaults to 5 minutes.
550
+ * @returns A promise that resolves when the command completes.
551
+ */
552
+ export function nonInteractiveSpawn(
553
+ applicationName: string,
554
+ command: string,
555
+ args: string[],
556
+ cwd: string,
557
+ timeoutMs: number = 5 * 60 * 1000
558
+ ): Promise<{ stdout: string; stderr: string; code: number }> {
559
+ return new Promise((resolve, reject) => {
560
+ logger
561
+ .loggerWithTag(`${applicationName}:spawn:${command}`)
562
+ .debug?.(`Executing \`${command} ${args.join(' ')}\` in ${cwd}`);
563
+
564
+ const env = { ...process.env };
565
+
566
+ const gitSSHCommand = getGitSSHCommand();
567
+ if (gitSSHCommand) {
568
+ env.GIT_SSH_COMMAND = gitSSHCommand;
569
+ }
570
+
571
+ const childProcess = spawn(command, args, {
572
+ shell: true,
573
+ cwd,
574
+ env,
575
+ stdio: ['ignore', 'pipe', 'pipe'],
576
+ });
577
+
578
+ const timeout = setTimeout(() => {
579
+ childProcess.kill();
580
+ reject(new Error(`Command\`${command} ${args.join(' ')}\` timed out after ${timeoutMs}ms`));
581
+ }, timeoutMs);
582
+
583
+ let stdout = '';
584
+ childProcess.stdout.on('data', (chunk) => {
585
+ // buffer stdout for later resolve
586
+ stdout += chunk.toString();
587
+ // log stdout lines immediately
588
+ // TODO: Technically nothing guarantees that a chunk will be a complete line so need to implement
589
+ // something here to buffer until a newline character, then log the complete line
590
+ logger.loggerWithTag(`${applicationName}:spawn:${command}:stdout`).debug?.(chunk.toString());
591
+ });
592
+
593
+ // buffer stderr
594
+ let stderr = '';
595
+ childProcess.stderr.on('data', (chunk) => {
596
+ stderr += chunk.toString();
597
+ });
598
+
599
+ childProcess.on('error', (error) => {
600
+ clearTimeout(timeout);
601
+ // Print out stderr before rejecting
602
+ if (stderr) {
603
+ printStd(applicationName, command, stderr, 'stderr');
604
+ }
605
+ reject(error);
606
+ });
607
+
608
+ childProcess.on('close', (code) => {
609
+ clearTimeout(timeout);
610
+ if (stderr) {
611
+ printStd(applicationName, command, stderr, 'stderr');
612
+ }
613
+ logger.loggerWithTag(`${applicationName}:spawn:${command}`).debug?.(`Process exited with code ${code}`);
614
+ resolve({
615
+ stdout,
616
+ stderr,
617
+ code,
618
+ });
619
+ });
620
+ });
621
+ }
622
+
623
+ export function getEnvBuiltInComponents() {
624
+ const builtInComponents: { name: string; packageIdentifier: string }[] = [];
625
+ if (process.env.HARPER_BUILTIN_COMPONENTS) {
626
+ for (const componentDefinition of process.env.HARPER_BUILTIN_COMPONENTS.split(',')) {
627
+ const [name, packageIdentifier] = componentDefinition.trim().split('=');
628
+ if (!componentDefinition) continue;
629
+ builtInComponents.push({ name, packageIdentifier });
630
+ }
631
+ }
632
+ return builtInComponents;
633
+ }
634
+
635
+ function printStd(
636
+ applicationName: string,
637
+ command: string,
638
+ stdString: string,
639
+ stdStreamLabel: 'stdout' | 'stderr',
640
+ level: 'debug' | 'warn' | 'error' = 'debug'
641
+ ) {
642
+ const stdLogger = logger.loggerWithTag(`${applicationName}:spawn:${command}:${stdStreamLabel}`);
643
+ for (const line of stdString.split('\n')) {
644
+ stdLogger[level]?.(line);
645
+ }
646
+ }
@@ -0,0 +1,49 @@
1
+ import type { Resources } from '../resources/Resources.ts';
2
+ import { type Server } from '../server/Server.ts';
3
+ import { loggerWithTag } from '../utility/logging/harper_logger.js';
4
+ import { scopedImport } from '../security/jsLoader.ts';
5
+ import * as env from '../utility/environment/environmentManager.js';
6
+ import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
7
+
8
+ export class MissingDefaultFilesOptionError extends Error {
9
+ constructor() {
10
+ super('No default files option exists. Ensure `files` is specified in config.yaml');
11
+ this.name = 'MissingDefaultFilesOptionError';
12
+ }
13
+ }
14
+
15
+ /**
16
+ * This class is used to represent the application scope for the VM context used for loading modules within an application
17
+ */
18
+ export class ApplicationScope {
19
+ logger: any;
20
+ resources: Resources;
21
+ server: Server;
22
+ mode?: 'none' | 'vm' | 'compartment'; // option to set this from the scope
23
+ dependencyContainment?: boolean; // option to set this from the scope
24
+ verifyPath?: string;
25
+ config: any;
26
+ constructor(name: string, resources: Resources, server: Server, verifyPath?: string) {
27
+ this.logger = loggerWithTag(name);
28
+
29
+ this.resources = resources;
30
+ this.server = server;
31
+
32
+ this.mode = env.get(CONFIG_PARAMS.APPLICATIONS_CONTAINMENT) ?? 'vm';
33
+ this.dependencyContainment = Boolean(env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYCONTAINMENT));
34
+ this.verifyPath = verifyPath;
35
+ }
36
+
37
+ /**
38
+ * The compartment that is used for this scope and any imports that it makes
39
+ */
40
+ compartment?: Promise<any>;
41
+ /**
42
+ * Import a file into the scope's sandbox.
43
+ * @param filePath - The path of the file to import.
44
+ * @returns A promise that resolves with the imported module or value.
45
+ */
46
+ async import(filePath: string): Promise<unknown> {
47
+ return scopedImport(filePath, this);
48
+ }
49
+ }