@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (444) hide show
  1. package/bin/BinObjects.js +17 -0
  2. package/bin/cliOperations.js +157 -0
  3. package/bin/copyDb.ts +280 -0
  4. package/bin/harper.js +156 -0
  5. package/bin/install.js +15 -0
  6. package/bin/lite.js +5 -0
  7. package/bin/restart.js +201 -0
  8. package/bin/run.js +409 -0
  9. package/bin/status.js +65 -0
  10. package/bin/stop.js +22 -0
  11. package/bin/upgrade.js +134 -0
  12. package/components/Application.ts +646 -0
  13. package/components/ApplicationScope.ts +49 -0
  14. package/components/Component.ts +53 -0
  15. package/components/ComponentV1.ts +342 -0
  16. package/components/DEFAULT_CONFIG.ts +18 -0
  17. package/components/EntryHandler.ts +227 -0
  18. package/components/Logger.ts +14 -0
  19. package/components/OptionsWatcher.ts +354 -0
  20. package/components/PluginModule.ts +6 -0
  21. package/components/Scope.ts +329 -0
  22. package/components/componentLoader.ts +529 -0
  23. package/components/deriveCommonPatternBase.ts +31 -0
  24. package/components/deriveGlobOptions.ts +44 -0
  25. package/components/deriveURLPath.ts +57 -0
  26. package/components/operations.js +658 -0
  27. package/components/operationsValidation.js +246 -0
  28. package/components/packageComponent.ts +39 -0
  29. package/components/requestRestart.ts +26 -0
  30. package/components/resolveBaseURLPath.ts +38 -0
  31. package/components/status/ComponentStatus.ts +110 -0
  32. package/components/status/ComponentStatusRegistry.ts +251 -0
  33. package/components/status/api.ts +153 -0
  34. package/components/status/crossThread.ts +405 -0
  35. package/components/status/errors.ts +152 -0
  36. package/components/status/index.ts +44 -0
  37. package/components/status/internal.ts +65 -0
  38. package/components/status/registry.ts +12 -0
  39. package/components/status/types.ts +96 -0
  40. package/config/RootConfigWatcher.ts +59 -0
  41. package/config/configHelpers.ts +11 -0
  42. package/config/configUtils.js +967 -0
  43. package/config/harperConfigEnvVars.ts +641 -0
  44. package/dataLayer/CreateAttributeObject.js +25 -0
  45. package/dataLayer/CreateTableObject.js +11 -0
  46. package/dataLayer/DataLayerObjects.js +43 -0
  47. package/dataLayer/DeleteBeforeObject.js +22 -0
  48. package/dataLayer/DeleteObject.js +25 -0
  49. package/dataLayer/DropAttributeObject.js +11 -0
  50. package/dataLayer/GetBackupObject.js +22 -0
  51. package/dataLayer/InsertObject.js +24 -0
  52. package/dataLayer/ReadAuditLogObject.js +24 -0
  53. package/dataLayer/SQLSearch.js +1335 -0
  54. package/dataLayer/SearchByConditionsObject.js +61 -0
  55. package/dataLayer/SearchByHashObject.js +21 -0
  56. package/dataLayer/SearchObject.js +45 -0
  57. package/dataLayer/SqlSearchObject.js +14 -0
  58. package/dataLayer/UpdateObject.js +23 -0
  59. package/dataLayer/UpsertObject.js +23 -0
  60. package/dataLayer/bulkLoad.js +813 -0
  61. package/dataLayer/dataObjects/BulkLoadObjects.js +27 -0
  62. package/dataLayer/dataObjects/UpsertObject.js +23 -0
  63. package/dataLayer/delete.js +164 -0
  64. package/dataLayer/export.js +381 -0
  65. package/dataLayer/getBackup.js +40 -0
  66. package/dataLayer/harperBridge/BridgeMethods.js +81 -0
  67. package/dataLayer/harperBridge/ResourceBridge.ts +633 -0
  68. package/dataLayer/harperBridge/bridgeUtility/insertUpdateReturnObj.js +28 -0
  69. package/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +88 -0
  70. package/dataLayer/harperBridge/harperBridge.js +21 -0
  71. package/dataLayer/harperBridge/lmdbBridge/LMDBBridge.js +119 -0
  72. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +19 -0
  73. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +112 -0
  74. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +67 -0
  75. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +31 -0
  76. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +94 -0
  77. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +98 -0
  78. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +89 -0
  79. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +109 -0
  80. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +107 -0
  81. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +137 -0
  82. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +35 -0
  83. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +111 -0
  84. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +28 -0
  85. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +29 -0
  86. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +207 -0
  87. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +156 -0
  88. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +21 -0
  89. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +30 -0
  90. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +19 -0
  91. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +64 -0
  92. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +70 -0
  93. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +22 -0
  94. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +23 -0
  95. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +22 -0
  96. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBTransactionObject.js +23 -0
  97. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +24 -0
  98. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +24 -0
  99. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +25 -0
  100. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +21 -0
  101. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +157 -0
  102. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +94 -0
  103. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +39 -0
  104. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +34 -0
  105. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +100 -0
  106. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +371 -0
  107. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +109 -0
  108. package/dataLayer/hdbInfoController.js +254 -0
  109. package/dataLayer/insert.js +266 -0
  110. package/dataLayer/readAuditLog.js +59 -0
  111. package/dataLayer/schema.js +366 -0
  112. package/dataLayer/schemaDescribe.js +289 -0
  113. package/dataLayer/search.js +60 -0
  114. package/dataLayer/transaction.js +17 -0
  115. package/dataLayer/update.js +124 -0
  116. package/dist/components/Logger.d.ts +12 -0
  117. package/dist/components/Logger.js +3 -0
  118. package/dist/components/Logger.js.map +1 -0
  119. package/dist/components/Scope.d.ts +14 -4
  120. package/dist/components/Scope.js +18 -10
  121. package/dist/components/Scope.js.map +1 -1
  122. package/dist/components/componentLoader.js +16 -9
  123. package/dist/components/componentLoader.js.map +1 -1
  124. package/dist/components/operations.js +2 -2
  125. package/dist/components/operations.js.map +1 -1
  126. package/dist/config/configUtils.d.ts +1 -1
  127. package/dist/config/configUtils.js +1 -1
  128. package/dist/config/configUtils.js.map +1 -1
  129. package/dist/dataLayer/CreateTableObject.d.ts +2 -2
  130. package/dist/dataLayer/CreateTableObject.js +2 -2
  131. package/dist/dataLayer/CreateTableObject.js.map +1 -1
  132. package/dist/dataLayer/delete.d.ts +1 -1
  133. package/dist/dataLayer/schema.js +6 -5
  134. package/dist/dataLayer/schema.js.map +1 -1
  135. package/dist/dataLayer/schemaDescribe.js +1 -1
  136. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  137. package/dist/index.d.ts +1 -1
  138. package/dist/index.js +2 -0
  139. package/dist/index.js.map +1 -1
  140. package/dist/resources/DatabaseTransaction.d.ts +1 -1
  141. package/dist/resources/IterableEventQueue.d.ts +1 -1
  142. package/dist/resources/LMDBTransaction.d.ts +5 -1
  143. package/dist/resources/Resource.d.ts +1 -1
  144. package/dist/resources/RocksIndexStore.d.ts +3 -3
  145. package/dist/resources/RocksTransactionLogStore.d.ts +6 -3
  146. package/dist/resources/Table.d.ts +15 -6
  147. package/dist/resources/Table.js +4 -1
  148. package/dist/resources/Table.js.map +1 -1
  149. package/dist/resources/analytics/read.js +32 -22
  150. package/dist/resources/analytics/read.js.map +1 -1
  151. package/dist/resources/analytics/write.js +3 -6
  152. package/dist/resources/analytics/write.js.map +1 -1
  153. package/dist/resources/auditStore.d.ts +3 -3
  154. package/dist/resources/blob.d.ts +25 -2
  155. package/dist/resources/databases.d.ts +12 -2
  156. package/dist/resources/databases.js +22 -19
  157. package/dist/resources/databases.js.map +1 -1
  158. package/dist/resources/search.js +11 -5
  159. package/dist/resources/search.js.map +1 -1
  160. package/dist/resources/transaction.d.ts +2 -1
  161. package/dist/security/auth.js +1 -1
  162. package/dist/security/auth.js.map +1 -1
  163. package/dist/security/cryptoHash.d.ts +2 -2
  164. package/dist/security/jsLoader.js +243 -66
  165. package/dist/security/jsLoader.js.map +1 -1
  166. package/dist/security/keys.js +4 -5
  167. package/dist/security/keys.js.map +1 -1
  168. package/dist/security/user.js +3 -3
  169. package/dist/security/user.js.map +1 -1
  170. package/dist/server/REST.js +16 -2
  171. package/dist/server/REST.js.map +1 -1
  172. package/dist/server/Server.d.ts +2 -1
  173. package/dist/server/Server.js.map +1 -1
  174. package/dist/server/fastifyRoutes/plugins/hdbCore.d.ts +6 -1
  175. package/dist/server/fastifyRoutes.js +2 -0
  176. package/dist/server/fastifyRoutes.js.map +1 -1
  177. package/dist/server/http.js +12 -6
  178. package/dist/server/http.js.map +1 -1
  179. package/dist/server/jobs/JobObject.d.ts +3 -3
  180. package/dist/server/loadRootComponents.js +1 -0
  181. package/dist/server/loadRootComponents.js.map +1 -1
  182. package/dist/server/operationsServer.js +3 -1
  183. package/dist/server/operationsServer.js.map +1 -1
  184. package/dist/server/serverHelpers/JSONStream.d.ts +3 -3
  185. package/dist/server/serverHelpers/Request.d.ts +5 -5
  186. package/dist/server/serverHelpers/requestTimePlugin.d.ts +1 -1
  187. package/dist/server/threads/manageThreads.d.ts +2 -2
  188. package/dist/server/threads/manageThreads.js +50 -35
  189. package/dist/server/threads/manageThreads.js.map +1 -1
  190. package/dist/server/threads/socketRouter.d.ts +1 -1
  191. package/dist/sqlTranslator/deleteTranslator.d.ts +1 -1
  192. package/dist/utility/AWS/AWSConnector.d.ts +3 -2
  193. package/dist/utility/common_utils.d.ts +3 -3
  194. package/dist/utility/environment/systemInformation.d.ts +1 -0
  195. package/dist/utility/functions/date/dateFunctions.d.ts +11 -11
  196. package/dist/utility/globalSchema.d.ts +1 -1
  197. package/dist/utility/hdbTerms.d.ts +3 -0
  198. package/dist/utility/hdbTerms.js +3 -0
  199. package/dist/utility/hdbTerms.js.map +1 -1
  200. package/dist/utility/installation.d.ts +2 -4
  201. package/dist/utility/installation.js.map +1 -1
  202. package/dist/utility/lmdb/commonUtility.d.ts +1 -0
  203. package/dist/utility/lmdb/deleteUtility.d.ts +1 -0
  204. package/dist/utility/lmdb/environmentUtility.d.ts +1 -0
  205. package/dist/utility/lmdb/searchUtility.d.ts +2 -1
  206. package/dist/utility/lmdb/writeUtility.d.ts +1 -0
  207. package/dist/utility/logging/harper_logger.d.ts +6 -6
  208. package/dist/utility/processManagement/processManagement.d.ts +1 -1
  209. package/dist/utility/processManagement/servicesConfig.d.ts +12 -6
  210. package/dist/validation/common_validators.d.ts +4 -3
  211. package/dist/validation/configValidator.d.ts +3 -2
  212. package/index.d.ts +56 -0
  213. package/index.js +41 -0
  214. package/json/systemSchema.json +373 -0
  215. package/launchServiceScripts/launchHarperDB.js +3 -0
  216. package/launchServiceScripts/utility/checkNodeVersion.js +15 -0
  217. package/package.json +21 -3
  218. package/resources/DatabaseTransaction.ts +378 -0
  219. package/resources/ErrorResource.ts +57 -0
  220. package/resources/IterableEventQueue.ts +94 -0
  221. package/resources/LMDBTransaction.ts +349 -0
  222. package/resources/RecordEncoder.ts +702 -0
  223. package/resources/RequestTarget.ts +134 -0
  224. package/resources/Resource.ts +789 -0
  225. package/resources/ResourceInterface.ts +221 -0
  226. package/resources/ResourceInterfaceV2.ts +53 -0
  227. package/resources/ResourceV2.ts +67 -0
  228. package/resources/Resources.ts +162 -0
  229. package/resources/RocksIndexStore.ts +70 -0
  230. package/resources/RocksTransactionLogStore.ts +352 -0
  231. package/resources/Table.ts +4527 -0
  232. package/resources/analytics/hostnames.ts +72 -0
  233. package/resources/analytics/metadata.ts +10 -0
  234. package/resources/analytics/read.ts +252 -0
  235. package/resources/analytics/write.ts +803 -0
  236. package/resources/auditStore.ts +556 -0
  237. package/resources/blob.ts +1268 -0
  238. package/resources/crdt.ts +125 -0
  239. package/resources/dataLoader.ts +527 -0
  240. package/resources/databases.ts +1290 -0
  241. package/resources/graphql.ts +221 -0
  242. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +638 -0
  243. package/resources/indexes/customIndexes.ts +7 -0
  244. package/resources/indexes/vector.ts +38 -0
  245. package/resources/jsResource.ts +86 -0
  246. package/resources/loadEnv.ts +22 -0
  247. package/resources/login.ts +18 -0
  248. package/resources/openApi.ts +409 -0
  249. package/resources/registrationDeprecated.ts +8 -0
  250. package/resources/replayLogs.ts +136 -0
  251. package/resources/roles.ts +98 -0
  252. package/resources/search.ts +1301 -0
  253. package/resources/tracked.ts +584 -0
  254. package/resources/transaction.ts +89 -0
  255. package/resources/transactionBroadcast.ts +258 -0
  256. package/security/auth.ts +376 -0
  257. package/security/certificateVerification/certificateVerificationSource.ts +84 -0
  258. package/security/certificateVerification/configValidation.ts +107 -0
  259. package/security/certificateVerification/crlVerification.ts +623 -0
  260. package/security/certificateVerification/index.ts +121 -0
  261. package/security/certificateVerification/ocspVerification.ts +148 -0
  262. package/security/certificateVerification/pkijs-ed25519-patch.ts +188 -0
  263. package/security/certificateVerification/types.ts +128 -0
  264. package/security/certificateVerification/verificationConfig.ts +138 -0
  265. package/security/certificateVerification/verificationUtils.ts +447 -0
  266. package/security/cryptoHash.js +42 -0
  267. package/security/data_objects/PermissionAttributeResponseObject.js +15 -0
  268. package/security/data_objects/PermissionResponseObject.js +115 -0
  269. package/security/data_objects/PermissionTableResponseObject.js +20 -0
  270. package/security/fastifyAuth.js +169 -0
  271. package/security/impersonation.ts +160 -0
  272. package/security/jsLoader.ts +716 -0
  273. package/security/keys.js +948 -0
  274. package/security/permissionsTranslator.js +300 -0
  275. package/security/role.js +218 -0
  276. package/security/tokenAuthentication.ts +228 -0
  277. package/security/user.ts +449 -0
  278. package/server/DurableSubscriptionsSession.ts +503 -0
  279. package/server/REST.ts +407 -0
  280. package/server/Server.ts +89 -0
  281. package/server/fastifyRoutes/helpers/getCORSOptions.js +36 -0
  282. package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +15 -0
  283. package/server/fastifyRoutes/helpers/getServerOptions.js +33 -0
  284. package/server/fastifyRoutes/plugins/hdbCore.js +39 -0
  285. package/server/fastifyRoutes.ts +205 -0
  286. package/server/graphqlQuerying.ts +700 -0
  287. package/server/http.ts +640 -0
  288. package/server/itc/serverHandlers.js +161 -0
  289. package/server/itc/utility/ITCEventObject.js +10 -0
  290. package/server/jobs/JobObject.js +24 -0
  291. package/server/jobs/jobProcess.js +69 -0
  292. package/server/jobs/jobRunner.js +162 -0
  293. package/server/jobs/jobs.js +304 -0
  294. package/server/loadRootComponents.js +44 -0
  295. package/server/mqtt.ts +485 -0
  296. package/server/nodeName.ts +75 -0
  297. package/server/operationsServer.ts +313 -0
  298. package/server/serverHelpers/Headers.ts +108 -0
  299. package/server/serverHelpers/JSONStream.ts +269 -0
  300. package/server/serverHelpers/OperationFunctionObject.ts +13 -0
  301. package/server/serverHelpers/Request.ts +158 -0
  302. package/server/serverHelpers/contentTypes.ts +637 -0
  303. package/server/serverHelpers/requestTimePlugin.js +57 -0
  304. package/server/serverHelpers/serverHandlers.js +148 -0
  305. package/server/serverHelpers/serverUtilities.ts +473 -0
  306. package/server/serverRegistry.ts +8 -0
  307. package/server/static.ts +187 -0
  308. package/server/status/definitions.ts +37 -0
  309. package/server/status/index.ts +125 -0
  310. package/server/storageReclamation.ts +93 -0
  311. package/server/threads/itc.js +89 -0
  312. package/server/threads/manageThreads.js +594 -0
  313. package/server/threads/socketRouter.ts +360 -0
  314. package/server/threads/threadServer.js +279 -0
  315. package/server/throttle.ts +73 -0
  316. package/sqlTranslator/SelectValidator.js +330 -0
  317. package/sqlTranslator/alasqlFunctionImporter.js +62 -0
  318. package/sqlTranslator/deleteTranslator.js +67 -0
  319. package/sqlTranslator/index.js +242 -0
  320. package/sqlTranslator/sql_statement_bucket.js +472 -0
  321. package/static/defaultConfig.yaml +3 -0
  322. package/studio/web/HDBDogOnly.svg +78 -0
  323. package/studio/web/assets/PPRadioGrotesk-Bold-DDaUYG8E.woff +0 -0
  324. package/studio/web/assets/fa-brands-400-CEJbCg16.woff +0 -0
  325. package/studio/web/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
  326. package/studio/web/assets/fa-brands-400-DnkPfk3o.eot +0 -0
  327. package/studio/web/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
  328. package/studio/web/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
  329. package/studio/web/assets/fa-regular-400-BhTwtT8w.eot +0 -0
  330. package/studio/web/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
  331. package/studio/web/assets/fa-regular-400-DFnMcJPd.woff +0 -0
  332. package/studio/web/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
  333. package/studio/web/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
  334. package/studio/web/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
  335. package/studio/web/assets/fa-solid-900-B6Axprfb.eot +0 -0
  336. package/studio/web/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
  337. package/studio/web/assets/fa-solid-900-DOXgCApm.woff +0 -0
  338. package/studio/web/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
  339. package/studio/web/assets/index-BTgXJX9d.js +235 -0
  340. package/studio/web/assets/index-BTgXJX9d.js.map +1 -0
  341. package/studio/web/assets/index-C-GXfcup.js +37 -0
  342. package/studio/web/assets/index-C-GXfcup.js.map +1 -0
  343. package/studio/web/assets/index-PFlNdimM.js +2 -0
  344. package/studio/web/assets/index-PFlNdimM.js.map +1 -0
  345. package/studio/web/assets/index-Y2g_iFpU.css +1 -0
  346. package/studio/web/assets/index-jiPwkrsB.css +1 -0
  347. package/studio/web/assets/index.lazy-C3TJZJ4o.js +266 -0
  348. package/studio/web/assets/index.lazy-C3TJZJ4o.js.map +1 -0
  349. package/studio/web/assets/profiler-DotzgiCJ.js +2 -0
  350. package/studio/web/assets/profiler-DotzgiCJ.js.map +1 -0
  351. package/studio/web/assets/react-redux-VxUEx_mU.js +6 -0
  352. package/studio/web/assets/react-redux-VxUEx_mU.js.map +1 -0
  353. package/studio/web/assets/startRecording-B_9J9Csd.js +3 -0
  354. package/studio/web/assets/startRecording-B_9J9Csd.js.map +1 -0
  355. package/studio/web/fabric-signup-background.webp +0 -0
  356. package/studio/web/fabric-signup-text.png +0 -0
  357. package/studio/web/favicon_purple.png +0 -0
  358. package/studio/web/github-icon.svg +15 -0
  359. package/studio/web/harper-fabric_black.png +0 -0
  360. package/studio/web/harper-fabric_white.png +0 -0
  361. package/studio/web/harper-studio_white.png +0 -0
  362. package/studio/web/index.html +16 -0
  363. package/studio/web/running.css +148 -0
  364. package/studio/web/running.html +147 -0
  365. package/studio/web/running.js +111 -0
  366. package/upgrade/UpgradeObjects.js +13 -0
  367. package/upgrade/directives/directivesController.js +90 -0
  368. package/upgrade/directivesManager.js +139 -0
  369. package/upgrade/upgradePrompt.js +124 -0
  370. package/upgrade/upgradeUtilities.js +28 -0
  371. package/utility/AWS/AWSConnector.js +29 -0
  372. package/utility/OperationFunctionCaller.js +63 -0
  373. package/utility/assignCmdEnvVariables.js +62 -0
  374. package/utility/common_utils.js +867 -0
  375. package/utility/environment/environmentManager.js +208 -0
  376. package/utility/environment/systemInformation.js +355 -0
  377. package/utility/errors/commonErrors.js +267 -0
  378. package/utility/errors/hdbError.js +146 -0
  379. package/utility/functions/date/dateFunctions.js +65 -0
  380. package/utility/functions/geo.js +355 -0
  381. package/utility/functions/sql/alaSQLExtension.js +104 -0
  382. package/utility/globalSchema.js +35 -0
  383. package/utility/hdbTerms.ts +819 -0
  384. package/utility/install/checkJWTTokensExist.js +62 -0
  385. package/utility/install/harperdb.conf +15 -0
  386. package/utility/install/harperdb.service +14 -0
  387. package/utility/install/installer.js +635 -0
  388. package/utility/installation.ts +30 -0
  389. package/utility/lmdb/DBIDefinition.js +20 -0
  390. package/utility/lmdb/DeleteRecordsResponseObject.js +25 -0
  391. package/utility/lmdb/InsertRecordsResponseObject.js +22 -0
  392. package/utility/lmdb/OpenDBIObject.js +31 -0
  393. package/utility/lmdb/OpenEnvironmentObject.js +41 -0
  394. package/utility/lmdb/UpdateRecordsResponseObject.js +25 -0
  395. package/utility/lmdb/UpsertRecordsResponseObject.js +22 -0
  396. package/utility/lmdb/cleanLMDBMap.js +65 -0
  397. package/utility/lmdb/commonUtility.js +119 -0
  398. package/utility/lmdb/deleteUtility.js +128 -0
  399. package/utility/lmdb/environmentUtility.js +477 -0
  400. package/utility/lmdb/searchCursorFunctions.js +187 -0
  401. package/utility/lmdb/searchUtility.js +918 -0
  402. package/utility/lmdb/terms.js +57 -0
  403. package/utility/lmdb/writeUtility.js +407 -0
  404. package/utility/logging/harper_logger.js +876 -0
  405. package/utility/logging/logRotator.js +157 -0
  406. package/utility/logging/logger.ts +24 -0
  407. package/utility/logging/readLog.js +355 -0
  408. package/utility/logging/transactionLog.js +57 -0
  409. package/utility/mount_hdb.js +59 -0
  410. package/utility/npmUtilities.js +102 -0
  411. package/utility/operationPermissions.ts +112 -0
  412. package/utility/operation_authorization.js +836 -0
  413. package/utility/packageUtils.js +55 -0
  414. package/utility/password.ts +99 -0
  415. package/utility/processManagement/processManagement.js +187 -0
  416. package/utility/processManagement/servicesConfig.js +56 -0
  417. package/utility/scripts/restartHdb.js +24 -0
  418. package/utility/scripts/user_data.sh +13 -0
  419. package/utility/signalling.js +36 -0
  420. package/utility/terms/certificates.js +81 -0
  421. package/utility/when.ts +20 -0
  422. package/v1.d.ts +39 -0
  423. package/v1.js +41 -0
  424. package/v2.d.ts +39 -0
  425. package/v2.js +41 -0
  426. package/validation/bulkDeleteValidator.js +24 -0
  427. package/validation/check_permissions.js +19 -0
  428. package/validation/common_validators.js +95 -0
  429. package/validation/configValidator.js +331 -0
  430. package/validation/deleteValidator.js +15 -0
  431. package/validation/fileLoadValidator.js +153 -0
  432. package/validation/insertValidator.js +40 -0
  433. package/validation/installValidator.js +37 -0
  434. package/validation/readLogValidator.js +64 -0
  435. package/validation/role_validation.js +320 -0
  436. package/validation/schemaMetadataValidator.js +42 -0
  437. package/validation/searchValidator.js +166 -0
  438. package/validation/statusValidator.ts +66 -0
  439. package/validation/transactionLogValidator.js +33 -0
  440. package/validation/user_validation.js +55 -0
  441. package/validation/validationWrapper.js +105 -0
  442. package/dist/resources/analytics/profile.d.ts +0 -2
  443. package/dist/resources/analytics/profile.js +0 -144
  444. package/dist/resources/analytics/profile.js.map +0 -1
@@ -0,0 +1,166 @@
1
+ const _ = require('lodash'),
2
+ validator = require('./validationWrapper.js');
3
+ const Joi = require('joi');
4
+ const hdbUtils = require('../utility/common_utils.js');
5
+ const { hdbSchemaTable, checkValidTable, hdbTable, hdbDatabase } = require('./common_validators.js');
6
+ const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.js');
7
+ const { getDatabases } = require('../resources/databases.ts');
8
+ const { HTTP_STATUS_CODES } = hdbErrors;
9
+
10
+ const searchByValueSchema = Joi.object({
11
+ database: hdbDatabase,
12
+ schema: hdbDatabase,
13
+ table: hdbTable,
14
+ attribute: hdbSchemaTable,
15
+ value: Joi.any().required(),
16
+ get_attributes: Joi.array().min(1).items(Joi.alternatives(hdbSchemaTable, Joi.object())).optional(),
17
+ desc: Joi.bool(),
18
+ limit: Joi.number().integer().min(1),
19
+ offset: Joi.number().integer().min(0),
20
+ });
21
+
22
+ const searchByConditionsSchema = Joi.object({
23
+ database: hdbDatabase,
24
+ schema: hdbDatabase,
25
+ table: hdbTable,
26
+ operator: Joi.string().valid('and', 'or').default('and').lowercase(),
27
+ offset: Joi.number().integer().min(0),
28
+ limit: Joi.number().integer().min(1),
29
+ get_attributes: Joi.array().min(1).items(Joi.alternatives(hdbSchemaTable, Joi.object())).optional(),
30
+ sort: Joi.object({
31
+ attribute: Joi.alternatives(hdbSchemaTable, Joi.array().min(1)),
32
+ descending: Joi.bool().optional(),
33
+ }).optional(),
34
+ conditions: Joi.array()
35
+ .min(1)
36
+ .items(
37
+ Joi.alternatives(
38
+ Joi.object({ operator: Joi.string().valid('and', 'or').default('and').lowercase(), conditions: Joi.array() }),
39
+ Joi.object({
40
+ attribute: Joi.alternatives(hdbSchemaTable, Joi.array().min(1)),
41
+ comparator: Joi.string()
42
+ .valid(
43
+ 'equals',
44
+ 'contains',
45
+ 'starts_with',
46
+ 'ends_with',
47
+ 'greater_than',
48
+ 'greater_than_equal',
49
+ 'less_than',
50
+ 'less_than_equal',
51
+ 'between',
52
+ 'not_equal'
53
+ )
54
+ .optional(),
55
+ value: Joi.when('comparator', {
56
+ switch: [
57
+ { is: 'equals', then: Joi.any() },
58
+ {
59
+ is: 'between',
60
+ then: Joi.array()
61
+ .items(Joi.alternatives([Joi.string(), Joi.number()]))
62
+ .length(2),
63
+ },
64
+ ],
65
+ otherwise: Joi.alternatives(Joi.string(), Joi.number()),
66
+ }).required(),
67
+ })
68
+ )
69
+ )
70
+ .required(),
71
+ });
72
+
73
+ module.exports = function (searchObject, type) {
74
+ let validationError = null;
75
+ switch (type) {
76
+ case 'value':
77
+ validationError = validator.validateBySchema(searchObject, searchByValueSchema);
78
+ break;
79
+ case 'hashes':
80
+ let errors;
81
+ addError(checkValidTable('database', searchObject.schema));
82
+ addError(checkValidTable('table', searchObject.table));
83
+ if (!searchObject.hash_values) addError(`'hash_values' is required`);
84
+ else if (!Array.isArray(searchObject.hash_values)) addError(`'hash_values' must be an array`);
85
+ else if (!searchObject.hash_values.every((value) => typeof value === 'string' || typeof value === 'number'))
86
+ addError(`'hash_values' must be strings or numbers`);
87
+ if (!searchObject.get_attributes) addError(`'get_attributes' is required`);
88
+ else if (!Array.isArray(searchObject.get_attributes)) addError(`'get_attributes' must be an array`);
89
+ else if (searchObject.get_attributes.length === 0) addError(`'get_attributes' must contain at least 1 item`);
90
+ else if (!searchObject.get_attributes.every((value) => typeof value === 'string' || typeof value === 'number'))
91
+ addError(`'get_attributes' must be strings or numbers`);
92
+ function addError(error) {
93
+ if (errors) errors += '. ' + error;
94
+ else errors = error;
95
+ }
96
+ if (errors) validationError = new Error(errors.trim());
97
+ break;
98
+ case 'conditions':
99
+ validationError = validator.validateBySchema(searchObject, searchByConditionsSchema);
100
+ break;
101
+ default:
102
+ throw new Error(`Error validating search, unknown type: ${type}`);
103
+ }
104
+
105
+ // validate table and attribute if format validation is valid
106
+ if (!validationError && searchObject.schema !== 'system') {
107
+ // skip validation for system schema
108
+ //check if schema.table does not exist throw error
109
+ let checkSchemaTable = hdbUtils.checkGlobalSchemaTable(searchObject.schema, searchObject.table);
110
+ if (checkSchemaTable) {
111
+ return handleHDBError(new Error(), checkSchemaTable, HTTP_STATUS_CODES.NOT_FOUND);
112
+ }
113
+
114
+ let tableSchema = getDatabases()[searchObject.schema][searchObject.table];
115
+ let allTableAttributes = tableSchema.attributes;
116
+
117
+ //this clones the get_attributes array
118
+ let checkAttributes = searchObject.get_attributes ? [...searchObject.get_attributes] : [];
119
+
120
+ if (type === 'value') {
121
+ checkAttributes.push(searchObject.attribute);
122
+ }
123
+
124
+ //if search type is conditions add conditions fields to see if the fields exist
125
+ const addConditions = (searchObject) => {
126
+ //this is used to validate condition attributes exist in the schema
127
+ for (const condition of searchObject.conditions) {
128
+ if (condition.conditions) addConditions(condition);
129
+ else checkAttributes.push(condition.attribute);
130
+ }
131
+ };
132
+ if (type === 'conditions') {
133
+ addConditions(searchObject);
134
+ }
135
+
136
+ let unknownAttributes = _.filter(
137
+ checkAttributes,
138
+ (attribute) =>
139
+ attribute !== '*' &&
140
+ !attribute.startsWith?.('$') && // meta attributes
141
+ attribute.attribute !== '*' && // skip check for asterisk attribute
142
+ !Array.isArray(attribute) &&
143
+ !attribute.name && // nested attribute
144
+ !_.some(
145
+ allTableAttributes,
146
+ (
147
+ tableAttribute // attribute should match one of the attribute in global
148
+ ) =>
149
+ tableAttribute === attribute ||
150
+ tableAttribute.attribute === attribute ||
151
+ tableAttribute.attribute === attribute.attribute
152
+ )
153
+ );
154
+
155
+ // if any unknown attributes present in the search request then list all indicated as unknown attribute to error message at once split in well format
156
+ // for instance "unknown attribute a, b and c" or "unknown attribute a"
157
+ if (unknownAttributes && unknownAttributes.length > 0) {
158
+ // return error with proper message - replace last comma with and
159
+ let errorMsg = unknownAttributes.join(', ');
160
+ errorMsg = errorMsg.replace(/,([^,]*)$/, ' and$1');
161
+ return new Error(`unknown attribute '${errorMsg}'`);
162
+ }
163
+ }
164
+
165
+ return validationError;
166
+ };
@@ -0,0 +1,66 @@
1
+ import Joi from 'joi';
2
+ import * as validator from './validationWrapper.js';
3
+ import { STATUS_DEFINITIONS, STATUS_IDS, DEFAULT_STATUS_ID, type StatusId } from '../server/status/definitions.ts';
4
+
5
+ // Re-export constants for backward compatibility
6
+ export const STATUS_SCHEMAS = STATUS_DEFINITIONS;
7
+ export const STATUS_ALLOWED = STATUS_IDS;
8
+ export const STATUS_DEFAULT = DEFAULT_STATUS_ID;
9
+
10
+ /**
11
+ * Pregenerate error messages to avoid repeated string concatenation
12
+ */
13
+ const ERROR_MESSAGES = Object.entries(STATUS_DEFINITIONS).reduce(
14
+ (messages, [id, definition]) => {
15
+ if (definition.allowedValues) {
16
+ messages[id] = `Status "${id}" only accepts these values: ${definition.allowedValues.join(', ')}`;
17
+ }
18
+ return messages;
19
+ },
20
+ {} as Record<string, string>
21
+ );
22
+
23
+ /**
24
+ * Creates the status validation schema using the STATUS_DEFINITIONS
25
+ */
26
+ const createStatusValidationSchema = () => {
27
+ // Start with base schema
28
+ let statusSchema = Joi.string().min(1).max(512);
29
+
30
+ // Add conditional validations for each status type that has allowedValues
31
+ (Object.entries(STATUS_DEFINITIONS) as [StatusId, (typeof STATUS_DEFINITIONS)[StatusId]][]).forEach(
32
+ ([id, definition]) => {
33
+ if (definition.allowedValues) {
34
+ statusSchema = statusSchema.when('id', {
35
+ is: id,
36
+ then: Joi.string()
37
+ .valid(...definition.allowedValues)
38
+ .messages({
39
+ 'any.only': ERROR_MESSAGES[id],
40
+ }),
41
+ });
42
+ }
43
+ }
44
+ );
45
+
46
+ return statusSchema.required();
47
+ };
48
+
49
+ /**
50
+ * Joi schema for validating status operations
51
+ */
52
+ const setStatusSchema = Joi.object({
53
+ id: Joi.string()
54
+ .valid(...STATUS_ALLOWED)
55
+ .required(),
56
+ status: createStatusValidationSchema(),
57
+ });
58
+
59
+ /**
60
+ * Validates the status operation parameters
61
+ * @param obj The status operation parameters to validate
62
+ * @returns Error if validation fails, null otherwise
63
+ */
64
+ export function validateStatus(obj: any) {
65
+ return validator.validateBySchema(obj, setStatusSchema);
66
+ }
@@ -0,0 +1,33 @@
1
+ 'use strict';
2
+
3
+ const Joi = require('joi');
4
+ const validator = require('./validationWrapper.js');
5
+
6
+ module.exports = {
7
+ readTransactionLogValidator,
8
+ deleteTransactionLogsBeforeValidator,
9
+ };
10
+
11
+ function readTransactionLogValidator(req) {
12
+ const schema = Joi.object({
13
+ schema: Joi.string(),
14
+ database: Joi.string(),
15
+ table: Joi.string().required(),
16
+ from: Joi.date().timestamp(),
17
+ to: Joi.date().timestamp(),
18
+ limit: Joi.number().min(1),
19
+ });
20
+
21
+ return validator.validateBySchema(req, schema);
22
+ }
23
+
24
+ function deleteTransactionLogsBeforeValidator(req) {
25
+ const schema = Joi.object({
26
+ schema: Joi.string(),
27
+ database: Joi.string(),
28
+ table: Joi.string().required(),
29
+ timestamp: Joi.date().timestamp().required(),
30
+ });
31
+
32
+ return validator.validateBySchema(req, schema);
33
+ }
@@ -0,0 +1,55 @@
1
+ const validator = require('./validationWrapper.js');
2
+
3
+ const constraints = {
4
+ username: {
5
+ presence: true,
6
+ exclusion: {
7
+ within: ['system'],
8
+ message: 'You cannot create tables within the system schema',
9
+ },
10
+ },
11
+ password: {
12
+ presence: true,
13
+ },
14
+ role: {
15
+ presence: true,
16
+ format: '[\\w\\-\\_]+',
17
+ },
18
+ active: {
19
+ presence: true,
20
+ inclusion: {
21
+ within: [true, false],
22
+ message: 'must be a boolean',
23
+ },
24
+ },
25
+ };
26
+
27
+ function addUserValidation(object) {
28
+ constraints.password.presence = true;
29
+ constraints.username.presence = true;
30
+ constraints.role.presence = true;
31
+ constraints.active.presence = true;
32
+ return validator.validateObject(object, constraints);
33
+ }
34
+
35
+ function alterUserValidation(object) {
36
+ constraints.password.presence = false;
37
+ constraints.username.presence = true;
38
+ constraints.role.presence = false;
39
+ constraints.active.presence = false;
40
+ return validator.validateObject(object, constraints);
41
+ }
42
+
43
+ function dropUserValidation(object) {
44
+ constraints.password.presence = false;
45
+ constraints.username.presence = true;
46
+ constraints.role.presence = false;
47
+ constraints.active.presence = false;
48
+ return validator.validateObject(object, constraints);
49
+ }
50
+
51
+ module.exports = {
52
+ addUserValidation,
53
+ alterUserValidation,
54
+ dropUserValidation,
55
+ };
@@ -0,0 +1,105 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * validationWrapper.js
5
+ *
6
+ * This module is meant as a wrapper for calls to the validate module. This wrapper serves 2 purposes:
7
+ * it decouples the validate() call from the validate module, and it ensures a consistent "Error" return object
8
+ * so we can rely on it for logging and reporting.
9
+ *
10
+ * There are a few cases where the validate module is called directly for functions like isBoolean.
11
+ * These are rare enough for it not to be worth creating wrapper functions for those as well.
12
+ */
13
+
14
+ const validate = require('validate.js');
15
+
16
+ //This validator is added here b/c we are still on version 0.11.1 that does not include this build in functionality. When
17
+ // we do update, we can remove. The reason we have not is related to a breaking change on the "presence" validator rule
18
+ // that will require a lot of fixes on our end. More here - https://validatejs.org/#changelog-0-12-0
19
+ // TODO: We are on version 0.13.x+ now, so we should be able to get rid of this. - WSM 2026-01-14
20
+ validate.validators.type = function (value, options, _key, _attributes) {
21
+ // allow empty values by default (needs to be checked by "presence" check)
22
+ if (value === null || typeof value === 'undefined') {
23
+ return null;
24
+ }
25
+
26
+ return validate.validators.type.checks[options](value) ? null : ' must be a ' + `'${options}' value`;
27
+ };
28
+
29
+ validate.validators.type.checks = {
30
+ Object: function (value) {
31
+ return validate.isObject(value) && !validate.isArray(value);
32
+ },
33
+ Array: validate.isArray,
34
+ Integer: validate.isInteger,
35
+ Number: validate.isNumber,
36
+ String: validate.isString,
37
+ Date: validate.isDate,
38
+ Boolean: function (value) {
39
+ return typeof value === 'boolean';
40
+ },
41
+ };
42
+
43
+ validate.validators.hasValidFileExt = function (value, options) {
44
+ // allow non-string values by default (needs to be checked by "presence" and "type" checks)
45
+ if (!validate.isString(value)) {
46
+ return null;
47
+ }
48
+ if (value === '') return `can't be blank`;
49
+
50
+ return options.filter((ext) => value.endsWith(ext)).length > 0
51
+ ? null
52
+ : `must include one of the following valid file extensions - '${options.join("', '")}'`;
53
+ };
54
+
55
+ module.exports = {
56
+ validateObject,
57
+ validateObjectAsync,
58
+ validateBySchema,
59
+ };
60
+
61
+ function validateObject(object, fileConstraints) {
62
+ if (!object || !fileConstraints) {
63
+ return new Error('validateObject parameters were null');
64
+ }
65
+
66
+ let validateResult = validate(object, fileConstraints, { format: 'flat' });
67
+ if (!validateResult) return null;
68
+ return new Error(validateResult);
69
+ }
70
+
71
+ /**
72
+ * Use this function for calls that support async/await
73
+ * @param object - the json object being validated
74
+ * @param fileConstraints - validation rules for the json object
75
+ * @returns {Promise<Error|null>}
76
+ */
77
+ async function validateObjectAsync(object, fileConstraints) {
78
+ if (!object || !fileConstraints) {
79
+ return new Error('validateObject parameters were null');
80
+ }
81
+
82
+ try {
83
+ await validate.async(object, fileConstraints, { format: 'flat' });
84
+ } catch (err) {
85
+ // unroll the array and make a full error message.
86
+ let msg = err.join(`,`);
87
+ return new Error(msg);
88
+ }
89
+ // If no error, just return null so this will behave as the non async version.
90
+ return null;
91
+ }
92
+
93
+ /**
94
+ *
95
+ * @param {{}} object
96
+ * @param {Joi.ObjectSchema} schema
97
+ * @returns {*}
98
+ */
99
+ function validateBySchema(object, schema) {
100
+ let result = schema.validate(object, { allowUnknown: true, abortEarly: false, errors: { wrap: { label: "'" } } });
101
+
102
+ if (result.error) {
103
+ return new Error(result.error.message);
104
+ }
105
+ }
@@ -1,2 +0,0 @@
1
- export declare const userCodeFolders: any[];
2
- export declare function captureProfile(delayToNextCapture?: number): Promise<void>;
@@ -1,144 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.userCodeFolders = void 0;
37
- exports.captureProfile = captureProfile;
38
- /**
39
- * This module is responsible for profiling threads so we can determine how much CPU usage can be attributed
40
- * to user code, harper code, and individual "hot" functions
41
- */
42
- const write_ts_1 = require("./write.js");
43
- const environmentManager_js_1 = require("../../utility/environment/environmentManager.js");
44
- const hdbTerms_js_1 = require("../../utility/hdbTerms.js");
45
- const packageUtils_js_1 = require("../../utility/packageUtils.js");
46
- const node_fs_1 = require("node:fs");
47
- const pprof_1 = require("@datadog/pprof");
48
- const log = __importStar(require("../../utility/logging/harper_logger.js"));
49
- const basePath = (0, environmentManager_js_1.getHdbBasePath)();
50
- exports.userCodeFolders = basePath ? [basePath] : [];
51
- if (process.env.RUN_HDB_APP)
52
- exports.userCodeFolders.push((0, node_fs_1.realpathSync)(process.env.RUN_HDB_APP));
53
- let profilerTimer;
54
- const SAMPLING_INTERVAL_IN_MICROSECONDS = 50000;
55
- // TODO: Running this on the thread itself can be a problematic because the profiler snapshots are somewhat expensive
56
- // (calling timeProfiler.stop and getting the large block of JSON and parsing it). This can take a 5ms or more
57
- // which can have some impact on latency for users. However, the datadog profiler is much better than the node
58
- // profiler, so we'll keep this for now.
59
- (async () => {
60
- if (exports.userCodeFolders.length === 0)
61
- return;
62
- // start the profiler
63
- pprof_1.time.start({ intervalMicros: SAMPLING_INTERVAL_IN_MICROSECONDS });
64
- const PROFILE_PERIOD = ((0, environmentManager_js_1.get)(hdbTerms_js_1.CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) ?? 60) * 1000;
65
- if (PROFILE_PERIOD > 0) {
66
- profilerTimer = setTimeout(() => {
67
- captureProfile(PROFILE_PERIOD);
68
- }, PROFILE_PERIOD).unref();
69
- }
70
- })();
71
- async function captureProfile(delayToNextCapture = ((0, environmentManager_js_1.get)(hdbTerms_js_1.CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) ?? 60) * 1000) {
72
- clearTimeout(profilerTimer);
73
- const hitCountThreshold = 100;
74
- const secondsPerHit = SAMPLING_INTERVAL_IN_MICROSECONDS / 1_000_000;
75
- const locationById = new Map();
76
- const fileNameById = new Map();
77
- const samplesByLocationId = new Map();
78
- let totalUserCount = 0;
79
- let totalHarperCount = 0;
80
- try {
81
- const profile = pprof_1.time.stop(true);
82
- const strings = profile.stringTable.strings;
83
- for (let func of profile.function) {
84
- fileNameById.set(func.id, strings[func.filename]);
85
- }
86
- for (let location of profile.location) {
87
- locationById.set(location.id, location.line[0]);
88
- }
89
- for (const sample of profile.sample) {
90
- getUserHitCount(sample);
91
- }
92
- (0, write_ts_1.recordAction)(totalHarperCount * secondsPerHit, 'cpu-usage', 'harper');
93
- (0, write_ts_1.recordAction)(totalUserCount * secondsPerHit, 'cpu-usage', 'user');
94
- for (let [locationId, sampleCount] of samplesByLocationId) {
95
- if (sampleCount > hitCountThreshold) {
96
- const location = locationById.get(locationId);
97
- const locationName = fileNameById.get(location.functionId) + ':' + location.line;
98
- (0, write_ts_1.recordAction)(sampleCount * secondsPerHit, 'cpu-usage', locationName);
99
- }
100
- }
101
- }
102
- catch (error) {
103
- log.error?.('analytics profiler error:', error);
104
- }
105
- finally {
106
- // and start the profiler again
107
- if (delayToNextCapture > 0) {
108
- profilerTimer = setTimeout(() => {
109
- captureProfile();
110
- }, delayToNextCapture).unref();
111
- }
112
- else {
113
- // somehow this can later get set to a negative number which causes big problems (high-frequency restarts of the profiler)
114
- log.info?.('Profiling disabled');
115
- pprof_1.time.stop();
116
- }
117
- }
118
- // this traverses the nodes and returns the number of sampling hits for the sample and attributes it
119
- // to harper or user code (as opposed to execution of things like node internal modules or native code)
120
- function getUserHitCount(sample) {
121
- // if we can assign to user code or harper code, do so
122
- let recordedTopSample = false;
123
- for (let locationId of sample.locationId) {
124
- let fileName = fileNameById.get(locationById.get(locationId).functionId);
125
- if (exports.userCodeFolders.some((userCodeFolder) => fileName.startsWith(userCodeFolder))) {
126
- // the call frame location is in user code
127
- const sampleCount = sample.value[0];
128
- totalUserCount += sampleCount;
129
- if (!recordedTopSample)
130
- samplesByLocationId.set(locationId, (samplesByLocationId.get(locationId) ?? 0) + sampleCount);
131
- return; // if the highest point in the call stack is in user code, we don't need to check the rest of the call stack, this "counts" as user execution
132
- }
133
- if (fileName.startsWith(packageUtils_js_1.PACKAGE_ROOT)) {
134
- const sampleCount = sample.value[0];
135
- totalHarperCount += sampleCount;
136
- if (!recordedTopSample) {
137
- samplesByLocationId.set(locationId, (samplesByLocationId.get(locationId) ?? 0) + sampleCount);
138
- recordedTopSample = true;
139
- }
140
- }
141
- }
142
- }
143
- }
144
- //# sourceMappingURL=profile.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"profile.js","sourceRoot":"","sources":["../../../resources/analytics/profile.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCA,wCAwEC;AA5GD;;;GAGG;AACH,yCAA0C;AAC1C,2FAAgG;AAChG,2DAA0D;AAC1D,mEAA6D;AAC7D,qCAAuC;AACvC,0CAAsD;AACtD,4EAA8D;AAI9D,MAAM,QAAQ,GAAG,IAAA,sCAAc,GAAE,CAAC;AACrB,QAAA,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW;IAAE,uBAAe,CAAC,IAAI,CAAC,IAAA,sBAAY,EAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;AAEzF,IAAI,aAAyC,CAAC;AAC9C,MAAM,iCAAiC,GAAG,KAAK,CAAC;AAChD,qHAAqH;AACrH,+GAA+G;AAC/G,+GAA+G;AAC/G,yCAAyC;AACzC,CAAC,KAAK,IAAI,EAAE;IACX,IAAI,uBAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACzC,qBAAqB;IACrB,YAAY,CAAC,KAAK,CAAC,EAAE,cAAc,EAAE,iCAAiC,EAAE,CAAC,CAAC;IAC1E,MAAM,cAAc,GAAG,CAAC,IAAA,2BAAM,EAAC,2BAAa,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;IACtF,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACxB,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;YAC/B,cAAc,CAAC,cAAc,CAAC,CAAC;QAChC,CAAC,EAAE,cAAc,CAAC,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;AACF,CAAC,CAAC,EAAE,CAAC;AAEE,KAAK,UAAU,cAAc,CACnC,kBAAkB,GAAG,CAAC,IAAA,2BAAM,EAAC,2BAAa,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI;IAEnF,YAAY,CAAC,aAAa,CAAC,CAAC;IAC5B,MAAM,iBAAiB,GAAG,GAAG,CAAC;IAC9B,MAAM,aAAa,GAAG,iCAAiC,GAAG,SAAS,CAAC;IACpE,MAAM,YAAY,GAAG,IAAI,GAAG,EAAe,CAAC;IAC5C,MAAM,YAAY,GAAG,IAAI,GAAG,EAAe,CAAC;IAC5C,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtD,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;QAC5C,KAAK,IAAI,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACnC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAY,EAAE,OAAO,CAAC,IAAI,CAAC,QAAkB,CAAC,CAAC,CAAC;QACvE,CAAC;QACD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACrC,eAAe,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QACD,IAAA,uBAAY,EAAC,gBAAgB,GAAG,aAAa,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;QACtE,IAAA,uBAAY,EAAC,cAAc,GAAG,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QAClE,KAAK,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,IAAI,mBAAmB,EAAE,CAAC;YAC3D,IAAI,WAAW,GAAG,iBAAiB,EAAE,CAAC;gBACrC,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACjF,IAAA,uBAAY,EAAC,WAAW,GAAG,aAAa,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;YACtE,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,GAAG,CAAC,KAAK,EAAE,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;YAAS,CAAC;QACV,+BAA+B;QAC/B,IAAI,kBAAkB,GAAG,CAAC,EAAE,CAAC;YAC5B,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC/B,cAAc,EAAE,CAAC;YAClB,CAAC,EAAE,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;aAAM,CAAC;YACP,0HAA0H;YAC1H,GAAG,CAAC,IAAI,EAAE,CAAC,oBAAoB,CAAC,CAAC;YACjC,YAAY,CAAC,IAAI,EAAE,CAAC;QACrB,CAAC;IACF,CAAC;IACD,oGAAoG;IACpG,uGAAuG;IACvG,SAAS,eAAe,CAAC,MAAc;QACtC,sDAAsD;QACtD,IAAI,iBAAiB,GAAG,KAAK,CAAC;QAC9B,KAAK,IAAI,UAAU,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAC1C,IAAI,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC,CAAC;YACzE,IAAI,uBAAe,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;gBACnF,0CAA0C;gBAC1C,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpC,cAAc,IAAI,WAAW,CAAC;gBAC9B,IAAI,CAAC,iBAAiB;oBACrB,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;gBAC/F,OAAO,CAAC,6IAA6I;YACtJ,CAAC;YACD,IAAI,QAAQ,CAAC,UAAU,CAAC,8BAAY,CAAC,EAAE,CAAC;gBACvC,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpC,gBAAgB,IAAI,WAAW,CAAC;gBAChC,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBACxB,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;oBAC9F,iBAAiB,GAAG,IAAI,CAAC;gBAC1B,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC"}