@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (444) hide show
  1. package/bin/BinObjects.js +17 -0
  2. package/bin/cliOperations.js +157 -0
  3. package/bin/copyDb.ts +280 -0
  4. package/bin/harper.js +156 -0
  5. package/bin/install.js +15 -0
  6. package/bin/lite.js +5 -0
  7. package/bin/restart.js +201 -0
  8. package/bin/run.js +409 -0
  9. package/bin/status.js +65 -0
  10. package/bin/stop.js +22 -0
  11. package/bin/upgrade.js +134 -0
  12. package/components/Application.ts +646 -0
  13. package/components/ApplicationScope.ts +49 -0
  14. package/components/Component.ts +53 -0
  15. package/components/ComponentV1.ts +342 -0
  16. package/components/DEFAULT_CONFIG.ts +18 -0
  17. package/components/EntryHandler.ts +227 -0
  18. package/components/Logger.ts +14 -0
  19. package/components/OptionsWatcher.ts +354 -0
  20. package/components/PluginModule.ts +6 -0
  21. package/components/Scope.ts +329 -0
  22. package/components/componentLoader.ts +529 -0
  23. package/components/deriveCommonPatternBase.ts +31 -0
  24. package/components/deriveGlobOptions.ts +44 -0
  25. package/components/deriveURLPath.ts +57 -0
  26. package/components/operations.js +658 -0
  27. package/components/operationsValidation.js +246 -0
  28. package/components/packageComponent.ts +39 -0
  29. package/components/requestRestart.ts +26 -0
  30. package/components/resolveBaseURLPath.ts +38 -0
  31. package/components/status/ComponentStatus.ts +110 -0
  32. package/components/status/ComponentStatusRegistry.ts +251 -0
  33. package/components/status/api.ts +153 -0
  34. package/components/status/crossThread.ts +405 -0
  35. package/components/status/errors.ts +152 -0
  36. package/components/status/index.ts +44 -0
  37. package/components/status/internal.ts +65 -0
  38. package/components/status/registry.ts +12 -0
  39. package/components/status/types.ts +96 -0
  40. package/config/RootConfigWatcher.ts +59 -0
  41. package/config/configHelpers.ts +11 -0
  42. package/config/configUtils.js +967 -0
  43. package/config/harperConfigEnvVars.ts +641 -0
  44. package/dataLayer/CreateAttributeObject.js +25 -0
  45. package/dataLayer/CreateTableObject.js +11 -0
  46. package/dataLayer/DataLayerObjects.js +43 -0
  47. package/dataLayer/DeleteBeforeObject.js +22 -0
  48. package/dataLayer/DeleteObject.js +25 -0
  49. package/dataLayer/DropAttributeObject.js +11 -0
  50. package/dataLayer/GetBackupObject.js +22 -0
  51. package/dataLayer/InsertObject.js +24 -0
  52. package/dataLayer/ReadAuditLogObject.js +24 -0
  53. package/dataLayer/SQLSearch.js +1335 -0
  54. package/dataLayer/SearchByConditionsObject.js +61 -0
  55. package/dataLayer/SearchByHashObject.js +21 -0
  56. package/dataLayer/SearchObject.js +45 -0
  57. package/dataLayer/SqlSearchObject.js +14 -0
  58. package/dataLayer/UpdateObject.js +23 -0
  59. package/dataLayer/UpsertObject.js +23 -0
  60. package/dataLayer/bulkLoad.js +813 -0
  61. package/dataLayer/dataObjects/BulkLoadObjects.js +27 -0
  62. package/dataLayer/dataObjects/UpsertObject.js +23 -0
  63. package/dataLayer/delete.js +164 -0
  64. package/dataLayer/export.js +381 -0
  65. package/dataLayer/getBackup.js +40 -0
  66. package/dataLayer/harperBridge/BridgeMethods.js +81 -0
  67. package/dataLayer/harperBridge/ResourceBridge.ts +633 -0
  68. package/dataLayer/harperBridge/bridgeUtility/insertUpdateReturnObj.js +28 -0
  69. package/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +88 -0
  70. package/dataLayer/harperBridge/harperBridge.js +21 -0
  71. package/dataLayer/harperBridge/lmdbBridge/LMDBBridge.js +119 -0
  72. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +19 -0
  73. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +112 -0
  74. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +67 -0
  75. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +31 -0
  76. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +94 -0
  77. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +98 -0
  78. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +89 -0
  79. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +109 -0
  80. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +107 -0
  81. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +137 -0
  82. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +35 -0
  83. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +111 -0
  84. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +28 -0
  85. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +29 -0
  86. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +207 -0
  87. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +156 -0
  88. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +21 -0
  89. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +30 -0
  90. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +19 -0
  91. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +64 -0
  92. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +70 -0
  93. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +22 -0
  94. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +23 -0
  95. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +22 -0
  96. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBTransactionObject.js +23 -0
  97. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +24 -0
  98. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +24 -0
  99. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +25 -0
  100. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +21 -0
  101. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +157 -0
  102. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +94 -0
  103. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +39 -0
  104. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +34 -0
  105. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +100 -0
  106. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +371 -0
  107. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +109 -0
  108. package/dataLayer/hdbInfoController.js +254 -0
  109. package/dataLayer/insert.js +266 -0
  110. package/dataLayer/readAuditLog.js +59 -0
  111. package/dataLayer/schema.js +366 -0
  112. package/dataLayer/schemaDescribe.js +289 -0
  113. package/dataLayer/search.js +60 -0
  114. package/dataLayer/transaction.js +17 -0
  115. package/dataLayer/update.js +124 -0
  116. package/dist/components/Logger.d.ts +12 -0
  117. package/dist/components/Logger.js +3 -0
  118. package/dist/components/Logger.js.map +1 -0
  119. package/dist/components/Scope.d.ts +14 -4
  120. package/dist/components/Scope.js +18 -10
  121. package/dist/components/Scope.js.map +1 -1
  122. package/dist/components/componentLoader.js +16 -9
  123. package/dist/components/componentLoader.js.map +1 -1
  124. package/dist/components/operations.js +2 -2
  125. package/dist/components/operations.js.map +1 -1
  126. package/dist/config/configUtils.d.ts +1 -1
  127. package/dist/config/configUtils.js +1 -1
  128. package/dist/config/configUtils.js.map +1 -1
  129. package/dist/dataLayer/CreateTableObject.d.ts +2 -2
  130. package/dist/dataLayer/CreateTableObject.js +2 -2
  131. package/dist/dataLayer/CreateTableObject.js.map +1 -1
  132. package/dist/dataLayer/delete.d.ts +1 -1
  133. package/dist/dataLayer/schema.js +6 -5
  134. package/dist/dataLayer/schema.js.map +1 -1
  135. package/dist/dataLayer/schemaDescribe.js +1 -1
  136. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  137. package/dist/index.d.ts +1 -1
  138. package/dist/index.js +2 -0
  139. package/dist/index.js.map +1 -1
  140. package/dist/resources/DatabaseTransaction.d.ts +1 -1
  141. package/dist/resources/IterableEventQueue.d.ts +1 -1
  142. package/dist/resources/LMDBTransaction.d.ts +5 -1
  143. package/dist/resources/Resource.d.ts +1 -1
  144. package/dist/resources/RocksIndexStore.d.ts +3 -3
  145. package/dist/resources/RocksTransactionLogStore.d.ts +6 -3
  146. package/dist/resources/Table.d.ts +15 -6
  147. package/dist/resources/Table.js +4 -1
  148. package/dist/resources/Table.js.map +1 -1
  149. package/dist/resources/analytics/read.js +32 -22
  150. package/dist/resources/analytics/read.js.map +1 -1
  151. package/dist/resources/analytics/write.js +3 -6
  152. package/dist/resources/analytics/write.js.map +1 -1
  153. package/dist/resources/auditStore.d.ts +3 -3
  154. package/dist/resources/blob.d.ts +25 -2
  155. package/dist/resources/databases.d.ts +12 -2
  156. package/dist/resources/databases.js +22 -19
  157. package/dist/resources/databases.js.map +1 -1
  158. package/dist/resources/search.js +11 -5
  159. package/dist/resources/search.js.map +1 -1
  160. package/dist/resources/transaction.d.ts +2 -1
  161. package/dist/security/auth.js +1 -1
  162. package/dist/security/auth.js.map +1 -1
  163. package/dist/security/cryptoHash.d.ts +2 -2
  164. package/dist/security/jsLoader.js +243 -66
  165. package/dist/security/jsLoader.js.map +1 -1
  166. package/dist/security/keys.js +4 -5
  167. package/dist/security/keys.js.map +1 -1
  168. package/dist/security/user.js +3 -3
  169. package/dist/security/user.js.map +1 -1
  170. package/dist/server/REST.js +16 -2
  171. package/dist/server/REST.js.map +1 -1
  172. package/dist/server/Server.d.ts +2 -1
  173. package/dist/server/Server.js.map +1 -1
  174. package/dist/server/fastifyRoutes/plugins/hdbCore.d.ts +6 -1
  175. package/dist/server/fastifyRoutes.js +2 -0
  176. package/dist/server/fastifyRoutes.js.map +1 -1
  177. package/dist/server/http.js +12 -6
  178. package/dist/server/http.js.map +1 -1
  179. package/dist/server/jobs/JobObject.d.ts +3 -3
  180. package/dist/server/loadRootComponents.js +1 -0
  181. package/dist/server/loadRootComponents.js.map +1 -1
  182. package/dist/server/operationsServer.js +3 -1
  183. package/dist/server/operationsServer.js.map +1 -1
  184. package/dist/server/serverHelpers/JSONStream.d.ts +3 -3
  185. package/dist/server/serverHelpers/Request.d.ts +5 -5
  186. package/dist/server/serverHelpers/requestTimePlugin.d.ts +1 -1
  187. package/dist/server/threads/manageThreads.d.ts +2 -2
  188. package/dist/server/threads/manageThreads.js +50 -35
  189. package/dist/server/threads/manageThreads.js.map +1 -1
  190. package/dist/server/threads/socketRouter.d.ts +1 -1
  191. package/dist/sqlTranslator/deleteTranslator.d.ts +1 -1
  192. package/dist/utility/AWS/AWSConnector.d.ts +3 -2
  193. package/dist/utility/common_utils.d.ts +3 -3
  194. package/dist/utility/environment/systemInformation.d.ts +1 -0
  195. package/dist/utility/functions/date/dateFunctions.d.ts +11 -11
  196. package/dist/utility/globalSchema.d.ts +1 -1
  197. package/dist/utility/hdbTerms.d.ts +3 -0
  198. package/dist/utility/hdbTerms.js +3 -0
  199. package/dist/utility/hdbTerms.js.map +1 -1
  200. package/dist/utility/installation.d.ts +2 -4
  201. package/dist/utility/installation.js.map +1 -1
  202. package/dist/utility/lmdb/commonUtility.d.ts +1 -0
  203. package/dist/utility/lmdb/deleteUtility.d.ts +1 -0
  204. package/dist/utility/lmdb/environmentUtility.d.ts +1 -0
  205. package/dist/utility/lmdb/searchUtility.d.ts +2 -1
  206. package/dist/utility/lmdb/writeUtility.d.ts +1 -0
  207. package/dist/utility/logging/harper_logger.d.ts +6 -6
  208. package/dist/utility/processManagement/processManagement.d.ts +1 -1
  209. package/dist/utility/processManagement/servicesConfig.d.ts +12 -6
  210. package/dist/validation/common_validators.d.ts +4 -3
  211. package/dist/validation/configValidator.d.ts +3 -2
  212. package/index.d.ts +56 -0
  213. package/index.js +41 -0
  214. package/json/systemSchema.json +373 -0
  215. package/launchServiceScripts/launchHarperDB.js +3 -0
  216. package/launchServiceScripts/utility/checkNodeVersion.js +15 -0
  217. package/package.json +21 -3
  218. package/resources/DatabaseTransaction.ts +378 -0
  219. package/resources/ErrorResource.ts +57 -0
  220. package/resources/IterableEventQueue.ts +94 -0
  221. package/resources/LMDBTransaction.ts +349 -0
  222. package/resources/RecordEncoder.ts +702 -0
  223. package/resources/RequestTarget.ts +134 -0
  224. package/resources/Resource.ts +789 -0
  225. package/resources/ResourceInterface.ts +221 -0
  226. package/resources/ResourceInterfaceV2.ts +53 -0
  227. package/resources/ResourceV2.ts +67 -0
  228. package/resources/Resources.ts +162 -0
  229. package/resources/RocksIndexStore.ts +70 -0
  230. package/resources/RocksTransactionLogStore.ts +352 -0
  231. package/resources/Table.ts +4527 -0
  232. package/resources/analytics/hostnames.ts +72 -0
  233. package/resources/analytics/metadata.ts +10 -0
  234. package/resources/analytics/read.ts +252 -0
  235. package/resources/analytics/write.ts +803 -0
  236. package/resources/auditStore.ts +556 -0
  237. package/resources/blob.ts +1268 -0
  238. package/resources/crdt.ts +125 -0
  239. package/resources/dataLoader.ts +527 -0
  240. package/resources/databases.ts +1290 -0
  241. package/resources/graphql.ts +221 -0
  242. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +638 -0
  243. package/resources/indexes/customIndexes.ts +7 -0
  244. package/resources/indexes/vector.ts +38 -0
  245. package/resources/jsResource.ts +86 -0
  246. package/resources/loadEnv.ts +22 -0
  247. package/resources/login.ts +18 -0
  248. package/resources/openApi.ts +409 -0
  249. package/resources/registrationDeprecated.ts +8 -0
  250. package/resources/replayLogs.ts +136 -0
  251. package/resources/roles.ts +98 -0
  252. package/resources/search.ts +1301 -0
  253. package/resources/tracked.ts +584 -0
  254. package/resources/transaction.ts +89 -0
  255. package/resources/transactionBroadcast.ts +258 -0
  256. package/security/auth.ts +376 -0
  257. package/security/certificateVerification/certificateVerificationSource.ts +84 -0
  258. package/security/certificateVerification/configValidation.ts +107 -0
  259. package/security/certificateVerification/crlVerification.ts +623 -0
  260. package/security/certificateVerification/index.ts +121 -0
  261. package/security/certificateVerification/ocspVerification.ts +148 -0
  262. package/security/certificateVerification/pkijs-ed25519-patch.ts +188 -0
  263. package/security/certificateVerification/types.ts +128 -0
  264. package/security/certificateVerification/verificationConfig.ts +138 -0
  265. package/security/certificateVerification/verificationUtils.ts +447 -0
  266. package/security/cryptoHash.js +42 -0
  267. package/security/data_objects/PermissionAttributeResponseObject.js +15 -0
  268. package/security/data_objects/PermissionResponseObject.js +115 -0
  269. package/security/data_objects/PermissionTableResponseObject.js +20 -0
  270. package/security/fastifyAuth.js +169 -0
  271. package/security/impersonation.ts +160 -0
  272. package/security/jsLoader.ts +716 -0
  273. package/security/keys.js +948 -0
  274. package/security/permissionsTranslator.js +300 -0
  275. package/security/role.js +218 -0
  276. package/security/tokenAuthentication.ts +228 -0
  277. package/security/user.ts +449 -0
  278. package/server/DurableSubscriptionsSession.ts +503 -0
  279. package/server/REST.ts +407 -0
  280. package/server/Server.ts +89 -0
  281. package/server/fastifyRoutes/helpers/getCORSOptions.js +36 -0
  282. package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +15 -0
  283. package/server/fastifyRoutes/helpers/getServerOptions.js +33 -0
  284. package/server/fastifyRoutes/plugins/hdbCore.js +39 -0
  285. package/server/fastifyRoutes.ts +205 -0
  286. package/server/graphqlQuerying.ts +700 -0
  287. package/server/http.ts +640 -0
  288. package/server/itc/serverHandlers.js +161 -0
  289. package/server/itc/utility/ITCEventObject.js +10 -0
  290. package/server/jobs/JobObject.js +24 -0
  291. package/server/jobs/jobProcess.js +69 -0
  292. package/server/jobs/jobRunner.js +162 -0
  293. package/server/jobs/jobs.js +304 -0
  294. package/server/loadRootComponents.js +44 -0
  295. package/server/mqtt.ts +485 -0
  296. package/server/nodeName.ts +75 -0
  297. package/server/operationsServer.ts +313 -0
  298. package/server/serverHelpers/Headers.ts +108 -0
  299. package/server/serverHelpers/JSONStream.ts +269 -0
  300. package/server/serverHelpers/OperationFunctionObject.ts +13 -0
  301. package/server/serverHelpers/Request.ts +158 -0
  302. package/server/serverHelpers/contentTypes.ts +637 -0
  303. package/server/serverHelpers/requestTimePlugin.js +57 -0
  304. package/server/serverHelpers/serverHandlers.js +148 -0
  305. package/server/serverHelpers/serverUtilities.ts +473 -0
  306. package/server/serverRegistry.ts +8 -0
  307. package/server/static.ts +187 -0
  308. package/server/status/definitions.ts +37 -0
  309. package/server/status/index.ts +125 -0
  310. package/server/storageReclamation.ts +93 -0
  311. package/server/threads/itc.js +89 -0
  312. package/server/threads/manageThreads.js +594 -0
  313. package/server/threads/socketRouter.ts +360 -0
  314. package/server/threads/threadServer.js +279 -0
  315. package/server/throttle.ts +73 -0
  316. package/sqlTranslator/SelectValidator.js +330 -0
  317. package/sqlTranslator/alasqlFunctionImporter.js +62 -0
  318. package/sqlTranslator/deleteTranslator.js +67 -0
  319. package/sqlTranslator/index.js +242 -0
  320. package/sqlTranslator/sql_statement_bucket.js +472 -0
  321. package/static/defaultConfig.yaml +3 -0
  322. package/studio/web/HDBDogOnly.svg +78 -0
  323. package/studio/web/assets/PPRadioGrotesk-Bold-DDaUYG8E.woff +0 -0
  324. package/studio/web/assets/fa-brands-400-CEJbCg16.woff +0 -0
  325. package/studio/web/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
  326. package/studio/web/assets/fa-brands-400-DnkPfk3o.eot +0 -0
  327. package/studio/web/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
  328. package/studio/web/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
  329. package/studio/web/assets/fa-regular-400-BhTwtT8w.eot +0 -0
  330. package/studio/web/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
  331. package/studio/web/assets/fa-regular-400-DFnMcJPd.woff +0 -0
  332. package/studio/web/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
  333. package/studio/web/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
  334. package/studio/web/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
  335. package/studio/web/assets/fa-solid-900-B6Axprfb.eot +0 -0
  336. package/studio/web/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
  337. package/studio/web/assets/fa-solid-900-DOXgCApm.woff +0 -0
  338. package/studio/web/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
  339. package/studio/web/assets/index-BTgXJX9d.js +235 -0
  340. package/studio/web/assets/index-BTgXJX9d.js.map +1 -0
  341. package/studio/web/assets/index-C-GXfcup.js +37 -0
  342. package/studio/web/assets/index-C-GXfcup.js.map +1 -0
  343. package/studio/web/assets/index-PFlNdimM.js +2 -0
  344. package/studio/web/assets/index-PFlNdimM.js.map +1 -0
  345. package/studio/web/assets/index-Y2g_iFpU.css +1 -0
  346. package/studio/web/assets/index-jiPwkrsB.css +1 -0
  347. package/studio/web/assets/index.lazy-C3TJZJ4o.js +266 -0
  348. package/studio/web/assets/index.lazy-C3TJZJ4o.js.map +1 -0
  349. package/studio/web/assets/profiler-DotzgiCJ.js +2 -0
  350. package/studio/web/assets/profiler-DotzgiCJ.js.map +1 -0
  351. package/studio/web/assets/react-redux-VxUEx_mU.js +6 -0
  352. package/studio/web/assets/react-redux-VxUEx_mU.js.map +1 -0
  353. package/studio/web/assets/startRecording-B_9J9Csd.js +3 -0
  354. package/studio/web/assets/startRecording-B_9J9Csd.js.map +1 -0
  355. package/studio/web/fabric-signup-background.webp +0 -0
  356. package/studio/web/fabric-signup-text.png +0 -0
  357. package/studio/web/favicon_purple.png +0 -0
  358. package/studio/web/github-icon.svg +15 -0
  359. package/studio/web/harper-fabric_black.png +0 -0
  360. package/studio/web/harper-fabric_white.png +0 -0
  361. package/studio/web/harper-studio_white.png +0 -0
  362. package/studio/web/index.html +16 -0
  363. package/studio/web/running.css +148 -0
  364. package/studio/web/running.html +147 -0
  365. package/studio/web/running.js +111 -0
  366. package/upgrade/UpgradeObjects.js +13 -0
  367. package/upgrade/directives/directivesController.js +90 -0
  368. package/upgrade/directivesManager.js +139 -0
  369. package/upgrade/upgradePrompt.js +124 -0
  370. package/upgrade/upgradeUtilities.js +28 -0
  371. package/utility/AWS/AWSConnector.js +29 -0
  372. package/utility/OperationFunctionCaller.js +63 -0
  373. package/utility/assignCmdEnvVariables.js +62 -0
  374. package/utility/common_utils.js +867 -0
  375. package/utility/environment/environmentManager.js +208 -0
  376. package/utility/environment/systemInformation.js +355 -0
  377. package/utility/errors/commonErrors.js +267 -0
  378. package/utility/errors/hdbError.js +146 -0
  379. package/utility/functions/date/dateFunctions.js +65 -0
  380. package/utility/functions/geo.js +355 -0
  381. package/utility/functions/sql/alaSQLExtension.js +104 -0
  382. package/utility/globalSchema.js +35 -0
  383. package/utility/hdbTerms.ts +819 -0
  384. package/utility/install/checkJWTTokensExist.js +62 -0
  385. package/utility/install/harperdb.conf +15 -0
  386. package/utility/install/harperdb.service +14 -0
  387. package/utility/install/installer.js +635 -0
  388. package/utility/installation.ts +30 -0
  389. package/utility/lmdb/DBIDefinition.js +20 -0
  390. package/utility/lmdb/DeleteRecordsResponseObject.js +25 -0
  391. package/utility/lmdb/InsertRecordsResponseObject.js +22 -0
  392. package/utility/lmdb/OpenDBIObject.js +31 -0
  393. package/utility/lmdb/OpenEnvironmentObject.js +41 -0
  394. package/utility/lmdb/UpdateRecordsResponseObject.js +25 -0
  395. package/utility/lmdb/UpsertRecordsResponseObject.js +22 -0
  396. package/utility/lmdb/cleanLMDBMap.js +65 -0
  397. package/utility/lmdb/commonUtility.js +119 -0
  398. package/utility/lmdb/deleteUtility.js +128 -0
  399. package/utility/lmdb/environmentUtility.js +477 -0
  400. package/utility/lmdb/searchCursorFunctions.js +187 -0
  401. package/utility/lmdb/searchUtility.js +918 -0
  402. package/utility/lmdb/terms.js +57 -0
  403. package/utility/lmdb/writeUtility.js +407 -0
  404. package/utility/logging/harper_logger.js +876 -0
  405. package/utility/logging/logRotator.js +157 -0
  406. package/utility/logging/logger.ts +24 -0
  407. package/utility/logging/readLog.js +355 -0
  408. package/utility/logging/transactionLog.js +57 -0
  409. package/utility/mount_hdb.js +59 -0
  410. package/utility/npmUtilities.js +102 -0
  411. package/utility/operationPermissions.ts +112 -0
  412. package/utility/operation_authorization.js +836 -0
  413. package/utility/packageUtils.js +55 -0
  414. package/utility/password.ts +99 -0
  415. package/utility/processManagement/processManagement.js +187 -0
  416. package/utility/processManagement/servicesConfig.js +56 -0
  417. package/utility/scripts/restartHdb.js +24 -0
  418. package/utility/scripts/user_data.sh +13 -0
  419. package/utility/signalling.js +36 -0
  420. package/utility/terms/certificates.js +81 -0
  421. package/utility/when.ts +20 -0
  422. package/v1.d.ts +39 -0
  423. package/v1.js +41 -0
  424. package/v2.d.ts +39 -0
  425. package/v2.js +41 -0
  426. package/validation/bulkDeleteValidator.js +24 -0
  427. package/validation/check_permissions.js +19 -0
  428. package/validation/common_validators.js +95 -0
  429. package/validation/configValidator.js +331 -0
  430. package/validation/deleteValidator.js +15 -0
  431. package/validation/fileLoadValidator.js +153 -0
  432. package/validation/insertValidator.js +40 -0
  433. package/validation/installValidator.js +37 -0
  434. package/validation/readLogValidator.js +64 -0
  435. package/validation/role_validation.js +320 -0
  436. package/validation/schemaMetadataValidator.js +42 -0
  437. package/validation/searchValidator.js +166 -0
  438. package/validation/statusValidator.ts +66 -0
  439. package/validation/transactionLogValidator.js +33 -0
  440. package/validation/user_validation.js +55 -0
  441. package/validation/validationWrapper.js +105 -0
  442. package/dist/resources/analytics/profile.d.ts +0 -2
  443. package/dist/resources/analytics/profile.js +0 -144
  444. package/dist/resources/analytics/profile.js.map +0 -1
@@ -0,0 +1,637 @@
1
+ import { streamAsJSON, stringify, parse } from './JSONStream.ts';
2
+ import { pack, unpack, encodeIter } from 'msgpackr';
3
+ import { decode, Encoder, EncoderStream } from 'cbor-x';
4
+ import { createBrotliCompress, brotliCompress, constants } from 'zlib';
5
+ import { ClientError } from '../../utility/errors/hdbError.js';
6
+ import stream, { Readable } from 'stream';
7
+ import { server } from '../Server.ts';
8
+ import { _assignPackageExport } from '../../globals.js';
9
+ import envMgr from '../../utility/environment/environmentManager.js';
10
+ import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts';
11
+ import * as YAML from 'yaml';
12
+ import { logger } from '../../utility/logging/logger.ts';
13
+ import { Blob } from '../../resources/blob.ts';
14
+ import { Transform } from 'json2csv';
15
+ // TODO: Only load this if fastify is loaded
16
+ import fp from 'fastify-plugin';
17
+ const SERIALIZATION_BIGINT = envMgr.get(CONFIG_PARAMS.SERIALIZATION_BIGINT) !== false;
18
+ const JSONStringify = SERIALIZATION_BIGINT ? stringify : JSON.stringify;
19
+ const JSONParse = SERIALIZATION_BIGINT ? parse : JSON.parse;
20
+
21
+ const PUBLIC_ENCODE_OPTIONS = {
22
+ useRecords: false,
23
+ useToJSON: true,
24
+ };
25
+
26
+ type Deserialize = (data: Buffer) => { contentType?: string; data: unknown } | unknown;
27
+
28
+ const mediaTypes = new Map<
29
+ string,
30
+ {
31
+ serialize?: unknown;
32
+ deserialize?: Deserialize;
33
+ serializeStream?: unknown;
34
+ compressible?: boolean;
35
+ q?: number;
36
+ }
37
+ >();
38
+
39
+ export const contentTypes = mediaTypes;
40
+ server.contentTypes = contentTypes;
41
+ _assignPackageExport('contentTypes', contentTypes);
42
+ // TODO: Make these monomorphic for faster access. And use a Map
43
+ mediaTypes.set('application/json', {
44
+ serializeStream: streamAsJSON,
45
+ serialize: JSONStringify,
46
+ deserialize(data) {
47
+ return JSONParse(data);
48
+ },
49
+ q: 0.8,
50
+ });
51
+ const cborEncoder = new Encoder(PUBLIC_ENCODE_OPTIONS);
52
+ mediaTypes.set('application/cbor', {
53
+ serializeStream(data) {
54
+ if (data[Symbol.asyncIterator]) data[Symbol.iterator] = null; // choose async iteration if possible
55
+ return new EncoderStream(PUBLIC_ENCODE_OPTIONS).end(data);
56
+ },
57
+ serialize: cborEncoder.encode,
58
+ deserialize: cborEncoder.decode,
59
+ q: 1,
60
+ });
61
+ mediaTypes.set('application/x-msgpack', {
62
+ serializeStream(data: any) {
63
+ if ((data?.[Symbol.iterator] || data?.[Symbol.asyncIterator]) && !Array.isArray(data)) {
64
+ return Readable.from(encodeIter(data, PUBLIC_ENCODE_OPTIONS));
65
+ }
66
+ return pack(data);
67
+ },
68
+ serialize: pack,
69
+ deserialize: unpack,
70
+ q: 0.9,
71
+ });
72
+ mediaTypes.set('text/csv', {
73
+ serializeStream(data: any, response: Response) {
74
+ response.headers.set('Content-Disposition', 'attachment; filename="data.csv"');
75
+ return toCsvStream(data, data?.getColumns?.());
76
+ },
77
+ serialize(data: any, response: Response) {
78
+ response.headers.set('Content-Disposition', 'attachment; filename="data.csv"');
79
+ if (data && !data[Symbol.iterator]) data = [data.toJSON ? data.toJSON() : data];
80
+ return toCsvStream(data, data?.getColumns?.());
81
+ },
82
+ q: 0.1,
83
+ });
84
+ mediaTypes.set('text/plain', {
85
+ serialize(data: any) {
86
+ return data.toString();
87
+ },
88
+ serializeStream(data: any) {
89
+ return Readable.from(data.map ? data.map((d) => d.toString()) : data);
90
+ },
91
+ deserialize(data: Buffer) {
92
+ return data.toString();
93
+ },
94
+ q: 0.2,
95
+ });
96
+
97
+ mediaTypes.set('text/yaml', {
98
+ serialize(data) {
99
+ return YAML.stringify(data, { aliasDuplicateObjects: false });
100
+ },
101
+
102
+ q: 0.7,
103
+ });
104
+
105
+ mediaTypes.set('text/event-stream', {
106
+ // Server-Sent Events (SSE)
107
+ serializeStream: function (iterable) {
108
+ // create a readable stream that we use to stream out events from our subscription
109
+ return Readable.from(transformIterable(iterable, this.serialize));
110
+ },
111
+ serialize: function (message) {
112
+ if (message.acknowledge) message.acknowledge();
113
+ if (typeof message === 'object' && 'value' in message && message.timestamp) {
114
+ // native messages
115
+ message = {
116
+ data: message.value,
117
+ event: message.type,
118
+ id: message.timestamp,
119
+ };
120
+ }
121
+ if (message.data || message.event) {
122
+ let serialized = '';
123
+ if (message.event) serialized += 'event: ' + message.event + '\n';
124
+ if (message.data) {
125
+ let data = message.data;
126
+ if (typeof data === 'object') data = JSONStringify(data);
127
+ serialized += 'data: ' + data + '\n';
128
+ }
129
+ if (message.id) serialized += 'id: ' + message.id + '\n';
130
+ if (message.retry) serialized += 'retry: ' + message.retry + '\n';
131
+ return serialized + '\n';
132
+ } else {
133
+ if (typeof message === 'object') return `data: ${JSONStringify(message)}\n\n`;
134
+ return `data: ${message}\n\n`;
135
+ }
136
+ },
137
+ compressible: false,
138
+ q: 0.8,
139
+ });
140
+ // TODO: Support this as well:
141
+ //'multipart/form-data'
142
+ mediaTypes.set('application/x-www-form-urlencoded', {
143
+ deserialize(data) {
144
+ const stringData = Buffer.isBuffer(data) ? data.toString('utf8') : data;
145
+ const object: Record<string, string | string[]> = {};
146
+ for (const [key, value] of new URLSearchParams(stringData)) {
147
+ if (object.hasOwnProperty(key)) {
148
+ // in case there are multiple query params with the same name, convert them to an array
149
+ const last = object[key];
150
+ if (Array.isArray(last)) last.push(value);
151
+ else object.key = [last, value];
152
+ } else object[key] = value;
153
+ }
154
+ return object;
155
+ },
156
+ serialize(data) {
157
+ const usp = new URLSearchParams();
158
+ for (const key in data) {
159
+ usp.set(key, data);
160
+ }
161
+ return usp.toString();
162
+ },
163
+ });
164
+ const genericHandler = {
165
+ type: 'application/json',
166
+ serializeStream: streamAsJSON,
167
+ serialize: JSONStringify,
168
+ deserialize: tryJSONParse,
169
+ q: 0.5,
170
+ };
171
+ mediaTypes.set('*/*', genericHandler);
172
+ mediaTypes.set('', genericHandler);
173
+ // try to JSON parse, but since we don't know for sure, this will return the body
174
+ // otherwise
175
+ function tryJSONParse(input) {
176
+ try {
177
+ if (input?.[0] === 123) return JSONParse(input);
178
+ else return input;
179
+ } catch {
180
+ return input;
181
+ }
182
+ }
183
+ export function registerContentHandlers(app) {
184
+ app.register(registerFastifySerializers, {
185
+ serializers: [
186
+ {
187
+ regex: /^application\/json$/,
188
+ serializer: streamAsJSON,
189
+ },
190
+ {
191
+ regex: /^application\/cbor$/,
192
+ serializer: function (data) {
193
+ return new EncoderStream(PUBLIC_ENCODE_OPTIONS).end(data);
194
+ },
195
+ },
196
+ {
197
+ regex: /^application\/(x-)?msgpack$/,
198
+ serializer: function (data) {
199
+ if ((data?.[Symbol.iterator] || data?.[Symbol.asyncIterator]) && !Array.isArray(data)) {
200
+ return Readable.from(encodeIter(data, PUBLIC_ENCODE_OPTIONS));
201
+ }
202
+ return pack(data);
203
+ },
204
+ },
205
+ {
206
+ regex: /^text\/csv$/,
207
+ serializer: function (data) {
208
+ this.header('Content-Disposition', 'attachment; filename="data.csv"');
209
+ return toCsvStream(data);
210
+ },
211
+ },
212
+ ],
213
+ });
214
+ app.addContentTypeParser('application/x-msgpack', { parseAs: 'buffer' }, (req, body, done) => {
215
+ try {
216
+ done(null, unpack(body));
217
+ } catch (error) {
218
+ error.statusCode = 400;
219
+ done(error);
220
+ }
221
+ });
222
+
223
+ app.addContentTypeParser('application/cbor', { parseAs: 'buffer' }, (req, body, done) => {
224
+ try {
225
+ done(null, decode(body));
226
+ } catch (error) {
227
+ error.statusCode = 400;
228
+ done(error);
229
+ }
230
+ });
231
+ }
232
+
233
+ const registerFastifySerializers = fp(
234
+ function (fastify, opts, done) {
235
+ // eslint-disable-next-line require-await
236
+ fastify.addHook('preSerialization', async (request, reply) => {
237
+ const contentType = reply.raw.getHeader('content-type');
238
+ if (contentType) return;
239
+ const { serializer, type } = findBestSerializer(request.raw);
240
+ reply.type(type);
241
+ reply.serializer(function (data: any) {
242
+ let serialize: (data: any, context: any) => any;
243
+ if (
244
+ typeof data === 'object' &&
245
+ data &&
246
+ (data[Symbol.iterator] || data[Symbol.asyncIterator]) &&
247
+ serializer.serializeStream
248
+ ) {
249
+ if (data.mapError) {
250
+ // indicate that we want iterator errors to be returned so we can serialize them in a meaningful way, if possible
251
+ const getColumns = data.getColumns;
252
+ data = data.mapError((error) => {
253
+ // make errors serializable in a descriptive way
254
+ error.toJSON = () => ({ error: error.name, message: error.message, ...error.partialObject });
255
+ return error;
256
+ });
257
+ data.getColumns = getColumns;
258
+ }
259
+ serialize = serializer.serializeStream;
260
+ } else serialize = serializer.serialize;
261
+
262
+ return serialize(data, {
263
+ // a small header shim to allow us to set headers in serializers
264
+ headers: {
265
+ set: (key, value) => {
266
+ reply.header(key, value);
267
+ },
268
+ },
269
+ });
270
+ });
271
+ });
272
+ done();
273
+ },
274
+ { name: 'content-type-negotiation' }
275
+ );
276
+
277
+ /**
278
+ * This is returns the best serializer for the request's Accept header (content negotiation)
279
+ * @param incomingMessage
280
+ * @returns {{serializer, type: string, parameters: {q: number}}|{serializer(): void}}
281
+ */
282
+ export function findBestSerializer(incomingMessage) {
283
+ const headersObject = incomingMessage.headers.asObject || incomingMessage.headers;
284
+ const acceptType = incomingMessage.requestedContentType ?? headersObject.accept;
285
+ let bestSerializer;
286
+ let bestQuality = 0;
287
+ let bestType;
288
+ let bestParameters;
289
+ const acceptTypes = acceptType ? acceptType.toLowerCase().split(/\s*,\s*/) : [];
290
+ for (const acceptType of acceptTypes) {
291
+ const [type, ...parameterParts] = acceptType.split(/\s*;\s*/);
292
+ let clientQuality = 1;
293
+ const parameters = { q: 1 };
294
+ for (const part of parameterParts) {
295
+ const equalIndex = part.indexOf('=');
296
+ parameters[part.substring(0, equalIndex)] = part.substring(equalIndex + 1);
297
+ }
298
+ clientQuality = +parameters.q;
299
+ const serializer = mediaTypes.get(type);
300
+ if (serializer) {
301
+ const quality = (serializer.q || 1) * clientQuality;
302
+ if (quality > bestQuality) {
303
+ bestSerializer = serializer;
304
+ bestType = serializer.type || type;
305
+ bestQuality = quality;
306
+ bestParameters = parameters;
307
+ }
308
+ }
309
+ }
310
+ if (!bestSerializer) {
311
+ if (acceptType) {
312
+ throw new ClientError(
313
+ 'No supported content types found in Accept header, supported types include: ' +
314
+ Array.from(mediaTypes.keys()).join(', '),
315
+ 406
316
+ );
317
+ } else {
318
+ // default if Accept header is absent
319
+ bestSerializer = mediaTypes.get('application/json');
320
+ bestType = 'application/json';
321
+ }
322
+ }
323
+
324
+ return { serializer: bestSerializer, type: bestType, parameters: bestParameters };
325
+ }
326
+
327
+ // about an average TCP packet size (if headers included)
328
+ const COMPRESSION_THRESHOLD = envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD);
329
+ /**
330
+ * Serialize a response
331
+ * @param responseData
332
+ * @param request
333
+ * @param responseObject
334
+ * @returns {Uint8Array|*}
335
+ */
336
+ export function serialize(responseData, request, responseObject) {
337
+ // TODO: Maybe support other compression encodings; browsers basically universally support brotli, but Node's HTTP
338
+ // client itself actually (just) supports gzip/deflate
339
+ let canCompress = COMPRESSION_THRESHOLD && request.headers.asObject?.['accept-encoding']?.includes('br');
340
+ let responseBody;
341
+ if (responseData?.contentType != null && responseData.data != null) {
342
+ // we use this as a special marker for blobs of data that are explicitly one content type
343
+ responseObject.headers.set('Content-Type', responseData.contentType);
344
+ responseObject.headers.set('Vary', 'Accept-Encoding');
345
+ responseBody = responseData.data;
346
+ } else if (responseData instanceof Uint8Array || responseData instanceof Blob) {
347
+ // If a user function or property returns a direct Buffer of binary data, this is the most appropriate content
348
+ // type for it.
349
+ responseObject.headers.set('Content-Type', 'application/octet-stream');
350
+ responseObject.headers.set('Vary', 'Accept-Encoding');
351
+ responseBody = responseData;
352
+ } else {
353
+ const serializer = findBestSerializer(request);
354
+ if (serializer.serializer.compressible === false) canCompress = false;
355
+ // TODO: If a different content type is preferred, look through resources to see if there is one
356
+ // specifically for that content type (most useful for html).
357
+ responseObject.headers.set('Vary', 'Accept, Accept-Encoding');
358
+ responseObject.headers.set('Content-Type', serializer.type);
359
+ if (
360
+ typeof responseData === 'object' &&
361
+ responseData &&
362
+ (responseData[Symbol.iterator] || responseData[Symbol.asyncIterator]) &&
363
+ serializer.serializer.serializeStream
364
+ ) {
365
+ if (responseData.mapError) {
366
+ // indicate that we want iterator errors to be returned so we can serialize them in a meaningful way, if possible
367
+ const getColumns = responseData.getColumns;
368
+ responseData = responseData.mapError((error) => {
369
+ // make errors serializable in a descriptive way
370
+ error.toJSON = () => ({ error: error.name, message: error.message, ...error.partialObject });
371
+ logger.warn?.(`Error serializing error ${request?.url || request}: ${error}`);
372
+ return error;
373
+ });
374
+ responseData.getColumns = getColumns;
375
+ }
376
+ let stream = serializer.serializer.serializeStream(responseData, responseObject);
377
+ if (canCompress) {
378
+ responseObject.headers.set('Content-Encoding', 'br');
379
+ stream = stream.pipe(
380
+ createBrotliCompress({
381
+ params: {
382
+ [constants.BROTLI_PARAM_MODE]:
383
+ serializer.type.includes('json') || serializer.type.includes('text')
384
+ ? constants.BROTLI_MODE_TEXT
385
+ : constants.BROTLI_MODE_GENERIC,
386
+ [constants.BROTLI_PARAM_QUALITY]: 2, // go fast
387
+ },
388
+ })
389
+ );
390
+ }
391
+ return stream;
392
+ }
393
+ responseBody = serializer.serializer.serialize(responseData, responseObject);
394
+ }
395
+ if (canCompress && responseBody?.length > COMPRESSION_THRESHOLD) {
396
+ // TODO: Only do this if the size is large and we can cache the result (otherwise use logic above)
397
+ responseObject.headers.set('Content-Encoding', 'br');
398
+ // if we have a single buffer (or string) we compress in a single async call
399
+ return new Promise((resolve, reject) =>
400
+ brotliCompress(responseBody, (err, data) => {
401
+ if (err) reject(err);
402
+ else resolve(data);
403
+ })
404
+ );
405
+ }
406
+ return responseBody;
407
+ }
408
+
409
+ let asyncSerializations: Promise<void>[];
410
+ /**
411
+ * Serialize a message, may be use multiple times (like with WebSockets)
412
+ * @param message
413
+ * @param request
414
+ * @returns {*}
415
+ */
416
+ export function serializeMessage(
417
+ message: any,
418
+ request?: Request,
419
+ inAsyncContinuation?: boolean
420
+ ): Buffer | string | Promise<Buffer | string> {
421
+ if (message?.contentType != null && message.data != null) return message.data;
422
+ asyncSerializations = inAsyncContinuation ? undefined : [];
423
+ try {
424
+ let serialized: Buffer | string;
425
+ if (request) {
426
+ let serialize = request.serialize;
427
+ if (serialize) serialized = serialize(message);
428
+ else {
429
+ const serializer = findBestSerializer(request);
430
+ serialize = request.serialize = serializer.serializer.serialize;
431
+ serialized = serialize(message);
432
+ }
433
+ } else {
434
+ serialized = JSONStringify(message);
435
+ }
436
+ if (asyncSerializations?.length > 0)
437
+ // if there were any serialization attempts that must wait for async work to be done, we wait now and then retry the serialization
438
+ return (asyncSerializations.length === 1 ? asyncSerializations[0] : Promise.all(asyncSerializations)).then(() =>
439
+ serializeMessage(message, request, true)
440
+ );
441
+ return serialized;
442
+ } finally {
443
+ asyncSerializations = undefined;
444
+ }
445
+ }
446
+
447
+ /**
448
+ * This can be called during serialization indicating that an object requires asynchronous serialization (or async completion of a task prior to serialization) to be properly serialized.
449
+ * A promise for when the object is ready to be serialized. Typically serialization will be re-executed and this object should be ready to be synchronously serialized
450
+ * @param promiseToSerialize
451
+ */
452
+ export function asyncSerialization(promiseToSerialize: Promise<any>) {
453
+ if (asyncSerializations) asyncSerializations.push(promiseToSerialize);
454
+ else throw new Error('Unable to serialize asynchronously');
455
+ }
456
+ export function hasAsyncSerialization() {
457
+ return !!asyncSerializations;
458
+ }
459
+
460
+ function streamToBuffer(stream: Readable): Promise<Buffer> {
461
+ const MAX_REQUEST_BODY_SIZE = envMgr.get(CONFIG_PARAMS.HTTP_MAXREQUESTBODYSIZE) ?? 10_000_000;
462
+ return new Promise((resolve, reject) => {
463
+ const buffers = [];
464
+ let size = 0;
465
+ stream.on('data', (data) => {
466
+ size += data.length;
467
+ if (size > MAX_REQUEST_BODY_SIZE) {
468
+ const error = new ClientError(`Request body too large, maximum size is ${MAX_REQUEST_BODY_SIZE} bytes`, 413);
469
+ buffers.length = 0; // free up memory
470
+ reject(error);
471
+ return;
472
+ }
473
+ buffers.push(data);
474
+ });
475
+ stream.on('end', () => resolve(Buffer.concat(buffers)));
476
+ stream.on('error', reject);
477
+ });
478
+ }
479
+
480
+ /**
481
+ * An object based representation of a content-type header string.
482
+ * The parameters `charset` and `boundary` have been added to the type as
483
+ * they are common parameters for the `Content-Type` header, but HTTP specifies
484
+ * that any parameter can be included (hence the `[k: string]: string`).
485
+ *
486
+ * Use `parseContentType(contentType: string)` to create this object.
487
+ */
488
+ type ContentType = {
489
+ type: string;
490
+ parameters?: { charset?: string; boundary?: string; [k: string]: string };
491
+ };
492
+
493
+ const BUFFER_ENCODINGS = [
494
+ 'ascii',
495
+ 'utf8',
496
+ 'utf-8',
497
+ 'utf16le',
498
+ 'utf-16le',
499
+ 'ucs2',
500
+ 'ucs-2',
501
+ 'base64',
502
+ 'base64url',
503
+ 'latin1',
504
+ 'binary',
505
+ 'hex',
506
+ ];
507
+ function isBufferEncoding(value: string): value is NodeJS.BufferEncoding {
508
+ return BUFFER_ENCODINGS.includes(value);
509
+ }
510
+
511
+ /**
512
+ * Parse the content-type header for the type and parameters.
513
+ * @param contentType
514
+ */
515
+ function parseContentType(contentType: string): ContentType {
516
+ // Get the first `;` character to separate the type from the parameters
517
+ const parametersStart = contentType.indexOf(';');
518
+ let parameters: ContentType['parameters'];
519
+
520
+ // If the `;` exists, then parse the parameters
521
+ if (parametersStart > -1) {
522
+ parameters = {};
523
+ // Parameters are separated by `;` and key-value pairs are separated by `=`
524
+ // i.e. `multipart/form-data; charset=UTF-8; boundary=---123`
525
+ const parts = contentType.slice(parametersStart + 1).split(';');
526
+ for (const part of parts) {
527
+ const [key, value] = part.split('=');
528
+ parameters[key.trim()] = value.trim();
529
+ }
530
+ contentType = contentType.slice(0, parametersStart);
531
+ }
532
+
533
+ return { type: contentType, parameters };
534
+ }
535
+
536
+ /**
537
+ * Given a content-type header string, get a deserializer function that can be used to parse the body.
538
+ */
539
+ export function getDeserializer(contentTypeString: string, streaming: false): Deserialize;
540
+ export function getDeserializer(
541
+ contentTypeString: string,
542
+ streaming: true
543
+ ): (stream: Readable) => Promise<ReturnType<Deserialize>>;
544
+ export function getDeserializer(
545
+ contentTypeString: string = '',
546
+ streaming: boolean = false
547
+ ): Deserialize | ((stream: Readable) => Promise<ReturnType<Deserialize>>) {
548
+ const contentType = parseContentType(contentTypeString);
549
+
550
+ const deserialize =
551
+ (contentType.type && mediaTypes.get(contentType.type)?.deserialize) || deserializerUnknownType(contentType);
552
+
553
+ return streaming ? (stream: Readable) => streamToBuffer(stream).then(deserialize) : deserialize;
554
+ }
555
+
556
+ function deserializerUnknownType(contentType: ContentType): Deserialize {
557
+ // TODO: store the content-disposition too
558
+
559
+ if (contentType.type.startsWith('text/')) {
560
+ // convert the data to a string since it is text (using the provided charset if specified)
561
+ if (contentType.parameters?.charset && !isBufferEncoding(contentType.parameters.charset)) {
562
+ logger.info?.(`Unknown Buffer encoding ${contentType.parameters.charset} in content-type. Proceeding anyways.`);
563
+ }
564
+ return (data) => ({
565
+ contentType: contentType.type,
566
+ // @ts-expect-error We are okay with passing whatever the user has specified as the encoding to the `toString` method
567
+ data: data.toString(contentType.parameters?.charset || 'utf-8'),
568
+ });
569
+ } else if (contentType.type === 'application/octet-stream') {
570
+ // use this type as a way of directly transferring binary data (since that is what it means)
571
+ return (data) => data;
572
+ } else {
573
+ return (data) => {
574
+ if (contentType.type === '') {
575
+ // try to parse as JSON if no content type
576
+ try {
577
+ // if the first byte is `{` then it is likely JSON
578
+ if (data?.[0] === 123) return JSONParse(data);
579
+ } catch {
580
+ // continue if cannot parse as JSON
581
+ }
582
+ }
583
+ // else record the type and binary data as a pair
584
+ return { contentType: contentType.type || 'application/octet-stream', data };
585
+ };
586
+ }
587
+ }
588
+
589
+ function transformIterable(iterable, transform) {
590
+ return {
591
+ [Symbol.asyncIterator]() {
592
+ const iterator = iterable[Symbol.asyncIterator] ? iterable[Symbol.asyncIterator]() : iterable[Symbol.iterator]();
593
+ return {
594
+ next() {
595
+ const step = iterator.next();
596
+ if (step.then) {
597
+ return step.then((step) => ({
598
+ value: transform(step.value),
599
+ done: step.done,
600
+ }));
601
+ }
602
+ return {
603
+ value: transform(step.value),
604
+ done: step.done,
605
+ };
606
+ },
607
+ return(value) {
608
+ return iterator.return(value);
609
+ },
610
+ throw(error) {
611
+ return iterator.throw(error);
612
+ },
613
+ };
614
+ },
615
+ };
616
+ }
617
+
618
+ /**
619
+ * Converts JS objects/arrays/iterators to a CSV stream. Should support iterators with full backpressure handling
620
+ * @param data
621
+ * @returns stream
622
+ */
623
+ export function toCsvStream(data, columns) {
624
+ // ensure that we pass it an iterable
625
+ const readStream = stream.Readable.from(data?.[Symbol.iterator] || data?.[Symbol.asyncIterator] ? data : [data]);
626
+ const options = {};
627
+ if (columns)
628
+ options.fields = columns.map((column) => ({
629
+ label: column,
630
+ value: column,
631
+ }));
632
+ const transformOptions = { objectMode: true };
633
+ // Create a json2csv stream transform.
634
+ const json2csv = new Transform(options, transformOptions);
635
+ // Pipe the data read stream through json2csv which converts it to CSV
636
+ return readStream.pipe(json2csv);
637
+ }
@@ -0,0 +1,57 @@
1
+ const { recordAction, recordActionBinary } = require('../../resources/analytics/write.ts');
2
+ const fp = require('fastify-plugin');
3
+
4
+ const ESTIMATED_HEADER_SIZE = 200; // it is very expensive to actually measure HTTP response header size (we change it
5
+ // ourselves) with an unacceptable performance penalty, so we estimate this part
6
+
7
+ module.exports = fp(
8
+ function (fastify, opts, done) {
9
+ // eslint-disable-next-line require-await
10
+ fastify.addHook('onResponse', async (request, reply) => {
11
+ // elapsedTime has to be accessed in onResponse or it won't work
12
+ let _time = reply.elapsedTime;
13
+ });
14
+ // eslint-disable-next-line require-await
15
+ fastify.addHook('onSend', async (request, reply, payload) => {
16
+ let responseTime = reply.elapsedTime;
17
+ let startTransfer = performance.now();
18
+ let config = reply.request.routeOptions;
19
+ let action;
20
+ let type;
21
+ let method;
22
+ if (config.config?.isOperation) {
23
+ action = request.body?.operation;
24
+ type = 'operation';
25
+ } else {
26
+ action = config.url;
27
+ type = 'fastify-route';
28
+ method = config.method;
29
+ }
30
+ recordAction(responseTime, 'duration', action, method, type);
31
+ // TODO: Remove the "success" metric, since we have switch to using recording responses by status code
32
+ recordActionBinary(reply.raw.statusCode < 400, 'success', action, method, type);
33
+ recordActionBinary(1, 'response_' + reply.raw.statusCode, action, method, type);
34
+ let bytesSent = ESTIMATED_HEADER_SIZE;
35
+ if (payload?.pipe) {
36
+ // if we are sending a stream, track the bytes sent and wait for when it completes
37
+ payload.on('data', (data) => {
38
+ bytesSent += data.length;
39
+ });
40
+ payload.on('end', () => {
41
+ recordAction(performance.now() - startTransfer, 'transfer', action, method, type);
42
+ recordAction(bytesSent, 'bytes-sent', action, method, type);
43
+ });
44
+ } else {
45
+ // otherwise just record bytes sent
46
+ bytesSent += payload?.length || 0;
47
+ recordAction(bytesSent, 'bytes-sent', action, method, type);
48
+ }
49
+ let roundedTime = responseTime.toFixed(3);
50
+ let appServerTiming = reply.getHeader('Server-Timing');
51
+ let serverTiming = `db;dur=${roundedTime}`;
52
+ reply.header('Server-Timing', appServerTiming ? `${appServerTiming}, ${serverTiming}` : serverTiming);
53
+ });
54
+ done();
55
+ },
56
+ { name: 'hdb-request-time' }
57
+ );