@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (444) hide show
  1. package/bin/BinObjects.js +17 -0
  2. package/bin/cliOperations.js +157 -0
  3. package/bin/copyDb.ts +280 -0
  4. package/bin/harper.js +156 -0
  5. package/bin/install.js +15 -0
  6. package/bin/lite.js +5 -0
  7. package/bin/restart.js +201 -0
  8. package/bin/run.js +409 -0
  9. package/bin/status.js +65 -0
  10. package/bin/stop.js +22 -0
  11. package/bin/upgrade.js +134 -0
  12. package/components/Application.ts +646 -0
  13. package/components/ApplicationScope.ts +49 -0
  14. package/components/Component.ts +53 -0
  15. package/components/ComponentV1.ts +342 -0
  16. package/components/DEFAULT_CONFIG.ts +18 -0
  17. package/components/EntryHandler.ts +227 -0
  18. package/components/Logger.ts +14 -0
  19. package/components/OptionsWatcher.ts +354 -0
  20. package/components/PluginModule.ts +6 -0
  21. package/components/Scope.ts +329 -0
  22. package/components/componentLoader.ts +529 -0
  23. package/components/deriveCommonPatternBase.ts +31 -0
  24. package/components/deriveGlobOptions.ts +44 -0
  25. package/components/deriveURLPath.ts +57 -0
  26. package/components/operations.js +658 -0
  27. package/components/operationsValidation.js +246 -0
  28. package/components/packageComponent.ts +39 -0
  29. package/components/requestRestart.ts +26 -0
  30. package/components/resolveBaseURLPath.ts +38 -0
  31. package/components/status/ComponentStatus.ts +110 -0
  32. package/components/status/ComponentStatusRegistry.ts +251 -0
  33. package/components/status/api.ts +153 -0
  34. package/components/status/crossThread.ts +405 -0
  35. package/components/status/errors.ts +152 -0
  36. package/components/status/index.ts +44 -0
  37. package/components/status/internal.ts +65 -0
  38. package/components/status/registry.ts +12 -0
  39. package/components/status/types.ts +96 -0
  40. package/config/RootConfigWatcher.ts +59 -0
  41. package/config/configHelpers.ts +11 -0
  42. package/config/configUtils.js +967 -0
  43. package/config/harperConfigEnvVars.ts +641 -0
  44. package/dataLayer/CreateAttributeObject.js +25 -0
  45. package/dataLayer/CreateTableObject.js +11 -0
  46. package/dataLayer/DataLayerObjects.js +43 -0
  47. package/dataLayer/DeleteBeforeObject.js +22 -0
  48. package/dataLayer/DeleteObject.js +25 -0
  49. package/dataLayer/DropAttributeObject.js +11 -0
  50. package/dataLayer/GetBackupObject.js +22 -0
  51. package/dataLayer/InsertObject.js +24 -0
  52. package/dataLayer/ReadAuditLogObject.js +24 -0
  53. package/dataLayer/SQLSearch.js +1335 -0
  54. package/dataLayer/SearchByConditionsObject.js +61 -0
  55. package/dataLayer/SearchByHashObject.js +21 -0
  56. package/dataLayer/SearchObject.js +45 -0
  57. package/dataLayer/SqlSearchObject.js +14 -0
  58. package/dataLayer/UpdateObject.js +23 -0
  59. package/dataLayer/UpsertObject.js +23 -0
  60. package/dataLayer/bulkLoad.js +813 -0
  61. package/dataLayer/dataObjects/BulkLoadObjects.js +27 -0
  62. package/dataLayer/dataObjects/UpsertObject.js +23 -0
  63. package/dataLayer/delete.js +164 -0
  64. package/dataLayer/export.js +381 -0
  65. package/dataLayer/getBackup.js +40 -0
  66. package/dataLayer/harperBridge/BridgeMethods.js +81 -0
  67. package/dataLayer/harperBridge/ResourceBridge.ts +633 -0
  68. package/dataLayer/harperBridge/bridgeUtility/insertUpdateReturnObj.js +28 -0
  69. package/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +88 -0
  70. package/dataLayer/harperBridge/harperBridge.js +21 -0
  71. package/dataLayer/harperBridge/lmdbBridge/LMDBBridge.js +119 -0
  72. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +19 -0
  73. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +112 -0
  74. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +67 -0
  75. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +31 -0
  76. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +94 -0
  77. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +98 -0
  78. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +89 -0
  79. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +109 -0
  80. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +107 -0
  81. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +137 -0
  82. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +35 -0
  83. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +111 -0
  84. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +28 -0
  85. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +29 -0
  86. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +207 -0
  87. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +156 -0
  88. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +21 -0
  89. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +30 -0
  90. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +19 -0
  91. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +64 -0
  92. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +70 -0
  93. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +22 -0
  94. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +23 -0
  95. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +22 -0
  96. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBTransactionObject.js +23 -0
  97. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +24 -0
  98. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +24 -0
  99. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +25 -0
  100. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +21 -0
  101. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +157 -0
  102. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +94 -0
  103. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +39 -0
  104. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +34 -0
  105. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +100 -0
  106. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +371 -0
  107. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +109 -0
  108. package/dataLayer/hdbInfoController.js +254 -0
  109. package/dataLayer/insert.js +266 -0
  110. package/dataLayer/readAuditLog.js +59 -0
  111. package/dataLayer/schema.js +366 -0
  112. package/dataLayer/schemaDescribe.js +289 -0
  113. package/dataLayer/search.js +60 -0
  114. package/dataLayer/transaction.js +17 -0
  115. package/dataLayer/update.js +124 -0
  116. package/dist/components/Logger.d.ts +12 -0
  117. package/dist/components/Logger.js +3 -0
  118. package/dist/components/Logger.js.map +1 -0
  119. package/dist/components/Scope.d.ts +14 -4
  120. package/dist/components/Scope.js +18 -10
  121. package/dist/components/Scope.js.map +1 -1
  122. package/dist/components/componentLoader.js +16 -9
  123. package/dist/components/componentLoader.js.map +1 -1
  124. package/dist/components/operations.js +2 -2
  125. package/dist/components/operations.js.map +1 -1
  126. package/dist/config/configUtils.d.ts +1 -1
  127. package/dist/config/configUtils.js +1 -1
  128. package/dist/config/configUtils.js.map +1 -1
  129. package/dist/dataLayer/CreateTableObject.d.ts +2 -2
  130. package/dist/dataLayer/CreateTableObject.js +2 -2
  131. package/dist/dataLayer/CreateTableObject.js.map +1 -1
  132. package/dist/dataLayer/delete.d.ts +1 -1
  133. package/dist/dataLayer/schema.js +6 -5
  134. package/dist/dataLayer/schema.js.map +1 -1
  135. package/dist/dataLayer/schemaDescribe.js +1 -1
  136. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  137. package/dist/index.d.ts +1 -1
  138. package/dist/index.js +2 -0
  139. package/dist/index.js.map +1 -1
  140. package/dist/resources/DatabaseTransaction.d.ts +1 -1
  141. package/dist/resources/IterableEventQueue.d.ts +1 -1
  142. package/dist/resources/LMDBTransaction.d.ts +5 -1
  143. package/dist/resources/Resource.d.ts +1 -1
  144. package/dist/resources/RocksIndexStore.d.ts +3 -3
  145. package/dist/resources/RocksTransactionLogStore.d.ts +6 -3
  146. package/dist/resources/Table.d.ts +15 -6
  147. package/dist/resources/Table.js +4 -1
  148. package/dist/resources/Table.js.map +1 -1
  149. package/dist/resources/analytics/read.js +32 -22
  150. package/dist/resources/analytics/read.js.map +1 -1
  151. package/dist/resources/analytics/write.js +3 -6
  152. package/dist/resources/analytics/write.js.map +1 -1
  153. package/dist/resources/auditStore.d.ts +3 -3
  154. package/dist/resources/blob.d.ts +25 -2
  155. package/dist/resources/databases.d.ts +12 -2
  156. package/dist/resources/databases.js +22 -19
  157. package/dist/resources/databases.js.map +1 -1
  158. package/dist/resources/search.js +11 -5
  159. package/dist/resources/search.js.map +1 -1
  160. package/dist/resources/transaction.d.ts +2 -1
  161. package/dist/security/auth.js +1 -1
  162. package/dist/security/auth.js.map +1 -1
  163. package/dist/security/cryptoHash.d.ts +2 -2
  164. package/dist/security/jsLoader.js +243 -66
  165. package/dist/security/jsLoader.js.map +1 -1
  166. package/dist/security/keys.js +4 -5
  167. package/dist/security/keys.js.map +1 -1
  168. package/dist/security/user.js +3 -3
  169. package/dist/security/user.js.map +1 -1
  170. package/dist/server/REST.js +16 -2
  171. package/dist/server/REST.js.map +1 -1
  172. package/dist/server/Server.d.ts +2 -1
  173. package/dist/server/Server.js.map +1 -1
  174. package/dist/server/fastifyRoutes/plugins/hdbCore.d.ts +6 -1
  175. package/dist/server/fastifyRoutes.js +2 -0
  176. package/dist/server/fastifyRoutes.js.map +1 -1
  177. package/dist/server/http.js +12 -6
  178. package/dist/server/http.js.map +1 -1
  179. package/dist/server/jobs/JobObject.d.ts +3 -3
  180. package/dist/server/loadRootComponents.js +1 -0
  181. package/dist/server/loadRootComponents.js.map +1 -1
  182. package/dist/server/operationsServer.js +3 -1
  183. package/dist/server/operationsServer.js.map +1 -1
  184. package/dist/server/serverHelpers/JSONStream.d.ts +3 -3
  185. package/dist/server/serverHelpers/Request.d.ts +5 -5
  186. package/dist/server/serverHelpers/requestTimePlugin.d.ts +1 -1
  187. package/dist/server/threads/manageThreads.d.ts +2 -2
  188. package/dist/server/threads/manageThreads.js +50 -35
  189. package/dist/server/threads/manageThreads.js.map +1 -1
  190. package/dist/server/threads/socketRouter.d.ts +1 -1
  191. package/dist/sqlTranslator/deleteTranslator.d.ts +1 -1
  192. package/dist/utility/AWS/AWSConnector.d.ts +3 -2
  193. package/dist/utility/common_utils.d.ts +3 -3
  194. package/dist/utility/environment/systemInformation.d.ts +1 -0
  195. package/dist/utility/functions/date/dateFunctions.d.ts +11 -11
  196. package/dist/utility/globalSchema.d.ts +1 -1
  197. package/dist/utility/hdbTerms.d.ts +3 -0
  198. package/dist/utility/hdbTerms.js +3 -0
  199. package/dist/utility/hdbTerms.js.map +1 -1
  200. package/dist/utility/installation.d.ts +2 -4
  201. package/dist/utility/installation.js.map +1 -1
  202. package/dist/utility/lmdb/commonUtility.d.ts +1 -0
  203. package/dist/utility/lmdb/deleteUtility.d.ts +1 -0
  204. package/dist/utility/lmdb/environmentUtility.d.ts +1 -0
  205. package/dist/utility/lmdb/searchUtility.d.ts +2 -1
  206. package/dist/utility/lmdb/writeUtility.d.ts +1 -0
  207. package/dist/utility/logging/harper_logger.d.ts +6 -6
  208. package/dist/utility/processManagement/processManagement.d.ts +1 -1
  209. package/dist/utility/processManagement/servicesConfig.d.ts +12 -6
  210. package/dist/validation/common_validators.d.ts +4 -3
  211. package/dist/validation/configValidator.d.ts +3 -2
  212. package/index.d.ts +56 -0
  213. package/index.js +41 -0
  214. package/json/systemSchema.json +373 -0
  215. package/launchServiceScripts/launchHarperDB.js +3 -0
  216. package/launchServiceScripts/utility/checkNodeVersion.js +15 -0
  217. package/package.json +21 -3
  218. package/resources/DatabaseTransaction.ts +378 -0
  219. package/resources/ErrorResource.ts +57 -0
  220. package/resources/IterableEventQueue.ts +94 -0
  221. package/resources/LMDBTransaction.ts +349 -0
  222. package/resources/RecordEncoder.ts +702 -0
  223. package/resources/RequestTarget.ts +134 -0
  224. package/resources/Resource.ts +789 -0
  225. package/resources/ResourceInterface.ts +221 -0
  226. package/resources/ResourceInterfaceV2.ts +53 -0
  227. package/resources/ResourceV2.ts +67 -0
  228. package/resources/Resources.ts +162 -0
  229. package/resources/RocksIndexStore.ts +70 -0
  230. package/resources/RocksTransactionLogStore.ts +352 -0
  231. package/resources/Table.ts +4527 -0
  232. package/resources/analytics/hostnames.ts +72 -0
  233. package/resources/analytics/metadata.ts +10 -0
  234. package/resources/analytics/read.ts +252 -0
  235. package/resources/analytics/write.ts +803 -0
  236. package/resources/auditStore.ts +556 -0
  237. package/resources/blob.ts +1268 -0
  238. package/resources/crdt.ts +125 -0
  239. package/resources/dataLoader.ts +527 -0
  240. package/resources/databases.ts +1290 -0
  241. package/resources/graphql.ts +221 -0
  242. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +638 -0
  243. package/resources/indexes/customIndexes.ts +7 -0
  244. package/resources/indexes/vector.ts +38 -0
  245. package/resources/jsResource.ts +86 -0
  246. package/resources/loadEnv.ts +22 -0
  247. package/resources/login.ts +18 -0
  248. package/resources/openApi.ts +409 -0
  249. package/resources/registrationDeprecated.ts +8 -0
  250. package/resources/replayLogs.ts +136 -0
  251. package/resources/roles.ts +98 -0
  252. package/resources/search.ts +1301 -0
  253. package/resources/tracked.ts +584 -0
  254. package/resources/transaction.ts +89 -0
  255. package/resources/transactionBroadcast.ts +258 -0
  256. package/security/auth.ts +376 -0
  257. package/security/certificateVerification/certificateVerificationSource.ts +84 -0
  258. package/security/certificateVerification/configValidation.ts +107 -0
  259. package/security/certificateVerification/crlVerification.ts +623 -0
  260. package/security/certificateVerification/index.ts +121 -0
  261. package/security/certificateVerification/ocspVerification.ts +148 -0
  262. package/security/certificateVerification/pkijs-ed25519-patch.ts +188 -0
  263. package/security/certificateVerification/types.ts +128 -0
  264. package/security/certificateVerification/verificationConfig.ts +138 -0
  265. package/security/certificateVerification/verificationUtils.ts +447 -0
  266. package/security/cryptoHash.js +42 -0
  267. package/security/data_objects/PermissionAttributeResponseObject.js +15 -0
  268. package/security/data_objects/PermissionResponseObject.js +115 -0
  269. package/security/data_objects/PermissionTableResponseObject.js +20 -0
  270. package/security/fastifyAuth.js +169 -0
  271. package/security/impersonation.ts +160 -0
  272. package/security/jsLoader.ts +716 -0
  273. package/security/keys.js +948 -0
  274. package/security/permissionsTranslator.js +300 -0
  275. package/security/role.js +218 -0
  276. package/security/tokenAuthentication.ts +228 -0
  277. package/security/user.ts +449 -0
  278. package/server/DurableSubscriptionsSession.ts +503 -0
  279. package/server/REST.ts +407 -0
  280. package/server/Server.ts +89 -0
  281. package/server/fastifyRoutes/helpers/getCORSOptions.js +36 -0
  282. package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +15 -0
  283. package/server/fastifyRoutes/helpers/getServerOptions.js +33 -0
  284. package/server/fastifyRoutes/plugins/hdbCore.js +39 -0
  285. package/server/fastifyRoutes.ts +205 -0
  286. package/server/graphqlQuerying.ts +700 -0
  287. package/server/http.ts +640 -0
  288. package/server/itc/serverHandlers.js +161 -0
  289. package/server/itc/utility/ITCEventObject.js +10 -0
  290. package/server/jobs/JobObject.js +24 -0
  291. package/server/jobs/jobProcess.js +69 -0
  292. package/server/jobs/jobRunner.js +162 -0
  293. package/server/jobs/jobs.js +304 -0
  294. package/server/loadRootComponents.js +44 -0
  295. package/server/mqtt.ts +485 -0
  296. package/server/nodeName.ts +75 -0
  297. package/server/operationsServer.ts +313 -0
  298. package/server/serverHelpers/Headers.ts +108 -0
  299. package/server/serverHelpers/JSONStream.ts +269 -0
  300. package/server/serverHelpers/OperationFunctionObject.ts +13 -0
  301. package/server/serverHelpers/Request.ts +158 -0
  302. package/server/serverHelpers/contentTypes.ts +637 -0
  303. package/server/serverHelpers/requestTimePlugin.js +57 -0
  304. package/server/serverHelpers/serverHandlers.js +148 -0
  305. package/server/serverHelpers/serverUtilities.ts +473 -0
  306. package/server/serverRegistry.ts +8 -0
  307. package/server/static.ts +187 -0
  308. package/server/status/definitions.ts +37 -0
  309. package/server/status/index.ts +125 -0
  310. package/server/storageReclamation.ts +93 -0
  311. package/server/threads/itc.js +89 -0
  312. package/server/threads/manageThreads.js +594 -0
  313. package/server/threads/socketRouter.ts +360 -0
  314. package/server/threads/threadServer.js +279 -0
  315. package/server/throttle.ts +73 -0
  316. package/sqlTranslator/SelectValidator.js +330 -0
  317. package/sqlTranslator/alasqlFunctionImporter.js +62 -0
  318. package/sqlTranslator/deleteTranslator.js +67 -0
  319. package/sqlTranslator/index.js +242 -0
  320. package/sqlTranslator/sql_statement_bucket.js +472 -0
  321. package/static/defaultConfig.yaml +3 -0
  322. package/studio/web/HDBDogOnly.svg +78 -0
  323. package/studio/web/assets/PPRadioGrotesk-Bold-DDaUYG8E.woff +0 -0
  324. package/studio/web/assets/fa-brands-400-CEJbCg16.woff +0 -0
  325. package/studio/web/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
  326. package/studio/web/assets/fa-brands-400-DnkPfk3o.eot +0 -0
  327. package/studio/web/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
  328. package/studio/web/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
  329. package/studio/web/assets/fa-regular-400-BhTwtT8w.eot +0 -0
  330. package/studio/web/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
  331. package/studio/web/assets/fa-regular-400-DFnMcJPd.woff +0 -0
  332. package/studio/web/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
  333. package/studio/web/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
  334. package/studio/web/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
  335. package/studio/web/assets/fa-solid-900-B6Axprfb.eot +0 -0
  336. package/studio/web/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
  337. package/studio/web/assets/fa-solid-900-DOXgCApm.woff +0 -0
  338. package/studio/web/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
  339. package/studio/web/assets/index-BTgXJX9d.js +235 -0
  340. package/studio/web/assets/index-BTgXJX9d.js.map +1 -0
  341. package/studio/web/assets/index-C-GXfcup.js +37 -0
  342. package/studio/web/assets/index-C-GXfcup.js.map +1 -0
  343. package/studio/web/assets/index-PFlNdimM.js +2 -0
  344. package/studio/web/assets/index-PFlNdimM.js.map +1 -0
  345. package/studio/web/assets/index-Y2g_iFpU.css +1 -0
  346. package/studio/web/assets/index-jiPwkrsB.css +1 -0
  347. package/studio/web/assets/index.lazy-C3TJZJ4o.js +266 -0
  348. package/studio/web/assets/index.lazy-C3TJZJ4o.js.map +1 -0
  349. package/studio/web/assets/profiler-DotzgiCJ.js +2 -0
  350. package/studio/web/assets/profiler-DotzgiCJ.js.map +1 -0
  351. package/studio/web/assets/react-redux-VxUEx_mU.js +6 -0
  352. package/studio/web/assets/react-redux-VxUEx_mU.js.map +1 -0
  353. package/studio/web/assets/startRecording-B_9J9Csd.js +3 -0
  354. package/studio/web/assets/startRecording-B_9J9Csd.js.map +1 -0
  355. package/studio/web/fabric-signup-background.webp +0 -0
  356. package/studio/web/fabric-signup-text.png +0 -0
  357. package/studio/web/favicon_purple.png +0 -0
  358. package/studio/web/github-icon.svg +15 -0
  359. package/studio/web/harper-fabric_black.png +0 -0
  360. package/studio/web/harper-fabric_white.png +0 -0
  361. package/studio/web/harper-studio_white.png +0 -0
  362. package/studio/web/index.html +16 -0
  363. package/studio/web/running.css +148 -0
  364. package/studio/web/running.html +147 -0
  365. package/studio/web/running.js +111 -0
  366. package/upgrade/UpgradeObjects.js +13 -0
  367. package/upgrade/directives/directivesController.js +90 -0
  368. package/upgrade/directivesManager.js +139 -0
  369. package/upgrade/upgradePrompt.js +124 -0
  370. package/upgrade/upgradeUtilities.js +28 -0
  371. package/utility/AWS/AWSConnector.js +29 -0
  372. package/utility/OperationFunctionCaller.js +63 -0
  373. package/utility/assignCmdEnvVariables.js +62 -0
  374. package/utility/common_utils.js +867 -0
  375. package/utility/environment/environmentManager.js +208 -0
  376. package/utility/environment/systemInformation.js +355 -0
  377. package/utility/errors/commonErrors.js +267 -0
  378. package/utility/errors/hdbError.js +146 -0
  379. package/utility/functions/date/dateFunctions.js +65 -0
  380. package/utility/functions/geo.js +355 -0
  381. package/utility/functions/sql/alaSQLExtension.js +104 -0
  382. package/utility/globalSchema.js +35 -0
  383. package/utility/hdbTerms.ts +819 -0
  384. package/utility/install/checkJWTTokensExist.js +62 -0
  385. package/utility/install/harperdb.conf +15 -0
  386. package/utility/install/harperdb.service +14 -0
  387. package/utility/install/installer.js +635 -0
  388. package/utility/installation.ts +30 -0
  389. package/utility/lmdb/DBIDefinition.js +20 -0
  390. package/utility/lmdb/DeleteRecordsResponseObject.js +25 -0
  391. package/utility/lmdb/InsertRecordsResponseObject.js +22 -0
  392. package/utility/lmdb/OpenDBIObject.js +31 -0
  393. package/utility/lmdb/OpenEnvironmentObject.js +41 -0
  394. package/utility/lmdb/UpdateRecordsResponseObject.js +25 -0
  395. package/utility/lmdb/UpsertRecordsResponseObject.js +22 -0
  396. package/utility/lmdb/cleanLMDBMap.js +65 -0
  397. package/utility/lmdb/commonUtility.js +119 -0
  398. package/utility/lmdb/deleteUtility.js +128 -0
  399. package/utility/lmdb/environmentUtility.js +477 -0
  400. package/utility/lmdb/searchCursorFunctions.js +187 -0
  401. package/utility/lmdb/searchUtility.js +918 -0
  402. package/utility/lmdb/terms.js +57 -0
  403. package/utility/lmdb/writeUtility.js +407 -0
  404. package/utility/logging/harper_logger.js +876 -0
  405. package/utility/logging/logRotator.js +157 -0
  406. package/utility/logging/logger.ts +24 -0
  407. package/utility/logging/readLog.js +355 -0
  408. package/utility/logging/transactionLog.js +57 -0
  409. package/utility/mount_hdb.js +59 -0
  410. package/utility/npmUtilities.js +102 -0
  411. package/utility/operationPermissions.ts +112 -0
  412. package/utility/operation_authorization.js +836 -0
  413. package/utility/packageUtils.js +55 -0
  414. package/utility/password.ts +99 -0
  415. package/utility/processManagement/processManagement.js +187 -0
  416. package/utility/processManagement/servicesConfig.js +56 -0
  417. package/utility/scripts/restartHdb.js +24 -0
  418. package/utility/scripts/user_data.sh +13 -0
  419. package/utility/signalling.js +36 -0
  420. package/utility/terms/certificates.js +81 -0
  421. package/utility/when.ts +20 -0
  422. package/v1.d.ts +39 -0
  423. package/v1.js +41 -0
  424. package/v2.d.ts +39 -0
  425. package/v2.js +41 -0
  426. package/validation/bulkDeleteValidator.js +24 -0
  427. package/validation/check_permissions.js +19 -0
  428. package/validation/common_validators.js +95 -0
  429. package/validation/configValidator.js +331 -0
  430. package/validation/deleteValidator.js +15 -0
  431. package/validation/fileLoadValidator.js +153 -0
  432. package/validation/insertValidator.js +40 -0
  433. package/validation/installValidator.js +37 -0
  434. package/validation/readLogValidator.js +64 -0
  435. package/validation/role_validation.js +320 -0
  436. package/validation/schemaMetadataValidator.js +42 -0
  437. package/validation/searchValidator.js +166 -0
  438. package/validation/statusValidator.ts +66 -0
  439. package/validation/transactionLogValidator.js +33 -0
  440. package/validation/user_validation.js +55 -0
  441. package/validation/validationWrapper.js +105 -0
  442. package/dist/resources/analytics/profile.d.ts +0 -2
  443. package/dist/resources/analytics/profile.js +0 -144
  444. package/dist/resources/analytics/profile.js.map +0 -1
@@ -0,0 +1,313 @@
1
+ import cluster from 'cluster';
2
+ import zlib from 'node:zlib';
3
+ import env from '../utility/environment/environmentManager.js';
4
+ env.initSync();
5
+ import * as terms from '../utility/hdbTerms.ts';
6
+ import harperLogger from '../utility/logging/harper_logger.js';
7
+ import fastify, { FastifyInstance, FastifyReply, FastifyRequest, FastifyServerOptions } from 'fastify';
8
+ import fastifyCors, { type FastifyCorsOptions } from '@fastify/cors';
9
+ import fastifyCompress from '@fastify/compress';
10
+ import fastifyStatic from '@fastify/static';
11
+ import requestTimePlugin from './serverHelpers/requestTimePlugin.js';
12
+ import guidePath from 'path';
13
+ import { PACKAGE_ROOT } from '../utility/packageUtils.js';
14
+ import globalSchema from '../utility/globalSchema.js';
15
+ import commonUtils from '../utility/common_utils.js';
16
+ import * as userSchema from '../security/user.ts';
17
+ import { server as serverRegistration, type ServerOptions } from '../server/Server.ts';
18
+ import {
19
+ authHandler,
20
+ authAndEnsureUserOnRequest,
21
+ handlePostRequest,
22
+ serverErrorHandler,
23
+ reqBodyValidationHandler,
24
+ } from './serverHelpers/serverHandlers.js';
25
+ import { registerContentHandlers } from './serverHelpers/contentTypes.ts';
26
+ import type { OperationFunctionName } from './serverHelpers/serverUtilities.ts';
27
+ import type { ParsedSqlObject } from '../sqlTranslator/index.js';
28
+ import { generateJsonApi } from '../resources/openApi.ts';
29
+ import { Resources } from '../resources/Resources.ts';
30
+ import { ServerError } from '../utility/errors/hdbError.js';
31
+
32
+ const DEFAULT_HEADERS_TIMEOUT = 60000;
33
+ const REQ_MAX_BODY_SIZE = env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_MAXREQUESTBODYSIZE) ?? 1024 * 1024 * 1024; //this defaults to 1GB in bytes
34
+ const TRUE_COMPARE_VAL = 'TRUE';
35
+
36
+ const { CONFIG_PARAMS } = terms;
37
+ let server;
38
+
39
+ export { operationsServer as hdbServer };
40
+ export { operationsServer as start };
41
+
42
+ /**
43
+ * Builds a Harper server.
44
+ */
45
+ async function operationsServer(options: ServerOptions & { resources?: Resources }) {
46
+ try {
47
+ harperLogger.debug('In Fastify server' + process.cwd());
48
+ harperLogger.debug(`Running with NODE_ENV set as: ${process.env.NODE_ENV}`);
49
+ harperLogger.debug(`Harper server process ${process.pid} starting up.`);
50
+
51
+ global.clustering_on = false;
52
+ global.isMaster = cluster.isMaster;
53
+
54
+ await setUp();
55
+ // if we have a secure port, need to use the secure HTTP server for fastify (it can be used for HTTP as well)
56
+ const isHttps = options.securePort > 0;
57
+
58
+ //generate a Fastify server instance
59
+ server = buildServer(isHttps, options.resources);
60
+
61
+ //make sure the process waits for the server to be fully instantiated before moving forward
62
+ await server.ready();
63
+ if (!options) options = {};
64
+ options.usageType = 'operations-api';
65
+ // fastify can't clean up properly
66
+ try {
67
+ // now that server is fully loaded/ready, start listening on port provided in config settings or just use
68
+ // zero to wait for sockets from the main thread
69
+ serverRegistration.http(server.server, options);
70
+ if (!server.server.closeIdleConnections) {
71
+ // before Node v18, closeIdleConnections is not available, and we have to setup a listener for fastify
72
+ // to handle closing by setting up the dynamic port
73
+ await server.listen({ port: 0, host: '::' });
74
+ }
75
+ } catch (err) {
76
+ server.close();
77
+ harperLogger.error(err);
78
+ harperLogger.error(`Error configuring operations server`);
79
+ throw err;
80
+ }
81
+ } catch (err) {
82
+ console.error(`Failed to build server on ${process.pid}`, err);
83
+ harperLogger.fatal(err);
84
+ process.exit(1);
85
+ }
86
+ }
87
+
88
+ /**
89
+ * Makes sure global values are set and that clustering connections are set/ready before server starts.
90
+ */
91
+ async function setUp() {
92
+ harperLogger.trace('Configuring Harper process.');
93
+ globalSchema.setSchemaDataToGlobal();
94
+ return userSchema.setUsersWithRolesCache();
95
+ }
96
+
97
+ export interface ImpersonatePayload {
98
+ username?: string;
99
+ role?: {
100
+ permission: Partial<userSchema.UserRoleNamedPermissions & userSchema.UserRoleDatabasePermissions>;
101
+ };
102
+ role_name?: string;
103
+ }
104
+
105
+ interface BaseOperationRequestBody {
106
+ operation: OperationFunctionName;
107
+ bypassAuth: boolean;
108
+ hdb_user?: userSchema.User;
109
+ hdbAuthHeader?: unknown;
110
+ bypass_auth?: boolean;
111
+ impersonate?: ImpersonatePayload;
112
+ password?: string;
113
+ payload?: string;
114
+ sql?: string;
115
+ parsedSqlObject?: ParsedSqlObject;
116
+ [key: string]: unknown;
117
+ }
118
+
119
+ type SearchOperation = BaseOperationRequestBody;
120
+
121
+ interface SearchOperationRequestBody {
122
+ search_operation: SearchOperation;
123
+ }
124
+
125
+ export type OperationRequestBody = BaseOperationRequestBody & Partial<SearchOperationRequestBody>;
126
+
127
+ export interface OperationRequest {
128
+ body: OperationRequestBody;
129
+ }
130
+
131
+ export interface OperationResult {
132
+ message?: any;
133
+ }
134
+
135
+ /**
136
+ * This method configures and returns a Fastify server - for either HTTP or HTTPS - based on the provided config settings
137
+ */
138
+ function buildServer(isHttps: boolean, resources: Resources): FastifyInstance {
139
+ harperLogger.debug(`Harper process starting to build ${isHttps ? 'HTTPS' : 'HTTP'} server.`);
140
+ const serverOpts = getServerOptions(isHttps);
141
+
142
+ const app = fastify(serverOpts);
143
+
144
+ // Fastify does not set this property in the initial app construction
145
+ app.server.headersTimeout = getHeaderTimeoutConfig();
146
+
147
+ // Set a top-level error handler for the server - all errors caught/thrown within the API will bubble up to this
148
+ // handler so that they can be handled in a coordinated way
149
+ app.setErrorHandler(serverErrorHandler);
150
+
151
+ const corsOptions = getCORSOpts();
152
+ if (corsOptions) {
153
+ app.register(fastifyCors, corsOptions);
154
+ }
155
+
156
+ app.register(function (instance, options, done) {
157
+ instance.setNotFoundHandler(function (request, reply) {
158
+ if (reply.sent || reply.raw.headersSent || reply.raw.writableEnded) return;
159
+ app.server.emit('unhandled', request.raw, reply.raw);
160
+ });
161
+ done();
162
+ });
163
+
164
+ app.register(requestTimePlugin);
165
+
166
+ // This handles all get requests for the studio
167
+ app.register(fastifyCompress, {
168
+ brotliOptions: {
169
+ params: {
170
+ [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT, // useful for APIs that primarily return text
171
+ [zlib.constants.BROTLI_PARAM_QUALITY]: 2, // default is 4, max is 11, min is 0
172
+ },
173
+ },
174
+ });
175
+ registerContentHandlers(app);
176
+
177
+ // Add a simple health check
178
+ app.get('/health', () => 'Harper is running.');
179
+
180
+ // Add a top-level GET handler for browsers.
181
+ app.register(fastifyStatic, { root: guidePath.join(PACKAGE_ROOT, 'studio/web') });
182
+ const studioOn = env.get(terms.HDB_SETTINGS_NAMES.LOCAL_STUDIO_ON);
183
+ if (!commonUtils.isEmpty(studioOn) && studioOn.toString().toLowerCase() === 'true') {
184
+ app.get('/', (req, res) => res.sendFile('index.html'));
185
+ } else {
186
+ app.get('/', (req, res) => res.sendFile('running.html'));
187
+ }
188
+
189
+ // Describe the APIs.
190
+ app.get('/api/openapi/rest', { preValidation: [authAndEnsureUserOnRequest] }, restOpenAPIHandler(resources));
191
+
192
+ // Add the top-level POST handler.
193
+ app.post<{ Body: OperationRequestBody }, { isOperation?: boolean }>(
194
+ '/',
195
+ {
196
+ preValidation: [reqBodyValidationHandler, authHandler],
197
+ config: { isOperation: true },
198
+ },
199
+ handler
200
+ );
201
+
202
+ harperLogger.debug(`Harper process starting up ${isHttps ? 'HTTPS' : 'HTTP'} server listener.`);
203
+
204
+ return app;
205
+ }
206
+
207
+ function restOpenAPIHandler(resources: Resources) {
208
+ const httpPort = env.get(terms.CONFIG_PARAMS.HTTP_PORT);
209
+ const httpSecurePort = env.get(terms.CONFIG_PARAMS.HTTP_SECUREPORT);
210
+ return (req: FastifyRequest & { hdb_user?: { role?: { permission?: { super_user: boolean } } } }) => {
211
+ if (req.hdb_user?.role?.permission?.super_user) {
212
+ return generateJsonApi(resources, calculateRestHttpURL(httpPort, httpSecurePort, req));
213
+ } else {
214
+ harperLogger.warn(
215
+ `{"ip":"${req.socket.remoteAddress}", "error":"attempt to access /api/openapi/rest without being super_user"`
216
+ );
217
+ return new ServerError(`Forbidden`, 403);
218
+ }
219
+ };
220
+ }
221
+
222
+ export function calculateRestHttpURL(
223
+ httpPort: string | undefined,
224
+ httpSecurePort: string | undefined,
225
+ req: { hostname: string; protocol: string }
226
+ ): string {
227
+ const httpURL = new URL(`${req.protocol}://${req.hostname}`);
228
+ // note that the request is from fastify, which doesn't seem to have a correct hostname property (includes port), so the URL needs to be used for hostname
229
+ if (httpURL.hostname.toLowerCase() === 'localhost' || httpURL.hostname.match(/^[\d.:]+$/)) {
230
+ // Only use ports when running against localhost, or an ip address.
231
+ if (httpSecurePort) {
232
+ httpURL.port = httpSecurePort;
233
+ httpURL.protocol = 'https:';
234
+ } else if (httpPort) {
235
+ httpURL.port = httpPort;
236
+ httpURL.protocol = 'http:';
237
+ }
238
+ } else {
239
+ // Otherwise, assume that port forwarding is happening, and possibly SSL termination.
240
+ httpURL.port = '443';
241
+ httpURL.protocol = 'https:';
242
+ }
243
+ return httpURL.toString();
244
+ }
245
+
246
+ function handler(req: FastifyRequest<{ Body?: OperationRequestBody }>, reply: FastifyReply) {
247
+ // if the operation is a restart, we have to tell the client not to use keep alive on this connection
248
+ // anymore; it needs to be closed because this thread is going to be terminated
249
+ if (req.body?.operation?.startsWith('restart')) {
250
+ reply.header('Connection', 'close');
251
+ }
252
+ //if no error is thrown below, the response 'data' returned from the handler will be returned with 200/OK code
253
+ return handlePostRequest(req, reply);
254
+ }
255
+
256
+ interface HttpServerOptions extends FastifyServerOptions {
257
+ https?: boolean;
258
+ http2?: boolean;
259
+ }
260
+
261
+ /**
262
+ * Builds server options object to pass to Fastify when using server factory.
263
+ */
264
+ function getServerOptions(isHttps: boolean): HttpServerOptions {
265
+ const server_timeout = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_TIMEOUT);
266
+ const keep_alive_timeout = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_KEEPALIVETIMEOUT);
267
+ return {
268
+ bodyLimit: REQ_MAX_BODY_SIZE,
269
+ connectionTimeout: server_timeout,
270
+ keepAliveTimeout: keep_alive_timeout,
271
+ forceCloseConnections: true,
272
+ return503OnClosing: false,
273
+ http2: env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_HTTP2),
274
+ https: isHttps /* && {
275
+ allowHTTP1: true,
276
+ },*/,
277
+ };
278
+ }
279
+
280
+ /**
281
+ * Builds CORS options object to pass to cors plugin when/if it needs to be registered with Fastify
282
+ */
283
+ function getCORSOpts(): FastifyCorsOptions {
284
+ const propsCors = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORS);
285
+ const propsCorsAccesslist = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST);
286
+ let corsOptions: FastifyCorsOptions;
287
+
288
+ if (propsCors && (propsCors === true || propsCors.toUpperCase() === TRUE_COMPARE_VAL)) {
289
+ corsOptions = {
290
+ origin: true,
291
+ allowedHeaders: ['Content-Type', 'Authorization', 'Accept'],
292
+ credentials: false,
293
+ };
294
+ if (
295
+ propsCorsAccesslist &&
296
+ propsCorsAccesslist.length > 0 &&
297
+ propsCorsAccesslist[0] !== null &&
298
+ propsCorsAccesslist[0] !== '*'
299
+ ) {
300
+ corsOptions.origin = (origin, callback) => {
301
+ return callback(null, propsCorsAccesslist.indexOf(origin) !== -1);
302
+ };
303
+ }
304
+ }
305
+ return corsOptions;
306
+ }
307
+
308
+ /**
309
+ * Returns header timeout value from config file or, if not entered, the default value
310
+ */
311
+ function getHeaderTimeoutConfig(): number {
312
+ return env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_HEADERSTIMEOUT) ?? DEFAULT_HEADERS_TIMEOUT;
313
+ }
@@ -0,0 +1,108 @@
1
+ /**
2
+ * Fast implementation of standard Headers
3
+ */
4
+ export class Headers extends Map<string, string | string[]> {
5
+ constructor(init?: Headers | HeadersInit) {
6
+ if (init) {
7
+ if (init[Symbol.iterator]) {
8
+ super(init);
9
+ } else {
10
+ super();
11
+ for (const name in init) this.set(name, init[name]);
12
+ }
13
+ } else super();
14
+ }
15
+ set(name, value) {
16
+ if (typeof name !== 'string') name = '' + name;
17
+ if (Array.isArray(value)) {
18
+ // Ensure all array elements are strings
19
+ value = value.map((v) => (typeof v === 'string' ? v : '' + v));
20
+ } else if (typeof value !== 'string') {
21
+ value = '' + value;
22
+ }
23
+ return super.set(name.toLowerCase(), [name, value]);
24
+ }
25
+ get(name) {
26
+ if (typeof name !== 'string') name = '' + name;
27
+ return super.get(name.toLowerCase())?.[1];
28
+ }
29
+ has(name) {
30
+ if (typeof name !== 'string') name = '' + name;
31
+ return super.has(name.toLowerCase());
32
+ }
33
+ setIfNone(name, value) {
34
+ if (typeof name !== 'string') name = '' + name;
35
+ if (typeof value !== 'string') value = '' + value;
36
+ const lowerName = name.toLowerCase();
37
+ if (!super.has(lowerName)) return super.set(lowerName, [name, value]);
38
+ }
39
+ append(name, value, commaDelimited) {
40
+ if (typeof name !== 'string') name = '' + name;
41
+ if (typeof value !== 'string') value = '' + value;
42
+ const lowerName = name.toLowerCase();
43
+ const existing = super.get(lowerName);
44
+ if (existing) {
45
+ const existingValue = existing[1];
46
+ if (commaDelimited)
47
+ value = (typeof existingValue === 'string' ? existingValue : existingValue.join(', ')) + ', ' + value;
48
+ else if (typeof existingValue === 'string') value = [existingValue, value];
49
+ else {
50
+ existingValue.push(value);
51
+ return;
52
+ }
53
+ }
54
+ return super.set(lowerName, [name, value]);
55
+ }
56
+ [Symbol.iterator]() {
57
+ return super.values()[Symbol.iterator]();
58
+ }
59
+ }
60
+
61
+ export function appendHeader(headers, name, value, commaDelimited) {
62
+ if (headers.append) {
63
+ headers.append(name, value, commaDelimited);
64
+ } else if (headers.set) {
65
+ const existingValue = headers.get(name);
66
+ if (existingValue) {
67
+ if (commaDelimited)
68
+ value = (typeof existingValue === 'string' ? existingValue : existingValue.join(', ')) + ', ' + value;
69
+ else if (typeof existingValue === 'string') value = [existingValue, value];
70
+ else {
71
+ existingValue.push(value);
72
+ return;
73
+ }
74
+ }
75
+ return headers.set(name, value);
76
+ } else {
77
+ headers[name] = (headers[name] ? headers[name] + ', ' : '') + value;
78
+ }
79
+ }
80
+
81
+ /**
82
+ * Merge headers from source into target, ensuring that target is a Headers object, and avoiding any overwrite
83
+ * of existing headers in target.
84
+ * @param target
85
+ * @param source
86
+ */
87
+ export function mergeHeaders(target: any, source: Headers) {
88
+ // ensure target is a Headers object, which could be this Headers class, the global.Headers, or even a Map, which is ok
89
+ if (typeof target.set !== 'function' || typeof target.has !== 'function') target = new Headers(target);
90
+ for (const [name, value] of source) {
91
+ if (!target.has(name)) target.set(name, value);
92
+ else if (name.toLowerCase() === 'set-cookie') {
93
+ // Set-Cookie headers must NEVER be comma-delimited
94
+ // If value is an array, append each one separately; otherwise append the single value
95
+ const values = Array.isArray(value) ? value : [value];
96
+ if (target.append) {
97
+ for (const v of values) target.append(name, v);
98
+ } else {
99
+ // Fallback for Map or objects without append method
100
+ // We know existing exists because we're in the else-if branch (target.has(name) is true)
101
+ const existing = target.get(name);
102
+ const newValue = Array.isArray(existing) ? [...existing, ...values] : [existing, ...values];
103
+ target.set(name, newValue);
104
+ }
105
+ }
106
+ }
107
+ return target;
108
+ }
@@ -0,0 +1,269 @@
1
+ import { Readable } from 'stream';
2
+ import * as harperLogger from '../../utility/logging/harper_logger.js';
3
+ import { when } from '../../utility/when.ts';
4
+ import JSONbig from 'json-bigint-fixes';
5
+ const JSONbigint = JSONbig({ useNativeBigInt: true });
6
+ const BUFFER_SIZE = 10000;
7
+ const BIGINT_SERIALIZATION = { message: 'Cannot serialize BigInt to JSON' };
8
+ BigInt.prototype.toJSON = function () {
9
+ throw BIGINT_SERIALIZATION;
10
+ };
11
+ const { errorToString } = harperLogger;
12
+ export function streamAsJSON(value) {
13
+ return new JSONStream({ value });
14
+ }
15
+ // a readable stream for serializing a set of variables to a JSON stream
16
+ class JSONStream extends Readable {
17
+ constructor(options) {
18
+ // Calls the stream.Readable(options) constructor
19
+ super(options);
20
+ this.buffer = [];
21
+ this.bufferSize = 0;
22
+ this.iterator = this.serialize(options.value, true);
23
+ this.activeIterators = [];
24
+ }
25
+
26
+ *serialize(object) {
27
+ // using a generator to serialize JSON for convenience of recursive pause and resume functionality
28
+ // serialize a value to an iterator that can be consumed by streaming API
29
+ if (object && typeof object === 'object') {
30
+ const hasAsyncIterator = object[Symbol.asyncIterator];
31
+ const hasIterator = object[Symbol.iterator];
32
+ if ((hasIterator || hasAsyncIterator) && !object.then) {
33
+ yield '[';
34
+ let first = true;
35
+ if ((hasAsyncIterator || hasIterator) && !(object instanceof Array)) {
36
+ let iterator = hasAsyncIterator ? object[Symbol.asyncIterator]() : object[Symbol.iterator]();
37
+ this.activeIterators.push(iterator);
38
+ let iteratorResult;
39
+ while (true) {
40
+ try {
41
+ iteratorResult = iterator.next();
42
+ if (iteratorResult.then) {
43
+ yield iteratorResult.then(
44
+ (result) => {
45
+ // don't send anything here, assign the iteratorResult and continue
46
+ // with the serialization loop
47
+ iteratorResult = result;
48
+ return ''; // return nothing now and serialization will continue below
49
+ },
50
+ (error) => {
51
+ // try to properly serialize the error, but then finish the iterator
52
+ harperLogger.warn('Error serializing in stream', error);
53
+ iteratorResult = { done: false, value: { error: errorToString(error) } };
54
+ iterator = {
55
+ next: () => ({ done: true }),
56
+ };
57
+ return '';
58
+ }
59
+ );
60
+ }
61
+ } catch (error) {
62
+ iteratorResult = { done: false, value: { error: errorToString(error) } };
63
+ iterator = {
64
+ next: () => ({ done: true }),
65
+ };
66
+ }
67
+ if (iteratorResult.done) {
68
+ this.activeIterators.splice(this.activeIterators.indexOf(iterator), 1);
69
+ yield ']';
70
+ return;
71
+ } else {
72
+ if (first) {
73
+ first = false;
74
+ } else {
75
+ yield ',';
76
+ }
77
+ yield* this.serialize(iteratorResult.value);
78
+ }
79
+ }
80
+ }
81
+ for (const element of object) {
82
+ if (first) first = false;
83
+ else {
84
+ yield ',';
85
+ }
86
+ yield* this.serialize(element);
87
+ }
88
+ yield ']';
89
+ return;
90
+ }
91
+ if (object.then) {
92
+ try {
93
+ yield object.then((object) => this.serialize(object), handleError);
94
+ } catch (error) {
95
+ yield handleError(error);
96
+ }
97
+ } else {
98
+ yield stringify(object);
99
+ }
100
+ } else {
101
+ yield stringify(object);
102
+ }
103
+ }
104
+
105
+ _read() {
106
+ if (this._amReading) {
107
+ // I don't know why _read is called from within a push call, but if we are already reading, ignore the call
108
+ return;
109
+ }
110
+ this._amReading = true;
111
+ if (this.done) {
112
+ return this.push(null);
113
+ }
114
+ when(
115
+ this.readIterator(this.iterator),
116
+ (done) => {
117
+ if (done) {
118
+ this.done = true;
119
+ this.push(null);
120
+ } else {
121
+ this._amReading = false;
122
+ }
123
+ },
124
+ (error) => {
125
+ console.error(error);
126
+ this.done = true;
127
+ this.push(errorToString(error));
128
+ this.push(null);
129
+ }
130
+ );
131
+ }
132
+
133
+ push(content) {
134
+ if (content === null || content instanceof Buffer) {
135
+ if (this.bufferSize > 0) this.flush();
136
+ return super.push(content);
137
+ }
138
+ this.bufferSize += content.length || content.toString().length;
139
+ this.buffer.push(content);
140
+ if (this.bufferSize > BUFFER_SIZE) {
141
+ return this.flush();
142
+ }
143
+ return true;
144
+ }
145
+
146
+ flush() {
147
+ const pushResult = super.push(this.buffer.join(''));
148
+ this.buffer = [];
149
+ this.bufferSize = 0;
150
+ return pushResult;
151
+ }
152
+
153
+ readIterator(iterator) {
154
+ try {
155
+ // eventually we should be able to just put this around iterator.next()
156
+ let nextString;
157
+ if (iterator.childIterator) {
158
+ // resuming in a child iterator
159
+ return when(this.readIterator(iterator.childIterator), (done) => {
160
+ if (done) {
161
+ iterator.childIterator = null;
162
+ // continue on with the current iterator
163
+ return this.readIterator(iterator);
164
+ }
165
+ });
166
+ }
167
+ do {
168
+ const stepReturn = iterator.next();
169
+ if (stepReturn.done) {
170
+ return true;
171
+ }
172
+ nextString = stepReturn.value;
173
+ if (nextString == null) {
174
+ nextString = 'null';
175
+ } else {
176
+ if (nextString.then) {
177
+ this.flush();
178
+ return Promise.resolve(nextString).then((resolved) => {
179
+ if (resolved && typeof resolved.return === 'function') {
180
+ iterator.childIterator = resolved;
181
+ return this.readIterator(iterator);
182
+ } else if (this.push(resolved + '')) {
183
+ return this.readIterator(iterator);
184
+ } // else return false
185
+ });
186
+ }
187
+ if (typeof nextString.return === 'function') {
188
+ iterator.childIterator = nextString;
189
+ return this.readIterator(iterator);
190
+ }
191
+ }
192
+ } while (this.push(nextString));
193
+ } catch (error) {
194
+ console.error(error);
195
+ this.push(errorToString(error));
196
+ this.push(null);
197
+ return true;
198
+ }
199
+ }
200
+
201
+ _destroy(error, callback) {
202
+ for (const iterator of this.activeIterators) {
203
+ if (error) iterator.throw(error);
204
+ else iterator.return();
205
+ }
206
+ callback();
207
+ }
208
+ }
209
+
210
+ function handleError(error) {
211
+ console.error(error);
212
+ return JSON.stringify(errorToString(error));
213
+ }
214
+
215
+ export function stringify(value) {
216
+ try {
217
+ return JSON.stringify(value) ?? 'null';
218
+ } catch (error) {
219
+ if (error === BIGINT_SERIALIZATION) {
220
+ return jsStringify(value);
221
+ }
222
+ if (error.resolution) {
223
+ return error.resolution.then(() => stringify(value));
224
+ }
225
+ throw error;
226
+ }
227
+ }
228
+
229
+ function jsStringify(value) {
230
+ const type = typeof value;
231
+ if (type === 'object') {
232
+ if (value === null) return 'null';
233
+ if (value.toJSON) value = value.toJSON();
234
+ let str;
235
+ if (Array.isArray(value)) {
236
+ str = '[';
237
+ for (let i = 0; i < value.length; i++) {
238
+ if (i > 0) str += ',';
239
+ // we continue to use jsStringify assuming that if one element has a BigInt, they all do
240
+ str += jsStringify(value[i]);
241
+ }
242
+ return str + ']';
243
+ } else {
244
+ str = '{';
245
+ let first = true;
246
+ for (const key in value) {
247
+ if (first) first = false;
248
+ else str += ',';
249
+ // we assume probably only one element has a BigInt, so we can use stringify for the rest
250
+ str += JSON.stringify(key) + ':' + stringify(value[key]);
251
+ }
252
+ return str + '}';
253
+ }
254
+ } else if (type === 'string') {
255
+ return JSON.stringify(value);
256
+ } else if (type === 'undefined') {
257
+ return 'null';
258
+ }
259
+ return value.toString(); // this handles bigint, number, boolean, undefined, symbol
260
+ }
261
+ const HAS_BIG_NUMBER = /[[,:]\s*-?\d{16,}/;
262
+ export function parse(json) {
263
+ if (!(json?.length > 0)) return null; // empty string or null
264
+ if (typeof json !== 'string') json = json.toString(); // make sure we just do this once
265
+ // we use JSONbig if there is a big number in the JSON, otherwise we use the native JSON parser
266
+ // because JSONbig is much slower (about 4x slower)
267
+ if (HAS_BIG_NUMBER.test(json)) return JSONbigint.parse(json);
268
+ else return JSON.parse(json);
269
+ }
@@ -0,0 +1,13 @@
1
+ /**
2
+ * represents the operation function object used for get operation
3
+ */
4
+ export class OperationFunctionObject {
5
+ operation_function: Function;
6
+ job_operation_function: Function | undefined;
7
+ httpMethod?: string;
8
+
9
+ constructor(operation_function: Function, job_operation_function: Function = undefined) {
10
+ this.operation_function = operation_function;
11
+ this.job_operation_function = job_operation_function;
12
+ }
13
+ }