@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/bin/BinObjects.js +17 -0
  2. package/bin/cliOperations.js +157 -0
  3. package/bin/copyDb.ts +280 -0
  4. package/bin/harper.js +156 -0
  5. package/bin/install.js +15 -0
  6. package/bin/lite.js +5 -0
  7. package/bin/restart.js +201 -0
  8. package/bin/run.js +409 -0
  9. package/bin/status.js +65 -0
  10. package/bin/stop.js +22 -0
  11. package/bin/upgrade.js +134 -0
  12. package/components/Application.ts +646 -0
  13. package/components/ApplicationScope.ts +49 -0
  14. package/components/Component.ts +53 -0
  15. package/components/ComponentV1.ts +342 -0
  16. package/components/DEFAULT_CONFIG.ts +18 -0
  17. package/components/EntryHandler.ts +227 -0
  18. package/components/Logger.ts +14 -0
  19. package/components/OptionsWatcher.ts +354 -0
  20. package/components/PluginModule.ts +6 -0
  21. package/components/Scope.ts +329 -0
  22. package/components/componentLoader.ts +529 -0
  23. package/components/deriveCommonPatternBase.ts +31 -0
  24. package/components/deriveGlobOptions.ts +44 -0
  25. package/components/deriveURLPath.ts +57 -0
  26. package/components/operations.js +658 -0
  27. package/components/operationsValidation.js +246 -0
  28. package/components/packageComponent.ts +39 -0
  29. package/components/requestRestart.ts +26 -0
  30. package/components/resolveBaseURLPath.ts +38 -0
  31. package/components/status/ComponentStatus.ts +110 -0
  32. package/components/status/ComponentStatusRegistry.ts +251 -0
  33. package/components/status/api.ts +153 -0
  34. package/components/status/crossThread.ts +405 -0
  35. package/components/status/errors.ts +152 -0
  36. package/components/status/index.ts +44 -0
  37. package/components/status/internal.ts +65 -0
  38. package/components/status/registry.ts +12 -0
  39. package/components/status/types.ts +96 -0
  40. package/config/RootConfigWatcher.ts +59 -0
  41. package/config/configHelpers.ts +11 -0
  42. package/config/configUtils.js +967 -0
  43. package/config/harperConfigEnvVars.ts +641 -0
  44. package/dataLayer/CreateAttributeObject.js +25 -0
  45. package/dataLayer/CreateTableObject.js +11 -0
  46. package/dataLayer/DataLayerObjects.js +43 -0
  47. package/dataLayer/DeleteBeforeObject.js +22 -0
  48. package/dataLayer/DeleteObject.js +25 -0
  49. package/dataLayer/DropAttributeObject.js +11 -0
  50. package/dataLayer/GetBackupObject.js +22 -0
  51. package/dataLayer/InsertObject.js +24 -0
  52. package/dataLayer/ReadAuditLogObject.js +24 -0
  53. package/dataLayer/SQLSearch.js +1335 -0
  54. package/dataLayer/SearchByConditionsObject.js +61 -0
  55. package/dataLayer/SearchByHashObject.js +21 -0
  56. package/dataLayer/SearchObject.js +45 -0
  57. package/dataLayer/SqlSearchObject.js +14 -0
  58. package/dataLayer/UpdateObject.js +23 -0
  59. package/dataLayer/UpsertObject.js +23 -0
  60. package/dataLayer/bulkLoad.js +813 -0
  61. package/dataLayer/dataObjects/BulkLoadObjects.js +27 -0
  62. package/dataLayer/dataObjects/UpsertObject.js +23 -0
  63. package/dataLayer/delete.js +164 -0
  64. package/dataLayer/export.js +381 -0
  65. package/dataLayer/getBackup.js +40 -0
  66. package/dataLayer/harperBridge/BridgeMethods.js +81 -0
  67. package/dataLayer/harperBridge/ResourceBridge.ts +633 -0
  68. package/dataLayer/harperBridge/bridgeUtility/insertUpdateReturnObj.js +28 -0
  69. package/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +88 -0
  70. package/dataLayer/harperBridge/harperBridge.js +21 -0
  71. package/dataLayer/harperBridge/lmdbBridge/LMDBBridge.js +119 -0
  72. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +19 -0
  73. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +112 -0
  74. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +67 -0
  75. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +31 -0
  76. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +94 -0
  77. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +98 -0
  78. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +89 -0
  79. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +109 -0
  80. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +107 -0
  81. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +137 -0
  82. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +35 -0
  83. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +111 -0
  84. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +28 -0
  85. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +29 -0
  86. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +207 -0
  87. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +156 -0
  88. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +21 -0
  89. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +30 -0
  90. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +19 -0
  91. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +64 -0
  92. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +70 -0
  93. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +22 -0
  94. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +23 -0
  95. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +22 -0
  96. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBTransactionObject.js +23 -0
  97. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +24 -0
  98. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +24 -0
  99. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +25 -0
  100. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +21 -0
  101. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +157 -0
  102. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +94 -0
  103. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +39 -0
  104. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +34 -0
  105. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +100 -0
  106. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +371 -0
  107. package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +109 -0
  108. package/dataLayer/hdbInfoController.js +254 -0
  109. package/dataLayer/insert.js +266 -0
  110. package/dataLayer/readAuditLog.js +59 -0
  111. package/dataLayer/schema.js +366 -0
  112. package/dataLayer/schemaDescribe.js +289 -0
  113. package/dataLayer/search.js +60 -0
  114. package/dataLayer/transaction.js +17 -0
  115. package/dataLayer/update.js +124 -0
  116. package/dist/components/Logger.d.ts +12 -0
  117. package/dist/{resources/ResourceInterfaceV2.js → components/Logger.js} +1 -1
  118. package/dist/components/Logger.js.map +1 -0
  119. package/dist/components/Scope.d.ts +14 -4
  120. package/dist/components/Scope.js +18 -10
  121. package/dist/components/Scope.js.map +1 -1
  122. package/dist/components/componentLoader.js +17 -10
  123. package/dist/components/componentLoader.js.map +1 -1
  124. package/dist/components/operations.js +2 -2
  125. package/dist/components/operations.js.map +1 -1
  126. package/dist/config/configUtils.d.ts +1 -1
  127. package/dist/config/configUtils.js +1 -1
  128. package/dist/config/configUtils.js.map +1 -1
  129. package/dist/dataLayer/CreateTableObject.d.ts +2 -2
  130. package/dist/dataLayer/CreateTableObject.js +2 -2
  131. package/dist/dataLayer/CreateTableObject.js.map +1 -1
  132. package/dist/dataLayer/delete.d.ts +1 -1
  133. package/dist/dataLayer/schema.js +6 -5
  134. package/dist/dataLayer/schema.js.map +1 -1
  135. package/dist/dataLayer/schemaDescribe.js +1 -1
  136. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  137. package/dist/index.d.ts +1 -1
  138. package/dist/index.js +2 -0
  139. package/dist/index.js.map +1 -1
  140. package/dist/resources/DatabaseTransaction.d.ts +1 -1
  141. package/dist/resources/IterableEventQueue.d.ts +1 -1
  142. package/dist/resources/LMDBTransaction.d.ts +5 -1
  143. package/dist/resources/Resource.d.ts +1 -1
  144. package/dist/resources/ResourceInterface.d.ts +1 -1
  145. package/dist/resources/RocksIndexStore.d.ts +3 -3
  146. package/dist/resources/RocksTransactionLogStore.d.ts +6 -3
  147. package/dist/resources/Table.d.ts +15 -6
  148. package/dist/resources/Table.js +12 -4
  149. package/dist/resources/Table.js.map +1 -1
  150. package/dist/resources/analytics/read.js +32 -22
  151. package/dist/resources/analytics/read.js.map +1 -1
  152. package/dist/resources/analytics/write.js +3 -6
  153. package/dist/resources/analytics/write.js.map +1 -1
  154. package/dist/resources/auditStore.d.ts +3 -3
  155. package/dist/resources/blob.d.ts +25 -2
  156. package/dist/resources/databases.d.ts +12 -2
  157. package/dist/resources/databases.js +22 -19
  158. package/dist/resources/databases.js.map +1 -1
  159. package/dist/resources/search.js +11 -5
  160. package/dist/resources/search.js.map +1 -1
  161. package/dist/resources/transaction.d.ts +2 -1
  162. package/dist/security/auth.js +1 -1
  163. package/dist/security/auth.js.map +1 -1
  164. package/dist/security/cryptoHash.d.ts +2 -2
  165. package/dist/security/jsLoader.js +265 -73
  166. package/dist/security/jsLoader.js.map +1 -1
  167. package/dist/security/keys.js +11 -12
  168. package/dist/security/keys.js.map +1 -1
  169. package/dist/security/user.js +3 -3
  170. package/dist/security/user.js.map +1 -1
  171. package/dist/server/REST.js +16 -2
  172. package/dist/server/REST.js.map +1 -1
  173. package/dist/server/Server.d.ts +2 -1
  174. package/dist/server/Server.js.map +1 -1
  175. package/dist/server/fastifyRoutes/plugins/hdbCore.d.ts +6 -1
  176. package/dist/server/fastifyRoutes.js +2 -0
  177. package/dist/server/fastifyRoutes.js.map +1 -1
  178. package/dist/server/http.js +12 -6
  179. package/dist/server/http.js.map +1 -1
  180. package/dist/server/jobs/JobObject.d.ts +3 -3
  181. package/dist/server/loadRootComponents.js +1 -0
  182. package/dist/server/loadRootComponents.js.map +1 -1
  183. package/dist/server/operationsServer.js +3 -1
  184. package/dist/server/operationsServer.js.map +1 -1
  185. package/dist/server/serverHelpers/JSONStream.d.ts +3 -3
  186. package/dist/server/serverHelpers/Request.d.ts +5 -5
  187. package/dist/server/serverHelpers/requestTimePlugin.d.ts +1 -1
  188. package/dist/server/threads/manageThreads.d.ts +2 -2
  189. package/dist/server/threads/manageThreads.js +52 -35
  190. package/dist/server/threads/manageThreads.js.map +1 -1
  191. package/dist/server/threads/socketRouter.d.ts +1 -1
  192. package/dist/sqlTranslator/deleteTranslator.d.ts +1 -1
  193. package/dist/utility/AWS/AWSConnector.d.ts +3 -2
  194. package/dist/utility/common_utils.d.ts +3 -3
  195. package/dist/utility/environment/systemInformation.d.ts +1 -0
  196. package/dist/utility/functions/date/dateFunctions.d.ts +11 -11
  197. package/dist/utility/globalSchema.d.ts +1 -1
  198. package/dist/utility/hdbTerms.d.ts +3 -0
  199. package/dist/utility/hdbTerms.js +3 -0
  200. package/dist/utility/hdbTerms.js.map +1 -1
  201. package/dist/utility/installation.d.ts +2 -4
  202. package/dist/utility/installation.js.map +1 -1
  203. package/dist/utility/lmdb/commonUtility.d.ts +2 -1
  204. package/dist/utility/lmdb/commonUtility.js +20 -13
  205. package/dist/utility/lmdb/commonUtility.js.map +1 -1
  206. package/dist/utility/lmdb/deleteUtility.d.ts +1 -0
  207. package/dist/utility/lmdb/environmentUtility.d.ts +1 -0
  208. package/dist/utility/lmdb/searchUtility.d.ts +2 -1
  209. package/dist/utility/lmdb/writeUtility.d.ts +1 -0
  210. package/dist/utility/logging/harper_logger.d.ts +6 -6
  211. package/dist/utility/processManagement/processManagement.d.ts +1 -1
  212. package/dist/utility/processManagement/servicesConfig.d.ts +12 -6
  213. package/dist/validation/common_validators.d.ts +4 -3
  214. package/dist/validation/configValidator.d.ts +3 -2
  215. package/index.d.ts +56 -0
  216. package/index.js +41 -0
  217. package/json/systemSchema.json +373 -0
  218. package/launchServiceScripts/launchHarperDB.js +3 -0
  219. package/launchServiceScripts/utility/checkNodeVersion.js +15 -0
  220. package/package.json +35 -16
  221. package/resources/DatabaseTransaction.ts +378 -0
  222. package/resources/ErrorResource.ts +57 -0
  223. package/resources/IterableEventQueue.ts +94 -0
  224. package/resources/LMDBTransaction.ts +349 -0
  225. package/resources/RecordEncoder.ts +702 -0
  226. package/resources/RequestTarget.ts +134 -0
  227. package/resources/Resource.ts +789 -0
  228. package/resources/ResourceInterface.ts +221 -0
  229. package/resources/Resources.ts +162 -0
  230. package/resources/RocksIndexStore.ts +70 -0
  231. package/resources/RocksTransactionLogStore.ts +352 -0
  232. package/resources/Table.ts +4531 -0
  233. package/resources/analytics/hostnames.ts +72 -0
  234. package/resources/analytics/metadata.ts +10 -0
  235. package/resources/analytics/read.ts +252 -0
  236. package/resources/analytics/write.ts +803 -0
  237. package/resources/auditStore.ts +556 -0
  238. package/resources/blob.ts +1268 -0
  239. package/resources/crdt.ts +125 -0
  240. package/resources/dataLoader.ts +527 -0
  241. package/resources/databases.ts +1290 -0
  242. package/resources/graphql.ts +221 -0
  243. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +638 -0
  244. package/resources/indexes/customIndexes.ts +7 -0
  245. package/resources/indexes/vector.ts +38 -0
  246. package/resources/jsResource.ts +86 -0
  247. package/resources/loadEnv.ts +22 -0
  248. package/resources/login.ts +18 -0
  249. package/resources/openApi.ts +409 -0
  250. package/resources/registrationDeprecated.ts +8 -0
  251. package/resources/replayLogs.ts +136 -0
  252. package/resources/roles.ts +98 -0
  253. package/resources/search.ts +1301 -0
  254. package/resources/tracked.ts +584 -0
  255. package/resources/transaction.ts +89 -0
  256. package/resources/transactionBroadcast.ts +258 -0
  257. package/security/auth.ts +376 -0
  258. package/security/certificateVerification/certificateVerificationSource.ts +84 -0
  259. package/security/certificateVerification/configValidation.ts +107 -0
  260. package/security/certificateVerification/crlVerification.ts +623 -0
  261. package/security/certificateVerification/index.ts +121 -0
  262. package/security/certificateVerification/ocspVerification.ts +148 -0
  263. package/security/certificateVerification/pkijs-ed25519-patch.ts +188 -0
  264. package/security/certificateVerification/types.ts +128 -0
  265. package/security/certificateVerification/verificationConfig.ts +138 -0
  266. package/security/certificateVerification/verificationUtils.ts +447 -0
  267. package/security/cryptoHash.js +42 -0
  268. package/security/data_objects/PermissionAttributeResponseObject.js +15 -0
  269. package/security/data_objects/PermissionResponseObject.js +115 -0
  270. package/security/data_objects/PermissionTableResponseObject.js +20 -0
  271. package/security/fastifyAuth.js +169 -0
  272. package/security/impersonation.ts +160 -0
  273. package/security/jsLoader.ts +733 -0
  274. package/security/keys.js +948 -0
  275. package/security/permissionsTranslator.js +300 -0
  276. package/security/role.js +218 -0
  277. package/security/tokenAuthentication.ts +228 -0
  278. package/security/user.ts +449 -0
  279. package/server/DurableSubscriptionsSession.ts +503 -0
  280. package/server/REST.ts +407 -0
  281. package/server/Server.ts +89 -0
  282. package/server/fastifyRoutes/helpers/getCORSOptions.js +36 -0
  283. package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +15 -0
  284. package/server/fastifyRoutes/helpers/getServerOptions.js +33 -0
  285. package/server/fastifyRoutes/plugins/hdbCore.js +39 -0
  286. package/server/fastifyRoutes.ts +205 -0
  287. package/server/graphqlQuerying.ts +700 -0
  288. package/server/http.ts +640 -0
  289. package/server/itc/serverHandlers.js +161 -0
  290. package/server/itc/utility/ITCEventObject.js +10 -0
  291. package/server/jobs/JobObject.js +24 -0
  292. package/server/jobs/jobProcess.js +69 -0
  293. package/server/jobs/jobRunner.js +162 -0
  294. package/server/jobs/jobs.js +304 -0
  295. package/server/loadRootComponents.js +44 -0
  296. package/server/mqtt.ts +485 -0
  297. package/server/nodeName.ts +75 -0
  298. package/server/operationsServer.ts +313 -0
  299. package/server/serverHelpers/Headers.ts +108 -0
  300. package/server/serverHelpers/JSONStream.ts +269 -0
  301. package/server/serverHelpers/OperationFunctionObject.ts +13 -0
  302. package/server/serverHelpers/Request.ts +158 -0
  303. package/server/serverHelpers/contentTypes.ts +637 -0
  304. package/server/serverHelpers/requestTimePlugin.js +57 -0
  305. package/server/serverHelpers/serverHandlers.js +148 -0
  306. package/server/serverHelpers/serverUtilities.ts +473 -0
  307. package/server/serverRegistry.ts +8 -0
  308. package/server/static.ts +187 -0
  309. package/server/status/definitions.ts +37 -0
  310. package/server/status/index.ts +125 -0
  311. package/server/storageReclamation.ts +93 -0
  312. package/server/threads/itc.js +89 -0
  313. package/server/threads/manageThreads.js +596 -0
  314. package/server/threads/socketRouter.ts +360 -0
  315. package/server/threads/threadServer.js +279 -0
  316. package/server/throttle.ts +73 -0
  317. package/sqlTranslator/SelectValidator.js +330 -0
  318. package/sqlTranslator/alasqlFunctionImporter.js +62 -0
  319. package/sqlTranslator/deleteTranslator.js +67 -0
  320. package/sqlTranslator/index.js +242 -0
  321. package/sqlTranslator/sql_statement_bucket.js +472 -0
  322. package/static/defaultConfig.yaml +3 -0
  323. package/studio/web/HDBDogOnly.svg +78 -0
  324. package/studio/web/assets/PPRadioGrotesk-Bold-DDaUYG8E.woff +0 -0
  325. package/studio/web/assets/fa-brands-400-CEJbCg16.woff +0 -0
  326. package/studio/web/assets/fa-brands-400-CSYNqBb_.ttf +0 -0
  327. package/studio/web/assets/fa-brands-400-DnkPfk3o.eot +0 -0
  328. package/studio/web/assets/fa-brands-400-UxlILjvJ.woff2 +0 -0
  329. package/studio/web/assets/fa-brands-400-cH1MgKbP.svg +3717 -0
  330. package/studio/web/assets/fa-regular-400-BhTwtT8w.eot +0 -0
  331. package/studio/web/assets/fa-regular-400-D1vz6WBx.ttf +0 -0
  332. package/studio/web/assets/fa-regular-400-DFnMcJPd.woff +0 -0
  333. package/studio/web/assets/fa-regular-400-DGzu1beS.woff2 +0 -0
  334. package/studio/web/assets/fa-regular-400-gwj8Pxq-.svg +801 -0
  335. package/studio/web/assets/fa-solid-900-B4ZZ7kfP.svg +5034 -0
  336. package/studio/web/assets/fa-solid-900-B6Axprfb.eot +0 -0
  337. package/studio/web/assets/fa-solid-900-BUswJgRo.woff2 +0 -0
  338. package/studio/web/assets/fa-solid-900-DOXgCApm.woff +0 -0
  339. package/studio/web/assets/fa-solid-900-mxuxnBEa.ttf +0 -0
  340. package/studio/web/assets/index-C1G-Jo6n.js +37 -0
  341. package/studio/web/assets/index-C1G-Jo6n.js.map +1 -0
  342. package/studio/web/assets/index-D-CahN0-.js +2 -0
  343. package/studio/web/assets/index-D-CahN0-.js.map +1 -0
  344. package/studio/web/assets/index-DxlZI0PX.js +235 -0
  345. package/studio/web/assets/index-DxlZI0PX.js.map +1 -0
  346. package/studio/web/assets/index-Y2g_iFpU.css +1 -0
  347. package/studio/web/assets/index-jiPwkrsB.css +1 -0
  348. package/studio/web/assets/index.lazy-BUXDDqq9.js +266 -0
  349. package/studio/web/assets/index.lazy-BUXDDqq9.js.map +1 -0
  350. package/studio/web/assets/profiler-CU93QiSW.js +2 -0
  351. package/studio/web/assets/profiler-CU93QiSW.js.map +1 -0
  352. package/studio/web/assets/react-redux-B8k9Ep7e.js +6 -0
  353. package/studio/web/assets/react-redux-B8k9Ep7e.js.map +1 -0
  354. package/studio/web/assets/startRecording-DFeBXGk6.js +3 -0
  355. package/studio/web/assets/startRecording-DFeBXGk6.js.map +1 -0
  356. package/studio/web/fabric-signup-background.webp +0 -0
  357. package/studio/web/fabric-signup-text.png +0 -0
  358. package/studio/web/favicon_purple.png +0 -0
  359. package/studio/web/github-icon.svg +15 -0
  360. package/studio/web/harper-fabric_black.png +0 -0
  361. package/studio/web/harper-fabric_white.png +0 -0
  362. package/studio/web/harper-studio_white.png +0 -0
  363. package/studio/web/index.html +16 -0
  364. package/studio/web/running.css +148 -0
  365. package/studio/web/running.html +147 -0
  366. package/studio/web/running.js +111 -0
  367. package/upgrade/UpgradeObjects.js +13 -0
  368. package/upgrade/directives/directivesController.js +90 -0
  369. package/upgrade/directivesManager.js +139 -0
  370. package/upgrade/upgradePrompt.js +124 -0
  371. package/upgrade/upgradeUtilities.js +28 -0
  372. package/utility/AWS/AWSConnector.js +29 -0
  373. package/utility/OperationFunctionCaller.js +63 -0
  374. package/utility/assignCmdEnvVariables.js +62 -0
  375. package/utility/common_utils.js +867 -0
  376. package/utility/environment/environmentManager.js +208 -0
  377. package/utility/environment/systemInformation.js +355 -0
  378. package/utility/errors/commonErrors.js +267 -0
  379. package/utility/errors/hdbError.js +146 -0
  380. package/utility/functions/date/dateFunctions.js +65 -0
  381. package/utility/functions/geo.js +355 -0
  382. package/utility/functions/sql/alaSQLExtension.js +104 -0
  383. package/utility/globalSchema.js +35 -0
  384. package/utility/hdbTerms.ts +819 -0
  385. package/utility/install/checkJWTTokensExist.js +62 -0
  386. package/utility/install/harperdb.conf +15 -0
  387. package/utility/install/harperdb.service +14 -0
  388. package/utility/install/installer.js +635 -0
  389. package/utility/installation.ts +30 -0
  390. package/utility/lmdb/DBIDefinition.js +20 -0
  391. package/utility/lmdb/DeleteRecordsResponseObject.js +25 -0
  392. package/utility/lmdb/InsertRecordsResponseObject.js +22 -0
  393. package/utility/lmdb/OpenDBIObject.js +31 -0
  394. package/utility/lmdb/OpenEnvironmentObject.js +41 -0
  395. package/utility/lmdb/UpdateRecordsResponseObject.js +25 -0
  396. package/utility/lmdb/UpsertRecordsResponseObject.js +22 -0
  397. package/utility/lmdb/cleanLMDBMap.js +65 -0
  398. package/utility/lmdb/commonUtility.js +130 -0
  399. package/utility/lmdb/deleteUtility.js +128 -0
  400. package/utility/lmdb/environmentUtility.js +477 -0
  401. package/utility/lmdb/searchCursorFunctions.js +187 -0
  402. package/utility/lmdb/searchUtility.js +918 -0
  403. package/utility/lmdb/terms.js +57 -0
  404. package/utility/lmdb/writeUtility.js +407 -0
  405. package/utility/logging/harper_logger.js +876 -0
  406. package/utility/logging/logRotator.js +157 -0
  407. package/utility/logging/logger.ts +24 -0
  408. package/utility/logging/readLog.js +355 -0
  409. package/utility/logging/transactionLog.js +57 -0
  410. package/utility/mount_hdb.js +59 -0
  411. package/utility/npmUtilities.js +102 -0
  412. package/utility/operationPermissions.ts +112 -0
  413. package/utility/operation_authorization.js +836 -0
  414. package/utility/packageUtils.js +55 -0
  415. package/utility/password.ts +99 -0
  416. package/utility/processManagement/processManagement.js +187 -0
  417. package/utility/processManagement/servicesConfig.js +56 -0
  418. package/utility/scripts/restartHdb.js +24 -0
  419. package/utility/scripts/user_data.sh +13 -0
  420. package/utility/signalling.js +36 -0
  421. package/utility/terms/certificates.js +81 -0
  422. package/utility/when.ts +20 -0
  423. package/validation/bulkDeleteValidator.js +24 -0
  424. package/validation/check_permissions.js +19 -0
  425. package/validation/common_validators.js +95 -0
  426. package/validation/configValidator.js +331 -0
  427. package/validation/deleteValidator.js +15 -0
  428. package/validation/fileLoadValidator.js +153 -0
  429. package/validation/insertValidator.js +40 -0
  430. package/validation/installValidator.js +37 -0
  431. package/validation/readLogValidator.js +64 -0
  432. package/validation/role_validation.js +320 -0
  433. package/validation/schemaMetadataValidator.js +42 -0
  434. package/validation/searchValidator.js +166 -0
  435. package/validation/statusValidator.ts +66 -0
  436. package/validation/transactionLogValidator.js +33 -0
  437. package/validation/user_validation.js +55 -0
  438. package/validation/validationWrapper.js +105 -0
  439. package/dist/resources/ResourceInterfaceV2.d.ts +0 -21
  440. package/dist/resources/ResourceInterfaceV2.js.map +0 -1
  441. package/dist/resources/ResourceV2.d.ts +0 -30
  442. package/dist/resources/ResourceV2.js +0 -27
  443. package/dist/resources/ResourceV2.js.map +0 -1
  444. package/dist/resources/analytics/profile.d.ts +0 -2
  445. package/dist/resources/analytics/profile.js +0 -144
  446. package/dist/resources/analytics/profile.js.map +0 -1
@@ -0,0 +1,1335 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * SQLSearch.js
5
+ * This class is used to receive the alasql generated AST from a SQL SELECT,
6
+ * process and return results by passing the raw values into the alasql SQL parser
7
+ */
8
+
9
+ const _ = require('lodash');
10
+ const alasql = require('alasql');
11
+ alasql.options.cache = false;
12
+ const alasqlFunctionImporter = require('../sqlTranslator/alasqlFunctionImporter.js');
13
+ const clone = require('clone');
14
+ const RecursiveIterator = require('recursive-iterator');
15
+ const log = require('../utility/logging/harper_logger.js');
16
+ const commonUtils = require('../utility/common_utils.js');
17
+ const harperBridge = require('./harperBridge/harperBridge.js');
18
+ const hdbTerms = require('../utility/hdbTerms.ts');
19
+ const { hdbErrors } = require('../utility/errors/hdbError.js');
20
+ const { getDatabases } = require('../resources/databases.ts');
21
+
22
+ const WHERE_CLAUSE_IS_NULL = 'IS NULL';
23
+ const SEARCH_ERROR_MSG = 'There was a problem performing this search. Please check the logs and try again.';
24
+
25
+ //here we call to define and import custom functions to alasql
26
+ alasqlFunctionImporter(alasql);
27
+
28
+ class SQLSearch {
29
+ /**
30
+ * Constructor for FileSearch class
31
+ *
32
+ * @param statement - the AST for the SQL SELECT to process
33
+ * @param attributes - all attributes that are part of the schema for the tables in select
34
+ */
35
+ constructor(statement, attributes) {
36
+ if (commonUtils.isEmpty(statement)) {
37
+ log.error('AST statement for SQL select process cannot be empty');
38
+ throw 'statement cannot be null';
39
+ }
40
+
41
+ this.statement = statement;
42
+ //this is every attribute that we need to pull data for
43
+ this.columns = {};
44
+
45
+ this.all_table_attributes = attributes;
46
+
47
+ this.fetch_attributes = [];
48
+ this.exact_search_values = {};
49
+ this.comparator_search_values = {};
50
+ this.tables = [];
51
+
52
+ //holds the data to be evaluated by the sql processor
53
+ this.data = {};
54
+
55
+ this.has_aggregator = false;
56
+ this.has_ordinal = false;
57
+ this.has_outer_join = false;
58
+
59
+ this._getColumns();
60
+ this._getTables();
61
+ this._conditionsToFetchAttributeValues();
62
+ this._setAliasesForColumns();
63
+ commonUtils.backtickASTSchemaItems(this.statement);
64
+ }
65
+
66
+ /**
67
+ * Starting point function to execute the search
68
+ * @returns {Promise<results|finalResults[]|Array>}
69
+ */
70
+ async search() {
71
+ let searchResults = undefined;
72
+ try {
73
+ let emptySqlResults = await this._checkEmptySQL();
74
+ if (!commonUtils.isEmptyOrZeroLength(emptySqlResults)) {
75
+ log.trace('No results returned from checkEmptySQL SQLSearch method.');
76
+ return emptySqlResults;
77
+ }
78
+ } catch (err) {
79
+ log.error('Error thrown from checkEmptySQL in SQLSearch class method search.');
80
+ log.error(err);
81
+ throw new Error(SEARCH_ERROR_MSG);
82
+ }
83
+
84
+ try {
85
+ // Search for fetch attribute values and consolidate them into this.data[table].__mergedData property
86
+ const simpleQueryResults = await this._getFetchAttributeValues();
87
+ if (simpleQueryResults) {
88
+ return simpleQueryResults;
89
+ }
90
+ } catch (err) {
91
+ log.error('Error thrown from getFetchAttributeValues in SQLSearch class method search.');
92
+ log.error(err);
93
+ throw new Error(SEARCH_ERROR_MSG);
94
+ }
95
+
96
+ // In the instance of null data this.data would not have schema/table defined or created as there is no data backing up what would sit in data.
97
+ if (Object.keys(this.data).length === 0) {
98
+ log.trace('SQLSearch class field: "data" is empty.');
99
+ return [];
100
+ }
101
+
102
+ let joinResults;
103
+ try {
104
+ // Consolidate initial data required for first pass of sql join - narrows list of hash ids for second pass to collect all data resulting from sql request
105
+ joinResults = await this._processJoins();
106
+ } catch (err) {
107
+ log.error('Error thrown from processJoins in SQLSearch class method search.');
108
+ log.error(err);
109
+ throw new Error(SEARCH_ERROR_MSG);
110
+ }
111
+
112
+ try {
113
+ // Decide the most efficient way to make the second/final pass for collecting all additional data needed for sql request
114
+ await this._getFinalAttributeData(joinResults.existing_attributes, joinResults.joined_length);
115
+ } catch (err) {
116
+ log.error('Error thrown from getFinalAttributeData in SQLSearch class method search.');
117
+ log.error(err);
118
+ throw new Error(SEARCH_ERROR_MSG);
119
+ }
120
+
121
+ try {
122
+ searchResults = await this._finalSQL();
123
+ return searchResults;
124
+ } catch (err) {
125
+ log.error('Error thrown from finalSQL in SQLSearch class method search.');
126
+ log.error(err);
127
+ throw new Error(SEARCH_ERROR_MSG);
128
+ }
129
+ }
130
+
131
+ /**
132
+ * Gets the raw column from each section of the statement and puts them in a map
133
+ * @private
134
+ */
135
+ _getColumns() {
136
+ let iterator = new RecursiveIterator(this.statement);
137
+ for (let { node, path } of iterator) {
138
+ if (node && node.columnid) {
139
+ if (!this.columns[path[0]]) {
140
+ this.columns[path[0]] = [];
141
+ }
142
+ this.columns[path[0]].push(clone(node));
143
+ }
144
+ }
145
+ }
146
+
147
+ /**
148
+ * Extracts the table info from the attributes
149
+ * @private
150
+ */
151
+ _getTables() {
152
+ let tbls = [];
153
+ this.all_table_attributes.forEach((attribute) => {
154
+ tbls.push(attribute.table);
155
+ });
156
+
157
+ this.tables = _.uniqBy(tbls, (tbl) => [tbl.databaseid, tbl.tableid, tbl.as].join());
158
+ this.tables.forEach((table) => {
159
+ const schemaTable = `${table.databaseid}_${table.as ? table.as : table.tableid}`;
160
+ this.data[schemaTable] = {};
161
+ this.data[schemaTable].__hashName = getDatabases()[table.databaseid][table.tableid].primaryKey;
162
+ this.data[schemaTable].__mergedData = {};
163
+ this.data[schemaTable].__mergedAttributes = [];
164
+ this.data[schemaTable].__mergedAttrMap = {};
165
+ });
166
+ }
167
+
168
+ /**
169
+ * Iterates the where AST with the goal of finding exact values to match directly on. Matching on values allows us to skip parsing an index
170
+ * If a condition has a columnid, and op of '=' or 'IN' and only is comparing to raw values we will limit the column to the raw value match.
171
+ * If a column condition does not have these criteria or another condition for the same column does not adhere to the criteria then we ignore it for exact matching.
172
+ * @private
173
+ */
174
+ _conditionsToFetchAttributeValues() {
175
+ //TODO - CORE-1095 - update how WHERE clause value that include escaped characters is used to do initial
176
+ // searchByValue query - this value is set to this.exact_search_values in this method
177
+ if (commonUtils.isEmpty(this.statement.where)) {
178
+ log.trace('AST "where" statement is empty.');
179
+ return;
180
+ }
181
+
182
+ //if there is an OR in the where clause we will not perform exact match search on attributes as it ends up excluding values incorrectly.
183
+ let totalIgnore = false;
184
+
185
+ //check for OR statement (see not above) and update numeric hash values set as strings in the statement to evaluate the table data
186
+ // correctly as numbers in alasql which evaluates based on data types
187
+ for (let { node } of new RecursiveIterator(this.statement.where)) {
188
+ if (node && node.op && node.op === 'OR') {
189
+ totalIgnore = true;
190
+ }
191
+ if (!commonUtils.isEmpty(node) && node.right) {
192
+ if (commonUtils.isNotEmptyAndHasValue(node.right.value)) {
193
+ const whereVal = commonUtils.autoCast(node.right.value);
194
+ if ([true, false].indexOf(whereVal) >= 0) {
195
+ node.right = new alasql.yy.LogicValue({ value: whereVal });
196
+ }
197
+ } else if (Array.isArray(node.right)) {
198
+ node.right.forEach((col, i) => {
199
+ const whereVal = commonUtils.autoCast(col.value);
200
+ if ([true, false].indexOf(whereVal) >= 0) {
201
+ node.right[i] = new alasql.yy.LogicValue({ value: whereVal });
202
+ } else if (
203
+ col instanceof alasql.yy.StringValue &&
204
+ commonUtils.autoCasterIsNumberCheck(whereVal.toString())
205
+ ) {
206
+ node.right[i] = new alasql.yy.NumValue({ value: whereVal });
207
+ }
208
+ });
209
+ }
210
+ }
211
+ }
212
+
213
+ if (totalIgnore) {
214
+ log.trace('Where clause contains "OR", exact match search not performed on attributes.');
215
+ return;
216
+ }
217
+
218
+ for (let { node } of new RecursiveIterator(this.statement.where)) {
219
+ if (node && node.left && node.right && (node.left.columnid || node.right.value) && node.op) {
220
+ let values = new Set();
221
+ let column = node.left.columnid ? node.left : node.right;
222
+ let foundColumn = this._findColumn(column);
223
+ if (!foundColumn) {
224
+ continue;
225
+ }
226
+ //Specifically a slash delimited string for consistency
227
+ let attributeKey = [foundColumn.table.databaseid, foundColumn.table.tableid, foundColumn.attribute].join('/');
228
+
229
+ // Check for value range search first
230
+ if (!commonUtils.isEmpty(hdbTerms.VALUE_SEARCH_COMPARATORS_REVERSE_LOOKUP[node.op])) {
231
+ if (commonUtils.isEmpty(this.comparator_search_values[attributeKey])) {
232
+ this.comparator_search_values[attributeKey] = {
233
+ ignore: false,
234
+ comparators: [],
235
+ };
236
+ }
237
+
238
+ if (!this.comparator_search_values[attributeKey].ignore) {
239
+ if (
240
+ commonUtils.isEmptyOrZeroLength(node.left.columnid) ||
241
+ commonUtils.isEmptyOrZeroLength(node.right.value)
242
+ ) {
243
+ this.comparator_search_values[attributeKey].ignore = true;
244
+ this.comparator_search_values[attributeKey].comparators = [];
245
+ continue;
246
+ }
247
+
248
+ this.comparator_search_values[attributeKey].comparators.push({
249
+ attribute: node.left.columnid,
250
+ operation: node.op,
251
+ value: node.right.value,
252
+ });
253
+ }
254
+ continue;
255
+ }
256
+
257
+ if (commonUtils.isEmpty(this.exact_search_values[attributeKey])) {
258
+ this.exact_search_values[attributeKey] = {
259
+ ignore: false,
260
+ values: new Set(),
261
+ };
262
+ }
263
+
264
+ if (!this.exact_search_values[attributeKey].ignore) {
265
+ let ignore = false;
266
+
267
+ switch (node.op) {
268
+ case '=':
269
+ if (!commonUtils.isEmpty(node.right.value) || !commonUtils.isEmpty(node.left.value)) {
270
+ values.add(!commonUtils.isEmpty(node.right.value) ? node.right.value : node.left.value);
271
+ } else {
272
+ ignore = true;
273
+ }
274
+ break;
275
+ case 'IN':
276
+ let inArray = Array.isArray(node.right) ? node.right : node.left;
277
+
278
+ for (let x = 0; x < inArray.length; x++) {
279
+ if (inArray[x].value) {
280
+ values.add(inArray[x].value);
281
+ } else {
282
+ ignore = true;
283
+ break;
284
+ }
285
+ }
286
+ break;
287
+ default:
288
+ ignore = true;
289
+ break;
290
+ }
291
+ this.exact_search_values[attributeKey].ignore = ignore;
292
+
293
+ //if we are ignoring the column for exact matches we clear out it's values to match later
294
+ if (ignore) {
295
+ this.exact_search_values[attributeKey].values = new Set();
296
+ } else {
297
+ this.exact_search_values[attributeKey].values = new Set([
298
+ ...this.exact_search_values[attributeKey].values,
299
+ ...values,
300
+ ]);
301
+ }
302
+ }
303
+ }
304
+ }
305
+ }
306
+
307
+ /**
308
+ * Iterates the columns in the AST and assigns an alias to each column if one does not exist. This is necessary to ensure
309
+ * that the final result returned from alasql include the correct column header
310
+ * @private
311
+ */
312
+ _setAliasesForColumns() {
313
+ //this scenario is reached by doing a select with only calculations and, therefore, this step can be skipped.
314
+ if (
315
+ commonUtils.isEmptyOrZeroLength(this.all_table_attributes) &&
316
+ commonUtils.isEmptyOrZeroLength(this.statement.from) &&
317
+ commonUtils.isEmptyOrZeroLength(this.columns.columns)
318
+ ) {
319
+ return;
320
+ }
321
+ let wildcardIndexes = [];
322
+ let dupAttrCount = {};
323
+ this.statement.columns.forEach((col, index) => {
324
+ if (col.columnid === '*') {
325
+ wildcardIndexes.push(index);
326
+ return;
327
+ }
328
+
329
+ if (col.aggregatorid) {
330
+ this.has_aggregator = true;
331
+ }
332
+
333
+ if (!col.aggregatorid && !col.funcid) {
334
+ col.as_orig = col.as ? col.as : col.columnid;
335
+ if (this.statement.joins) {
336
+ if (dupAttrCount[col.as_orig] >= 0) {
337
+ const attrCount = dupAttrCount[col.as_orig] + 1;
338
+ col.as = `[${col.as_orig + attrCount}]`;
339
+ dupAttrCount[col.as_orig] = attrCount;
340
+ } else {
341
+ col.as = `[${col.as_orig}]`;
342
+ dupAttrCount[col.as_orig] = 0;
343
+ }
344
+ } else {
345
+ col.as = `[${col.as_orig}]`;
346
+ }
347
+ }
348
+
349
+ if (!col.aggregatorid && col.funcid && col.args) {
350
+ col.as_orig = col.as ? col.as : col.toString().replace(/'/g, '"');
351
+ col.as = `[${col.as_orig}]`;
352
+ }
353
+
354
+ if (col.aggregatorid && col.expression.columnid !== '*') {
355
+ col.as_orig = col.as
356
+ ? col.as
357
+ : col.expression.tableid
358
+ ? `${col.aggregatorid}(${col.expression.tableid}.${col.expression.columnid})`
359
+ : `${col.aggregatorid}(${col.expression.columnid})`;
360
+ col.as = `[${col.as_orig}]`;
361
+ }
362
+ });
363
+
364
+ if (this.statement.columns.length > 1 && wildcardIndexes.length > 0) {
365
+ _.pullAt(this.statement.columns, wildcardIndexes);
366
+ }
367
+ }
368
+
369
+ /**
370
+ * Searches the attributes for the matching column based on attribute & table name/alias
371
+ *
372
+ * @param column - the column to search for
373
+ * @returns {foundColumns}
374
+ * @private
375
+ */
376
+ _findColumn(column) {
377
+ //look to see if this attribute exists on one of the tables we are selecting from
378
+ let foundColumns = this.all_table_attributes.filter((attribute) => {
379
+ if (column.columnid_orig && column.tableid_orig) {
380
+ return (
381
+ (attribute.table.as === column.tableid_orig || attribute.table.tableid === column.tableid_orig) &&
382
+ attribute.attribute === column.columnid_orig
383
+ );
384
+ }
385
+
386
+ if (column.tableid) {
387
+ return (
388
+ (attribute.table.as === column.tableid || attribute.table.tableid === column.tableid) &&
389
+ attribute.attribute === column.columnid
390
+ );
391
+ }
392
+
393
+ const colName = column.columnid_orig ? column.columnid_orig : column.columnid;
394
+ return attribute.attribute === colName;
395
+ });
396
+
397
+ //this is to handle aliases. if we did not find the actual column we look at the aliases in the select columns and then return the matching column from allTableAttrs, if it exists
398
+ if (commonUtils.isEmptyOrZeroLength(foundColumns)) {
399
+ const foundAlias = this.columns.columns.filter((selectColumn) =>
400
+ selectColumn.as ? column.columnid === selectColumn.as : false
401
+ );
402
+ if (!commonUtils.isEmptyOrZeroLength(foundAlias)) {
403
+ foundColumns = this.all_table_attributes.filter(
404
+ (col) =>
405
+ col.attribute === foundAlias[0].columnid &&
406
+ foundAlias[0].tableid &&
407
+ foundAlias[0].tableid === (col.table.as ? col.table.as : col.table.tableid)
408
+ );
409
+ }
410
+ }
411
+
412
+ return foundColumns[0];
413
+ }
414
+
415
+ /**
416
+ * This function check to see if there is no from and no columns, or the table has been created but no data has been entered yet
417
+ * if there are not then this is a SELECT used to solely perform a calculation such as SELECT 2*4, or SELECT SQRT(4)
418
+ * @returns {Promise<[]>}
419
+ * @private
420
+ */
421
+ async _checkEmptySQL() {
422
+ let results = [];
423
+ //the scenario that allows this to occur is the table has been created but no data has been entered yet, in this case we return an empty array
424
+ if (
425
+ commonUtils.isEmptyOrZeroLength(this.all_table_attributes) &&
426
+ !commonUtils.isEmptyOrZeroLength(this.columns.columns)
427
+ ) {
428
+ //purpose of this is to break out of the waterfall but return an empty array
429
+ return results;
430
+ } else if (
431
+ commonUtils.isEmptyOrZeroLength(this.all_table_attributes) &&
432
+ commonUtils.isEmptyOrZeroLength(this.statement.from)
433
+ ) {
434
+ //this scenario is reached by doing a select with only calculations
435
+ try {
436
+ let sql = this._buildSQL(false);
437
+ results = await alasql.promise(sql);
438
+ } catch (e) {
439
+ log.error('Error thrown from AlaSQL in SQLSearch class method checkEmptySQL.');
440
+ log.error(e);
441
+ throw new Error('There was a problem with the SQL statement');
442
+ }
443
+ }
444
+ return results;
445
+ }
446
+
447
+ /**
448
+ * Iterates an ast segment columns and returns the found column. Typically fetch columns are columns specified in a
449
+ * join, where, or orderby clause.
450
+ * @param segmentAttributes
451
+ * @private
452
+ */
453
+ _addFetchColumns(segmentAttributes) {
454
+ if (segmentAttributes && segmentAttributes.length > 0) {
455
+ segmentAttributes.forEach((attribute) => {
456
+ let found = this._findColumn(attribute);
457
+ if (found) {
458
+ this.fetch_attributes.push(clone(found));
459
+ }
460
+ });
461
+ }
462
+ }
463
+
464
+ /**
465
+ * Adds new attribute metadata for the specified table to enable more easily accessing/adding/updating row data being built out
466
+ * @param schemaTable <String> the table to add the metadata to
467
+ * @param attr <String> the attribute to add to the table row metadata
468
+ * @private
469
+ */
470
+ _addColumnToMergedAttributes(schemaTable, attr) {
471
+ this.data[schemaTable].__mergedAttributes.push(attr);
472
+ this.data[schemaTable].__mergedAttrMap[attr] = this.data[schemaTable].__mergedAttributes.length - 1;
473
+ }
474
+
475
+ /**
476
+ * Adds the hash attribute to the specified table - this is similar to the above but unique for hash attributes because we always
477
+ * add hash keys to the first index position in the table metadata and do not need to add it to the `__mergedAttrMap`
478
+ * @param schemaTable <String> the table to add the metadata to
479
+ * @param hashValue <String> the hash key to add to the table row metadata
480
+ * @private
481
+ */
482
+ _setMergedHashAttribute(schemaTable, hashValue) {
483
+ this.data[schemaTable].__mergedData[hashValue].splice(0, 1, hashValue);
484
+ }
485
+
486
+ /**
487
+ * Updates the table row data for a specific hash value
488
+ * @param schemaTable <String> the table to update the hash value row in
489
+ * @param hashValue <String> the hash value to update an attr for
490
+ * @param attr <String> the attr to update in the table row
491
+ * @param updateValue <String> the value to update in the table row
492
+ * @private
493
+ */
494
+ _updateMergedAttribute(schemaTable, hashValue, attr, updateValue) {
495
+ const attrIndex = this.data[schemaTable].__mergedAttrMap[attr];
496
+ this.data[schemaTable].__mergedData[hashValue].splice(attrIndex, 1, updateValue);
497
+ }
498
+
499
+ /**
500
+ * Gets all values for the where, join, & order by attributes and converts the raw indexed data into individual
501
+ * rows by hash attribute consolidated based on tables. If the SQL statement is a simple SELECT query, this method
502
+ * will return the results from that select and bypass the additional alasql steps.
503
+ * @returns {Promise<void>}
504
+ * @private
505
+ */
506
+ async _getFetchAttributeValues() {
507
+ //If there are no columns in the AST at this point, it means that this query was a select * on a table that the
508
+ // user had read access to but has no access to read any of the attributes so we just return empty results.
509
+ if (commonUtils.isEmptyOrZeroLength(Object.keys(this.columns))) {
510
+ return [];
511
+ }
512
+ //get all unique attributes
513
+ this._addFetchColumns(this.columns.joins);
514
+
515
+ let whereString = null;
516
+ try {
517
+ whereString = this.statement.where ? this.statement.where.toString() : '';
518
+ } catch {
519
+ throw new Error('Could not generate proper where clause');
520
+ }
521
+ if (this.columns.where) {
522
+ this._addFetchColumns(this.columns.where);
523
+ }
524
+
525
+ //We need to check if statement only includes basic columns and a from value in the statement
526
+ // - if so, cannot treat as a simple select query and need to run through alasql
527
+ const simpleSelectQuery = this._isSimpleSelect();
528
+ if (simpleSelectQuery) {
529
+ this._addFetchColumns(this.columns.columns);
530
+ }
531
+ //the bitwise or '|' is intentionally used because I want both conditions checked regardless of whether the left condition is false
532
+ else if (
533
+ (!this.columns.where && this.fetch_attributes.length === 0) |
534
+ (whereString.indexOf(WHERE_CLAUSE_IS_NULL) > -1)
535
+ ) {
536
+ //get unique ids of tables if there is no join or the where is performing an is null check
537
+ this.tables.forEach((table) => {
538
+ let hash_attribute = {
539
+ columnid: getDatabases()[table.databaseid][table.tableid].primaryKey,
540
+ tableid: table.tableid,
541
+ };
542
+ this._addFetchColumns([hash_attribute]);
543
+ });
544
+ }
545
+
546
+ if (this.statement.order) {
547
+ this._updateOrderByToAliases();
548
+ this._addNonAggregatorsToFetchColumns();
549
+ }
550
+
551
+ // do we need this uniqueby, could just use object as map
552
+ this.fetch_attributes = _.uniqBy(this.fetch_attributes, (attribute) =>
553
+ [
554
+ attribute.table.databaseid,
555
+ attribute.table.as ? attribute.table.as : attribute.table.tableid,
556
+ attribute.attribute,
557
+ ].join()
558
+ );
559
+
560
+ if (simpleSelectQuery) {
561
+ return await this._simpleSQLQuery();
562
+ }
563
+
564
+ // create a template for each table row to ensure each row has a null value for attrs not returned in the search
565
+ const fetchAttrRowTemplates = this.fetch_attributes.reduce((acc, attr) => {
566
+ const schemaTable = `${attr.table.databaseid}_${attr.table.as ? attr.table.as : attr.table.tableid}`;
567
+ const hashName = this.data[schemaTable].__hashName;
568
+
569
+ if (!acc[schemaTable]) {
570
+ acc[schemaTable] = [];
571
+ acc[schemaTable].push(null);
572
+ this._addColumnToMergedAttributes(schemaTable, hashName);
573
+ }
574
+
575
+ if (attr.attribute !== hashName) {
576
+ acc[schemaTable].push(null);
577
+ this._addColumnToMergedAttributes(schemaTable, attr.attribute);
578
+ }
579
+
580
+ return acc;
581
+ }, {});
582
+
583
+ for (const attribute of this.fetch_attributes) {
584
+ const schemaTable = `${attribute.table.databaseid}_${
585
+ attribute.table.as ? attribute.table.as : attribute.table.tableid
586
+ }`;
587
+ let hashName = this.data[schemaTable].__hashName;
588
+
589
+ let searchObject = {
590
+ schema: attribute.table.databaseid,
591
+ table: attribute.table.tableid,
592
+ get_attributes: [attribute.attribute],
593
+ };
594
+ let isHash = false;
595
+ //Specifically a slash delimited string for consistency
596
+ let objectPath = [attribute.table.databaseid, attribute.table.tableid, attribute.attribute].join('/');
597
+
598
+ //check if this attribute is the hash attribute for a table, if it is we need to read the files from the __hdhHash
599
+ // folder, otherwise pull from the value index
600
+ if (attribute.attribute === hashName) {
601
+ isHash = true;
602
+ }
603
+
604
+ // if there exact match values for this attribute we just assign them to the attribute, otherwise we pull the
605
+ // index to get all values. This query will test the if statement below
606
+ // "sql":"select weightLbs, age, ownerName from dev.dog where ownerName = 'Kyle'"
607
+ if (
608
+ !commonUtils.isEmpty(this.exact_search_values[objectPath]) &&
609
+ !this.exact_search_values[objectPath].ignore &&
610
+ !commonUtils.isEmptyOrZeroLength(this.exact_search_values[objectPath].values)
611
+ ) {
612
+ if (isHash) {
613
+ try {
614
+ searchObject.hash_values = Array.from(this.exact_search_values[objectPath].values);
615
+ const attributeValues = await harperBridge.getDataByHash(searchObject);
616
+
617
+ for (const hashVal of searchObject.hash_values) {
618
+ if (attributeValues.get(hashVal) && !this.data[schemaTable].__mergedData[hashVal]) {
619
+ this.data[schemaTable].__mergedData[hashVal] = [...fetchAttrRowTemplates[schemaTable]];
620
+ this._setMergedHashAttribute(schemaTable, hashVal);
621
+ }
622
+ }
623
+ } catch (err) {
624
+ log.error(
625
+ 'Error thrown from getDataByHash function in SQLSearch class method getFetchAttributeValues exact match.'
626
+ );
627
+ log.error(err);
628
+ throw new Error(SEARCH_ERROR_MSG);
629
+ }
630
+ } else {
631
+ try {
632
+ searchObject.attribute = attribute.attribute;
633
+ await Promise.all(
634
+ Array.from(this.exact_search_values[objectPath].values).map(async (value) => {
635
+ let exactSearchObject = { ...searchObject };
636
+ exactSearchObject.value = value;
637
+ const attributeValues = await harperBridge.getDataByValue(exactSearchObject);
638
+
639
+ for (const [hashVal, record] of attributeValues) {
640
+ if (!this.data[schemaTable].__mergedData[hashVal]) {
641
+ this.data[schemaTable].__mergedData[hashVal] = [...fetchAttrRowTemplates[schemaTable]];
642
+ this._updateMergedAttribute(schemaTable, hashVal, attribute.attribute, record[attribute.attribute]);
643
+ this._setMergedHashAttribute(schemaTable, hashVal);
644
+ } else {
645
+ this._updateMergedAttribute(schemaTable, hashVal, attribute.attribute, record[attribute.attribute]);
646
+ }
647
+ }
648
+ })
649
+ );
650
+ } catch (err) {
651
+ log.error(
652
+ 'Error thrown from getDataByValue function in SQLSearch class method getFetchAttributeValues exact match.'
653
+ );
654
+ log.error(err);
655
+ throw new Error(SEARCH_ERROR_MSG);
656
+ }
657
+ }
658
+ } else if (
659
+ !commonUtils.isEmpty(this.comparator_search_values[objectPath]) &&
660
+ !this.comparator_search_values[objectPath].ignore &&
661
+ !commonUtils.isEmptyOrZeroLength(this.comparator_search_values[objectPath].comparators)
662
+ ) {
663
+ try {
664
+ const searchValueComparators = this.comparator_search_values[objectPath].comparators;
665
+ for (let i = 0, len = searchValueComparators.length; i < len; i++) {
666
+ const comp = searchValueComparators[i];
667
+ searchObject.attribute = comp.attribute;
668
+ searchObject.value = comp.value;
669
+ const matchingData = await harperBridge.getDataByValue(searchObject, comp.operation);
670
+
671
+ if (isHash) {
672
+ for (const [hashVal] of matchingData) {
673
+ if (!this.data[schemaTable].__mergedData[hashVal]) {
674
+ this.data[schemaTable].__mergedData[hashVal] = [...fetchAttrRowTemplates[schemaTable]];
675
+ this._setMergedHashAttribute(schemaTable, hashVal);
676
+ }
677
+ }
678
+ } else {
679
+ for (const [hashVal, record] of matchingData) {
680
+ if (!this.data[schemaTable].__mergedData[hashVal]) {
681
+ this.data[schemaTable].__mergedData[hashVal] = [...fetchAttrRowTemplates[schemaTable]];
682
+ this._updateMergedAttribute(schemaTable, hashVal, attribute.attribute, record[attribute.attribute]);
683
+ this._setMergedHashAttribute(schemaTable, hashVal);
684
+ } else {
685
+ this._updateMergedAttribute(schemaTable, hashVal, attribute.attribute, record[attribute.attribute]);
686
+ }
687
+ }
688
+ }
689
+ }
690
+ } catch (err) {
691
+ log.error(
692
+ 'Error thrown from getDataByValue function in SQLSearch class method getFetchAttributeValues comparator search values.'
693
+ );
694
+ log.error(err);
695
+ throw new Error(SEARCH_ERROR_MSG);
696
+ }
697
+ } else {
698
+ try {
699
+ searchObject.attribute = attribute.attribute;
700
+ searchObject.value = '*';
701
+ const matchingData = await harperBridge.getDataByValue(searchObject);
702
+
703
+ if (isHash) {
704
+ for (const [hashVal] of matchingData) {
705
+ if (!this.data[schemaTable].__mergedData[hashVal]) {
706
+ this.data[schemaTable].__mergedData[hashVal] = [...fetchAttrRowTemplates[schemaTable]];
707
+ this._setMergedHashAttribute(schemaTable, hashVal);
708
+ }
709
+ }
710
+ } else {
711
+ for (const [hashVal, record] of matchingData) {
712
+ if (!this.data[schemaTable].__mergedData[hashVal]) {
713
+ this.data[schemaTable].__mergedData[hashVal] = [...fetchAttrRowTemplates[schemaTable]];
714
+ this._updateMergedAttribute(schemaTable, hashVal, attribute.attribute, record[attribute.attribute]);
715
+ this._setMergedHashAttribute(schemaTable, hashVal);
716
+ } else {
717
+ this._updateMergedAttribute(schemaTable, hashVal, attribute.attribute, record[attribute.attribute]);
718
+ }
719
+ }
720
+ }
721
+ } catch (err) {
722
+ log.error(
723
+ 'Error thrown from getDataByValue function in SQLSearch class method getFetchAttributeValues no comparator search values.'
724
+ );
725
+ log.error(err);
726
+ throw new Error(SEARCH_ERROR_MSG);
727
+ }
728
+ }
729
+ }
730
+ }
731
+
732
+ /**
733
+ * Checks if SQL statement only includes basic SELECT columns FROM one table
734
+ * @returns {boolean} is SQL statement a simple select
735
+ * @private
736
+ */
737
+ _isSimpleSelect() {
738
+ let isSimpleSelect = true;
739
+
740
+ if (
741
+ Object.keys(this.statement).length !== 2 ||
742
+ !this.statement.columns ||
743
+ !this.statement.from ||
744
+ this.statement.from.length !== 1
745
+ ) {
746
+ isSimpleSelect = false;
747
+ return isSimpleSelect;
748
+ }
749
+
750
+ this.statement.columns.forEach((col) => {
751
+ if (!(col instanceof alasql.yy.Column)) {
752
+ isSimpleSelect = false;
753
+ }
754
+ });
755
+
756
+ return isSimpleSelect;
757
+ }
758
+
759
+ /**
760
+ * Updates the AST order by values to utilize the aliases already set for the corresponding column values. This is required to
761
+ * resolve a bug in alasql where column values/references in the order by are not parsed by the library correctly.
762
+ * @private
763
+ */
764
+ _updateOrderByToAliases() {
765
+ this.statement.order.forEach((orderBy) => {
766
+ //We don't need to do anything with the alias if the orderby is an aggregator
767
+ if (orderBy.expression.aggregatorid) {
768
+ orderBy.is_aggregator = true;
769
+ return;
770
+ }
771
+
772
+ if (orderBy.expression.value) {
773
+ orderBy.is_ordinal = true;
774
+ this.has_ordinal = true;
775
+ return;
776
+ } else {
777
+ orderBy.is_ordinal = false;
778
+ }
779
+
780
+ let foundColumn = this.statement.columns.filter((col) => {
781
+ const colExpression = col.aggregatorid ? col.expression : col;
782
+ const colAlias = col.aggregatorid ? col.as_orig : colExpression.as_orig;
783
+
784
+ if (!orderBy.expression.tableid) {
785
+ return (
786
+ colExpression.columnid_orig === orderBy.expression.columnid_orig ||
787
+ orderBy.expression.columnid_orig === colAlias
788
+ );
789
+ } else {
790
+ return (
791
+ colExpression.columnid_orig === orderBy.expression.columnid_orig &&
792
+ colExpression.tableid_orig === orderBy.expression.tableid_orig
793
+ );
794
+ }
795
+ });
796
+
797
+ if (!foundColumn[0]) {
798
+ foundColumn.push(this._findColumn(orderBy.expression));
799
+ }
800
+
801
+ let selectColumn = foundColumn[0];
802
+
803
+ //These values are used in later steps to help evaluate how best to treat the order by statement in our logic
804
+ orderBy.is_func = !!selectColumn.funcid;
805
+ orderBy.is_aggregator = !!selectColumn.aggregatorid;
806
+
807
+ if (!selectColumn.as) {
808
+ orderBy.initial_select_column = Object.assign(new alasql.yy.Column(), orderBy.expression);
809
+ orderBy.initial_select_column.as = `[${orderBy.expression.columnid_orig}]`;
810
+ orderBy.expression.columnid = orderBy.initial_select_column.as;
811
+ return;
812
+ } else if (selectColumn.as && !orderBy.expression.tableid) {
813
+ orderBy.expression.columnid = selectColumn.as;
814
+ orderBy.expression.columnid_orig = selectColumn.as_orig;
815
+ } else {
816
+ let aliasExpression = new alasql.yy.Column();
817
+ aliasExpression.columnid = selectColumn.as;
818
+ aliasExpression.columnid_orig = selectColumn.as_orig;
819
+ orderBy.expression = aliasExpression;
820
+ }
821
+ if (!orderBy.is_aggregator) {
822
+ const targetObj = orderBy.is_func ? new alasql.yy.FuncValue() : new alasql.yy.Column();
823
+ orderBy.initial_select_column = Object.assign(targetObj, selectColumn);
824
+ }
825
+ });
826
+ }
827
+
828
+ /**
829
+ * This ensures that the non-aggregator columns included in the order by statement are included in the table data for the
830
+ * first pass of alasql
831
+ * @private
832
+ */
833
+ _addNonAggregatorsToFetchColumns() {
834
+ const nonAggrOrderByCols = this.statement.order.filter((ob) => !ob.is_aggregator && !ob.is_ordinal);
835
+ const nonAggrColumnids = nonAggrOrderByCols.map((ob) => {
836
+ if (ob.is_func) {
837
+ const colIdArg = ob.initial_select_column.args.filter((arg) => !!arg.columnid_orig);
838
+ return { columnid: colIdArg[0].columnid_orig };
839
+ } else {
840
+ return { columnid: ob.expression.columnid_orig };
841
+ }
842
+ });
843
+ this._addFetchColumns(nonAggrColumnids);
844
+ }
845
+
846
+ /**
847
+ * Takes an initial pass on the data by processing just the joins, conditions and order by.
848
+ * This allows us to limit the broader select based on just the ids we need based on this pass
849
+ * @returns {Promise<{existingAttributes, joined_length: number}>}
850
+ * @private
851
+ */
852
+ async _processJoins() {
853
+ let tableData = [];
854
+ let select = [];
855
+ //TODO need to loop from here to ensure cross joins are covered - i.e. 'from tablea a, tableb b, tablec c' -
856
+ // this is not high priority but is covered in CORE-894
857
+ let fromStatement = this.statement.from[0];
858
+ let tables = [fromStatement];
859
+ let fromClause = ['? ' + (fromStatement.as ? ' AS ' + fromStatement.as : fromStatement.tableid)];
860
+
861
+ tableData.push(
862
+ Object.values(
863
+ this.data[
864
+ `${fromStatement.databaseid_orig}_${fromStatement.as ? fromStatement.as_orig : fromStatement.tableid_orig}`
865
+ ].__mergedData
866
+ )
867
+ );
868
+
869
+ if (this.statement.joins) {
870
+ this.statement.joins.forEach((join) => {
871
+ if (join.joinmode && join.joinmode !== 'INNER') {
872
+ this.has_outer_join = true;
873
+ }
874
+ tables.push(join.table);
875
+ let from = join.joinmode + ' JOIN ? AS ' + (join.as ? join.as : join.table.tableid);
876
+
877
+ if (join.on) {
878
+ from += ' ON ' + join.on.toString();
879
+ }
880
+ fromClause.push(from);
881
+
882
+ tableData.push(
883
+ Object.values(
884
+ this.data[`${join.table.databaseid_orig}_${join.table.as ? join.table.as_orig : join.table.tableid_orig}`]
885
+ .__mergedData
886
+ )
887
+ );
888
+ });
889
+ }
890
+
891
+ //record the fetched attributes so we can compare to what else needs to be grabbed
892
+ let hashAttributes = [];
893
+ let existingAttributes = {};
894
+ tables.forEach((table) => {
895
+ let hash = this.data[`${table.databaseid_orig}_${table.as ? table.as_orig : table.tableid_orig}`].__hashName;
896
+ const tableKey = table.as ? table.as_orig : table.tableid_orig;
897
+ hashAttributes.push({
898
+ key: `'${tableKey}.${hash}'`,
899
+ schema: table.databaseid_orig,
900
+ table: table.as ? table.as_orig : table.tableid_orig,
901
+ keys: new Set(),
902
+ });
903
+ select.push(`${table.as ? table.as : table.tableid}.\`${hash}\` AS "${tableKey}.${hash}"`);
904
+
905
+ existingAttributes[table.as ? table.as_orig : table.tableid_orig] =
906
+ this.data[`${table.databaseid_orig}_${table.as ? table.as_orig : table.tableid_orig}`].__mergedAttributes;
907
+ });
908
+
909
+ //TODO there is an error with between statements being converted back to string. need to handle
910
+ //TODO - CORE-1095 - update how WHERE clause is translated back to SQL query for where expression values include escaped characters
911
+ let whereClause = this.statement.where ? 'WHERE ' + this.statement.where : '';
912
+ whereClause = whereClause.replace(/NOT\(NULL\)/g, 'NOT NULL');
913
+
914
+ let orderClause = '';
915
+ //the only time we need to include the order by statement in the first pass is when there are no aggregators,
916
+ // no ordinals in order by, and/or no group by statements AND there is a LIMIT because final sorting will be done on
917
+ // the data that is returned from the 2nd alasql pass
918
+ if (
919
+ this.statement.order &&
920
+ !this.has_ordinal &&
921
+ !this.has_aggregator &&
922
+ !this.statement.group &&
923
+ this.statement.limit
924
+ ) {
925
+ orderClause = 'ORDER BY ' + this.statement.order.toString();
926
+ //because of the alasql bug with orderby (CORE-929), we need to add the ORDER BY column to the select with the
927
+ // alias to ensure it's available for sorting in the first pass
928
+ this.statement.order.forEach((ob) => {
929
+ if (ob.is_func) {
930
+ select.push(ob.initial_select_column.toString());
931
+ } else if (ob.initial_select_column.tableid) {
932
+ select.push(
933
+ `${ob.initial_select_column.tableid}.${ob.initial_select_column.columnid} AS ${ob.expression.columnid}`
934
+ );
935
+ } else {
936
+ select.push(`${ob.initial_select_column.columnid} AS ${ob.expression.columnid}`);
937
+ }
938
+ });
939
+ }
940
+
941
+ let limit = '';
942
+ let offset = '';
943
+ if (!this.has_aggregator && !this.statement.group && !this.has_ordinal && !this.statement.joins) {
944
+ limit = this.statement.limit ? 'LIMIT ' + this.statement.limit : '';
945
+ offset = this.statement.offset ? 'OFFSET ' + this.statement.offset : '';
946
+ }
947
+
948
+ let joined = [];
949
+
950
+ try {
951
+ const initialSql = `SELECT ${select.join(', ')} FROM ${fromClause.join(
952
+ ' '
953
+ )} ${whereClause} ${orderClause} ${limit} ${offset}`;
954
+ const finalSqlOperation = this._convertColumnsToIndexes(initialSql, tables);
955
+ joined = await alasql.promise(finalSqlOperation, tableData);
956
+ tableData = null;
957
+ } catch (err) {
958
+ log.error('Error thrown from AlaSQL in SQLSearch class method processJoins.');
959
+ log.error(err);
960
+ throw new Error('There was a problem processing the data.');
961
+ }
962
+
963
+ //collect returned hash values and remove others from table's __mergedData
964
+ if (joined && joined.length > 0) {
965
+ for (let i = 0, len = joined.length; i < len; i++) {
966
+ const row = joined[i];
967
+ hashAttributes.forEach((hash) => {
968
+ if (row[hash.key] !== null && row[hash.key] !== undefined) {
969
+ hash.keys.add(row[hash.key]);
970
+ }
971
+ });
972
+ }
973
+
974
+ hashAttributes.forEach((hash) => {
975
+ let keys = Object.keys(this.data[`${hash.schema}_${hash.table}`].__mergedData);
976
+ let deleteKeys = _.difference(
977
+ keys,
978
+ [...hash.keys].map((key) => key.toString())
979
+ );
980
+ for (let i = 0, len = deleteKeys.length; i < len; i++) {
981
+ const key = deleteKeys[i];
982
+ delete this.data[`${hash.schema}_${hash.table}`].__mergedData[key];
983
+ }
984
+ });
985
+ }
986
+ return {
987
+ existing_attributes: existingAttributes,
988
+ joined_length: joined ? joined.length : 0,
989
+ };
990
+ }
991
+
992
+ /**
993
+ * Gets remaining attribute values for final SQL operation that were not grabbed during first pass
994
+ * @param existingAttributes
995
+ * @param rowCount
996
+ * @returns {Promise<void>}
997
+ * @private
998
+ */
999
+ async _getFinalAttributeData(existingAttributes, rowCount) {
1000
+ if (rowCount === 0) {
1001
+ return;
1002
+ }
1003
+
1004
+ let allColumns = [];
1005
+ let iterator = new RecursiveIterator(this.columns);
1006
+ for (let { node } of iterator) {
1007
+ if (node && node.columnid) {
1008
+ let found = this._findColumn(node);
1009
+ if (found) {
1010
+ let tableKey = found.table.as ? found.table.as : found.table.tableid;
1011
+ if (!existingAttributes[tableKey] || existingAttributes[tableKey].indexOf(found.attribute) < 0) {
1012
+ allColumns.push(found);
1013
+ }
1014
+ }
1015
+ }
1016
+ }
1017
+
1018
+ allColumns = _.uniqBy(allColumns, (attribute) =>
1019
+ [
1020
+ attribute.table.databaseid,
1021
+ attribute.table.as ? attribute.table.as : attribute.table.tableid,
1022
+ attribute.attribute,
1023
+ ].join()
1024
+ );
1025
+
1026
+ try {
1027
+ await this._getData(allColumns);
1028
+ } catch (e) {
1029
+ log.error('Error thrown from getData in SQLSearch class method getFinalAttributeData.');
1030
+ log.error(e);
1031
+ throw new Error(SEARCH_ERROR_MSG);
1032
+ }
1033
+ }
1034
+
1035
+ /**
1036
+ * Organizes the final data searches based on tables being search to ensure we are only searching each table once
1037
+ * @param allColumns - remaining columns to be searched in
1038
+ * @returns {Promise<void>}
1039
+ * @private
1040
+ */
1041
+ async _getData(allColumns) {
1042
+ try {
1043
+ const tableSearches = allColumns.reduce((acc, column) => {
1044
+ const tableKey = `${column.table.databaseid}_${column.table.as ? column.table.as : column.table.tableid}`;
1045
+ if (!acc[tableKey]) {
1046
+ acc[tableKey] = {
1047
+ schema: column.table.databaseid,
1048
+ table: column.table.tableid,
1049
+ columns: [column.attribute],
1050
+ };
1051
+ } else {
1052
+ acc[tableKey].columns.push(column.attribute);
1053
+ }
1054
+ return acc;
1055
+ }, {});
1056
+
1057
+ for (const schemaTable in tableSearches) {
1058
+ const table = tableSearches[schemaTable];
1059
+ const mergedData = this.data[schemaTable].__mergedData;
1060
+ const mergedHashKeys = [];
1061
+ for (let key in mergedData) {
1062
+ mergedHashKeys.push(mergedData[key][0]);
1063
+ }
1064
+ //we do not need to update the mergedAttrMap values here b/c we will use the index value from
1065
+ // __mergedAttributes when do the final translation of the SQL statement
1066
+ this.data[schemaTable].__mergedAttributes.push(...table.columns);
1067
+
1068
+ const searchObject = {
1069
+ schema: table.schema,
1070
+ table: table.table,
1071
+ hash_values: mergedHashKeys,
1072
+ get_attributes: table.columns,
1073
+ };
1074
+
1075
+ const searchResult = await harperBridge.getDataByHash(searchObject);
1076
+ const tableColsLength = table.columns.length;
1077
+
1078
+ for (let i = 0, len = mergedHashKeys.length; i < len; i++) {
1079
+ const theId = mergedHashKeys[i];
1080
+ const theRow = searchResult.get(theId);
1081
+ for (let j = 0; j < tableColsLength; j++) {
1082
+ const val = table.columns[j];
1083
+ const attrVal = theRow[val] === undefined ? null : theRow[val];
1084
+ this.data[schemaTable].__mergedData[theId].push(attrVal);
1085
+ }
1086
+ }
1087
+ }
1088
+ } catch (e) {
1089
+ log.error('Error thrown from getDataByHash function in SQLSearch class method getData.');
1090
+ log.error(e);
1091
+ throw e;
1092
+ }
1093
+ }
1094
+
1095
+ /**
1096
+ * Takes all of the raw data and executes the full SQL from the AST against the data.
1097
+ * @returns {Promise<[finalResults]>}
1098
+ * @private
1099
+ */
1100
+ async _finalSQL() {
1101
+ let tableData = [];
1102
+ //TODO need to loop from here to ensure cross joins are covered - i.e. 'from tablea a, tableb b, tablec c' -
1103
+ // this is not high priority but is covered in CORE-894
1104
+ let fromStatement = this.statement.from[0];
1105
+ tableData.push(
1106
+ Object.values(
1107
+ this.data[
1108
+ `${fromStatement.databaseid_orig}_${fromStatement.as ? fromStatement.as_orig : fromStatement.tableid_orig}`
1109
+ ].__mergedData
1110
+ )
1111
+ );
1112
+ fromStatement.as = fromStatement.as ? fromStatement.as : fromStatement.tableid;
1113
+ fromStatement.databaseid = '';
1114
+ fromStatement.tableid = '?';
1115
+
1116
+ if (this.statement.joins) {
1117
+ this.statement.joins.forEach((join) => {
1118
+ join.as = join.as ? join.as : join.table.tableid;
1119
+
1120
+ tableData.push(
1121
+ Object.values(
1122
+ this.data[`${join.table.databaseid_orig}_${join.table.as ? join.table.as_orig : join.table.tableid_orig}`]
1123
+ .__mergedData
1124
+ )
1125
+ );
1126
+ join.table.databaseid = '';
1127
+ join.table.tableid = '?';
1128
+ });
1129
+ }
1130
+
1131
+ if (this.statement.order) {
1132
+ this.statement.order.forEach((ob) => {
1133
+ if (ob.is_ordinal) {
1134
+ return;
1135
+ }
1136
+ const found = this.statement.columns.filter((col) => {
1137
+ const colExpression = col.aggregatorid ? col.expression : col;
1138
+ const colAlias = col.aggregatorid ? col.as_orig : colExpression.as_orig;
1139
+
1140
+ if (!ob.expression.tableid) {
1141
+ return (
1142
+ colExpression.columnid_orig === ob.expression.columnid_orig || ob.expression.columnid_orig === colAlias
1143
+ );
1144
+ } else {
1145
+ return (
1146
+ colExpression.columnid_orig === ob.expression.columnid_orig &&
1147
+ colExpression.tableid_orig === ob.expression.tableid_orig
1148
+ );
1149
+ }
1150
+ });
1151
+
1152
+ if (found.length === 0) {
1153
+ ob.expression.columnid = ob.initial_select_column.columnid;
1154
+ }
1155
+ });
1156
+ }
1157
+
1158
+ //if we processed the offset in first sql pass it will force it again which will cause no records to be returned
1159
+ // this deletes the offset and also the limit if they were already run in the first pass
1160
+ if (
1161
+ !this.has_aggregator &&
1162
+ !this.statement.group &&
1163
+ !this.has_ordinal &&
1164
+ this.statement.limit &&
1165
+ !this.statement.joins
1166
+ ) {
1167
+ delete this.statement.limit;
1168
+ delete this.statement.offset;
1169
+ }
1170
+
1171
+ let finalResults = undefined;
1172
+ try {
1173
+ let sql = this._buildSQL();
1174
+ log.trace(`Final SQL: ${sql}`);
1175
+ finalResults = await alasql.promise(sql, tableData);
1176
+ if (this.has_outer_join) {
1177
+ finalResults = this._translateUndefinedValues(finalResults);
1178
+ }
1179
+ log.trace(`Final AlaSQL results data included ${finalResults.length} rows`);
1180
+ } catch (err) {
1181
+ log.error('Error thrown from AlaSQL in SQLSearch class method finalSQL.');
1182
+ log.error(err);
1183
+ throw new Error('There was a problem running the generated sql.');
1184
+ }
1185
+
1186
+ return finalResults;
1187
+ }
1188
+
1189
+ _translateUndefinedValues(data) {
1190
+ try {
1191
+ let finalData = [];
1192
+ for (const row of data) {
1193
+ let finalRow = Object.create(null);
1194
+ Object.keys(row).forEach((key) => {
1195
+ if (row[key] === undefined) {
1196
+ finalRow[key] = null;
1197
+ } else {
1198
+ finalRow[key] = row[key];
1199
+ }
1200
+ });
1201
+ finalData.push(finalRow);
1202
+ }
1203
+ return finalData;
1204
+ } catch (e) {
1205
+ log.error(hdbErrors.HDB_ERROR_MSGS.OUTER_JOIN_TRANSLATION_ERROR);
1206
+ log.trace(e.stack);
1207
+ return data;
1208
+ }
1209
+ }
1210
+
1211
+ /**
1212
+ * There is a bug in alasql where functions with aliases get their alias duplicated in the sql string.
1213
+ * we need to parse out the duplicate and replace with an empty string
1214
+ * @returns {string}
1215
+ * @private
1216
+ */
1217
+ _buildSQL(callConvertToIndexes = true) {
1218
+ let sql = this.statement.toString();
1219
+ sql = sql.replace(/NOT\(NULL\)/g, 'NOT NULL');
1220
+
1221
+ this.statement.columns.forEach((column) => {
1222
+ if (column.funcid && column.as) {
1223
+ let columnString = column.toString().replace(' AS ' + column.as, '');
1224
+ sql = sql.replace(column.toString(), columnString);
1225
+ }
1226
+ });
1227
+
1228
+ if (callConvertToIndexes === true) {
1229
+ return this._convertColumnsToIndexes(sql, this.tables);
1230
+ }
1231
+
1232
+ return sql;
1233
+ }
1234
+
1235
+ /**
1236
+ * Updates the sqlStatment string to use index values instead of table column names
1237
+ * @param sqlStatement
1238
+ * @param tables
1239
+ * @returns {*}
1240
+ * @private
1241
+ */
1242
+ _convertColumnsToIndexes(sqlStatement, tables) {
1243
+ let finalSql = sqlStatement;
1244
+ const tablesMap = {};
1245
+ tables.forEach((table) => {
1246
+ if (table.databaseid_orig) {
1247
+ tablesMap[`${table.databaseid_orig}_${table.as ? table.as_orig : table.tableid_orig}`] = table.as
1248
+ ? table.as
1249
+ : table.tableid;
1250
+ } else {
1251
+ tablesMap[`${table.databaseid}_${table.as ? table.as : table.tableid}`] = `\`${
1252
+ table.as ? table.as : table.tableid
1253
+ }\``;
1254
+ }
1255
+ });
1256
+ for (const schemaTable in this.data) {
1257
+ this.data[schemaTable].__mergedAttributes.forEach((attr, index) => {
1258
+ const table = tablesMap[schemaTable];
1259
+ let find = new RegExp(`${table}.\`${attr}\``, 'g');
1260
+ let replace = `${table}.[${index}]`;
1261
+
1262
+ finalSql = finalSql.replace(find, replace);
1263
+ });
1264
+ }
1265
+
1266
+ for (const schemaTable in this.data) {
1267
+ this.data[schemaTable].__mergedAttributes.forEach((attr, index) => {
1268
+ let find = new RegExp(`\`${attr}\``, 'g');
1269
+ let replace = `[${index}]`;
1270
+
1271
+ finalSql = finalSql.replace(find, replace);
1272
+ });
1273
+ }
1274
+ return finalSql;
1275
+ }
1276
+
1277
+ /**
1278
+ * Builds out the final result JSON for a simple SQL query to return to the main search method without using alasql
1279
+ * @returns {Promise<unknown[]>}
1280
+ * @private
1281
+ */
1282
+ async _simpleSQLQuery() {
1283
+ let aliasMap = this.statement.columns.reduce((acc, col) => {
1284
+ if (col.as_orig && col.as_orig != col.columnid_orig) {
1285
+ acc[col.columnid_orig] = col.as_orig;
1286
+ } else if (!acc[col.columnid_orig]) {
1287
+ acc[col.columnid_orig] = col.columnid_orig;
1288
+ }
1289
+ return acc;
1290
+ }, {});
1291
+
1292
+ const fetchAttributesObjs = this.fetch_attributes.reduce((acc, attr) => {
1293
+ const schemaTable = `${attr.table.databaseid}_${attr.table.as ? attr.table.as : attr.table.tableid}`;
1294
+ if (!acc[schemaTable]) {
1295
+ acc[schemaTable] = {};
1296
+ }
1297
+ acc[schemaTable][aliasMap[attr.attribute]] = null;
1298
+ return acc;
1299
+ }, {});
1300
+
1301
+ for (const attribute of this.fetch_attributes) {
1302
+ const schemaTable = `${attribute.table.databaseid}_${
1303
+ attribute.table.as ? attribute.table.as : attribute.table.tableid
1304
+ }`;
1305
+
1306
+ let searchObject = {
1307
+ schema: attribute.table.databaseid,
1308
+ table: attribute.table.tableid,
1309
+ get_attributes: [attribute.attribute],
1310
+ };
1311
+
1312
+ try {
1313
+ searchObject.attribute = attribute.attribute;
1314
+ searchObject.value = '*';
1315
+ const matchingData = await harperBridge.getDataByValue(searchObject);
1316
+
1317
+ for (const [hashVal, record] of matchingData) {
1318
+ if (!this.data[schemaTable].__mergedData[hashVal]) {
1319
+ if (record[attribute.attribute] === undefined) record[attribute.attribute] = null;
1320
+ this.data[schemaTable].__mergedData[hashVal] = { ...fetchAttributesObjs[schemaTable] };
1321
+ }
1322
+ this.data[schemaTable].__mergedData[hashVal][aliasMap[attribute.attribute]] =
1323
+ record[attribute.attribute] ?? null;
1324
+ }
1325
+ } catch (err) {
1326
+ log.error('There was an error when processing this SQL operation. Check your logs');
1327
+ log.error(err);
1328
+ throw new Error(SEARCH_ERROR_MSG);
1329
+ }
1330
+ }
1331
+ return Object.values(Object.values(this.data)[0].__mergedData);
1332
+ }
1333
+ }
1334
+
1335
+ module.exports = SQLSearch;