@smythos/sre 1.6.1 → 1.6.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/CHANGELOG +111 -111
  2. package/LICENSE +18 -18
  3. package/README.md +135 -135
  4. package/dist/index.js +2 -2
  5. package/dist/index.js.map +1 -1
  6. package/dist/types/subsystems/LLMManager/ModelsProvider.service/connectors/SmythModelsProvider.class.d.ts +39 -0
  7. package/package.json +1 -1
  8. package/src/Components/APICall/APICall.class.ts +161 -161
  9. package/src/Components/APICall/AccessTokenManager.ts +166 -166
  10. package/src/Components/APICall/ArrayBufferResponse.helper.ts +58 -58
  11. package/src/Components/APICall/OAuth.helper.ts +447 -447
  12. package/src/Components/APICall/mimeTypeCategories.ts +46 -46
  13. package/src/Components/APICall/parseData.ts +167 -167
  14. package/src/Components/APICall/parseHeaders.ts +41 -41
  15. package/src/Components/APICall/parseProxy.ts +68 -68
  16. package/src/Components/APICall/parseUrl.ts +91 -91
  17. package/src/Components/APIEndpoint.class.ts +234 -234
  18. package/src/Components/APIOutput.class.ts +58 -58
  19. package/src/Components/AgentPlugin.class.ts +102 -102
  20. package/src/Components/Async.class.ts +155 -155
  21. package/src/Components/Await.class.ts +90 -90
  22. package/src/Components/Classifier.class.ts +158 -158
  23. package/src/Components/Component.class.ts +147 -147
  24. package/src/Components/ComponentHost.class.ts +38 -38
  25. package/src/Components/DataSourceCleaner.class.ts +92 -92
  26. package/src/Components/DataSourceIndexer.class.ts +181 -181
  27. package/src/Components/DataSourceLookup.class.ts +161 -161
  28. package/src/Components/ECMASandbox.class.ts +72 -72
  29. package/src/Components/FEncDec.class.ts +29 -29
  30. package/src/Components/FHash.class.ts +33 -33
  31. package/src/Components/FSign.class.ts +80 -80
  32. package/src/Components/FSleep.class.ts +25 -25
  33. package/src/Components/FTimestamp.class.ts +66 -66
  34. package/src/Components/FileStore.class.ts +78 -78
  35. package/src/Components/ForEach.class.ts +97 -97
  36. package/src/Components/GPTPlugin.class.ts +70 -70
  37. package/src/Components/GenAILLM.class.ts +586 -586
  38. package/src/Components/HuggingFace.class.ts +313 -313
  39. package/src/Components/Image/imageSettings.config.ts +70 -70
  40. package/src/Components/ImageGenerator.class.ts +483 -483
  41. package/src/Components/JSONFilter.class.ts +54 -54
  42. package/src/Components/LLMAssistant.class.ts +213 -213
  43. package/src/Components/LogicAND.class.ts +28 -28
  44. package/src/Components/LogicAtLeast.class.ts +85 -85
  45. package/src/Components/LogicAtMost.class.ts +86 -86
  46. package/src/Components/LogicOR.class.ts +29 -29
  47. package/src/Components/LogicXOR.class.ts +34 -34
  48. package/src/Components/MCPClient.class.ts +137 -137
  49. package/src/Components/MemoryDeleteKeyVal.class.ts +70 -70
  50. package/src/Components/MemoryReadKeyVal.class.ts +67 -67
  51. package/src/Components/MemoryWriteKeyVal.class.ts +62 -62
  52. package/src/Components/MemoryWriteObject.class.ts +97 -97
  53. package/src/Components/MultimodalLLM.class.ts +128 -128
  54. package/src/Components/OpenAPI.class.ts +72 -72
  55. package/src/Components/PromptGenerator.class.ts +122 -122
  56. package/src/Components/ScrapflyWebScrape.class.ts +183 -183
  57. package/src/Components/ServerlessCode.class.ts +123 -123
  58. package/src/Components/TavilyWebSearch.class.ts +103 -103
  59. package/src/Components/VisionLLM.class.ts +104 -104
  60. package/src/Components/ZapierAction.class.ts +127 -127
  61. package/src/Components/index.ts +97 -97
  62. package/src/Core/AgentProcess.helper.ts +240 -240
  63. package/src/Core/Connector.class.ts +123 -123
  64. package/src/Core/ConnectorsService.ts +197 -197
  65. package/src/Core/DummyConnector.ts +49 -49
  66. package/src/Core/HookService.ts +105 -105
  67. package/src/Core/SmythRuntime.class.ts +241 -241
  68. package/src/Core/SystemEvents.ts +16 -16
  69. package/src/Core/boot.ts +56 -56
  70. package/src/config.ts +15 -15
  71. package/src/constants.ts +126 -126
  72. package/src/data/hugging-face.params.json +579 -579
  73. package/src/helpers/AWSLambdaCode.helper.ts +624 -599
  74. package/src/helpers/BinaryInput.helper.ts +331 -331
  75. package/src/helpers/Conversation.helper.ts +1157 -1157
  76. package/src/helpers/ECMASandbox.helper.ts +64 -64
  77. package/src/helpers/JsonContent.helper.ts +97 -97
  78. package/src/helpers/LocalCache.helper.ts +97 -97
  79. package/src/helpers/Log.helper.ts +274 -274
  80. package/src/helpers/OpenApiParser.helper.ts +150 -150
  81. package/src/helpers/S3Cache.helper.ts +147 -147
  82. package/src/helpers/SmythURI.helper.ts +5 -5
  83. package/src/helpers/Sysconfig.helper.ts +95 -95
  84. package/src/helpers/TemplateString.helper.ts +243 -243
  85. package/src/helpers/TypeChecker.helper.ts +329 -329
  86. package/src/index.ts +198 -198
  87. package/src/index.ts.bak +198 -198
  88. package/src/subsystems/AgentManager/Agent.class.ts +1114 -1114
  89. package/src/subsystems/AgentManager/Agent.helper.ts +3 -3
  90. package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts +230 -230
  91. package/src/subsystems/AgentManager/AgentData.service/connectors/CLIAgentDataConnector.class.ts +66 -66
  92. package/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts +145 -145
  93. package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts +39 -39
  94. package/src/subsystems/AgentManager/AgentData.service/index.ts +18 -18
  95. package/src/subsystems/AgentManager/AgentLogger.class.ts +301 -301
  96. package/src/subsystems/AgentManager/AgentRequest.class.ts +51 -51
  97. package/src/subsystems/AgentManager/AgentRuntime.class.ts +557 -557
  98. package/src/subsystems/AgentManager/AgentSSE.class.ts +101 -101
  99. package/src/subsystems/AgentManager/AgentSettings.class.ts +52 -52
  100. package/src/subsystems/AgentManager/Component.service/ComponentConnector.ts +32 -32
  101. package/src/subsystems/AgentManager/Component.service/connectors/LocalComponentConnector.class.ts +60 -60
  102. package/src/subsystems/AgentManager/Component.service/index.ts +11 -11
  103. package/src/subsystems/AgentManager/EmbodimentSettings.class.ts +47 -47
  104. package/src/subsystems/AgentManager/ForkedAgent.class.ts +154 -154
  105. package/src/subsystems/AgentManager/OSResourceMonitor.ts +77 -77
  106. package/src/subsystems/ComputeManager/Code.service/CodeConnector.ts +98 -98
  107. package/src/subsystems/ComputeManager/Code.service/connectors/AWSLambdaCode.class.ts +171 -172
  108. package/src/subsystems/ComputeManager/Code.service/connectors/ECMASandbox.class.ts +131 -131
  109. package/src/subsystems/ComputeManager/Code.service/index.ts +13 -13
  110. package/src/subsystems/IO/CLI.service/CLIConnector.ts +47 -47
  111. package/src/subsystems/IO/CLI.service/index.ts +9 -9
  112. package/src/subsystems/IO/Log.service/LogConnector.ts +32 -32
  113. package/src/subsystems/IO/Log.service/connectors/ConsoleLog.class.ts +28 -28
  114. package/src/subsystems/IO/Log.service/index.ts +13 -13
  115. package/src/subsystems/IO/NKV.service/NKVConnector.ts +43 -43
  116. package/src/subsystems/IO/NKV.service/connectors/NKVLocalStorage.class.ts +234 -234
  117. package/src/subsystems/IO/NKV.service/connectors/NKVRAM.class.ts +204 -204
  118. package/src/subsystems/IO/NKV.service/connectors/NKVRedis.class.ts +182 -182
  119. package/src/subsystems/IO/NKV.service/index.ts +14 -14
  120. package/src/subsystems/IO/Router.service/RouterConnector.ts +21 -21
  121. package/src/subsystems/IO/Router.service/connectors/ExpressRouter.class.ts +48 -48
  122. package/src/subsystems/IO/Router.service/connectors/NullRouter.class.ts +40 -40
  123. package/src/subsystems/IO/Router.service/index.ts +11 -11
  124. package/src/subsystems/IO/Storage.service/SmythFS.class.ts +488 -488
  125. package/src/subsystems/IO/Storage.service/StorageConnector.ts +66 -66
  126. package/src/subsystems/IO/Storage.service/connectors/LocalStorage.class.ts +327 -327
  127. package/src/subsystems/IO/Storage.service/connectors/S3Storage.class.ts +482 -482
  128. package/src/subsystems/IO/Storage.service/index.ts +13 -13
  129. package/src/subsystems/IO/VectorDB.service/VectorDBConnector.ts +108 -108
  130. package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +465 -465
  131. package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts +387 -387
  132. package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts +408 -408
  133. package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts +107 -107
  134. package/src/subsystems/IO/VectorDB.service/embed/GoogleEmbedding.ts +118 -118
  135. package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts +109 -109
  136. package/src/subsystems/IO/VectorDB.service/embed/index.ts +26 -26
  137. package/src/subsystems/IO/VectorDB.service/index.ts +14 -14
  138. package/src/subsystems/LLMManager/LLM.helper.ts +251 -251
  139. package/src/subsystems/LLMManager/LLM.inference.ts +345 -345
  140. package/src/subsystems/LLMManager/LLM.service/LLMConnector.ts +492 -492
  141. package/src/subsystems/LLMManager/LLM.service/LLMCredentials.helper.ts +171 -171
  142. package/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts +666 -666
  143. package/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts +407 -407
  144. package/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts +92 -92
  145. package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts +983 -983
  146. package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts +319 -319
  147. package/src/subsystems/LLMManager/LLM.service/connectors/Ollama.class.ts +361 -361
  148. package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts +257 -257
  149. package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts +430 -430
  150. package/src/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.ts +503 -503
  151. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ChatCompletionsApiInterface.ts +524 -524
  152. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/OpenAIApiInterface.ts +100 -100
  153. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/OpenAIApiInterfaceFactory.ts +81 -81
  154. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.ts +1145 -1145
  155. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/constants.ts +13 -13
  156. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/index.ts +4 -4
  157. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/utils.ts +11 -11
  158. package/src/subsystems/LLMManager/LLM.service/connectors/openai/types.ts +32 -32
  159. package/src/subsystems/LLMManager/LLM.service/connectors/xAI.class.ts +478 -478
  160. package/src/subsystems/LLMManager/LLM.service/index.ts +47 -47
  161. package/src/subsystems/LLMManager/ModelsProvider.service/ModelsProviderConnector.ts +303 -303
  162. package/src/subsystems/LLMManager/ModelsProvider.service/connectors/JSONModelsProvider.class.ts +271 -271
  163. package/src/subsystems/LLMManager/ModelsProvider.service/index.ts +11 -11
  164. package/src/subsystems/LLMManager/custom-models.ts +854 -854
  165. package/src/subsystems/LLMManager/models.ts +2540 -2540
  166. package/src/subsystems/LLMManager/paramMappings.ts +69 -69
  167. package/src/subsystems/MemoryManager/Cache.service/CacheConnector.ts +86 -86
  168. package/src/subsystems/MemoryManager/Cache.service/connectors/LocalStorageCache.class.ts +297 -297
  169. package/src/subsystems/MemoryManager/Cache.service/connectors/RAMCache.class.ts +214 -214
  170. package/src/subsystems/MemoryManager/Cache.service/connectors/RedisCache.class.ts +252 -252
  171. package/src/subsystems/MemoryManager/Cache.service/connectors/S3Cache.class.ts +373 -373
  172. package/src/subsystems/MemoryManager/Cache.service/index.ts +15 -15
  173. package/src/subsystems/MemoryManager/LLMCache.ts +72 -72
  174. package/src/subsystems/MemoryManager/LLMContext.ts +124 -124
  175. package/src/subsystems/MemoryManager/LLMMemory.service/LLMMemoryConnector.ts +26 -26
  176. package/src/subsystems/MemoryManager/RuntimeContext.ts +277 -277
  177. package/src/subsystems/Security/AccessControl/ACL.class.ts +208 -208
  178. package/src/subsystems/Security/AccessControl/AccessCandidate.class.ts +82 -82
  179. package/src/subsystems/Security/AccessControl/AccessRequest.class.ts +52 -52
  180. package/src/subsystems/Security/Account.service/AccountConnector.ts +44 -44
  181. package/src/subsystems/Security/Account.service/connectors/DummyAccount.class.ts +130 -130
  182. package/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts +170 -170
  183. package/src/subsystems/Security/Account.service/connectors/MySQLAccount.class.ts +76 -76
  184. package/src/subsystems/Security/Account.service/index.ts +14 -14
  185. package/src/subsystems/Security/Credentials.helper.ts +62 -62
  186. package/src/subsystems/Security/ManagedVault.service/ManagedVaultConnector.ts +38 -38
  187. package/src/subsystems/Security/ManagedVault.service/connectors/NullManagedVault.class.ts +53 -53
  188. package/src/subsystems/Security/ManagedVault.service/connectors/SecretManagerManagedVault.ts +154 -154
  189. package/src/subsystems/Security/ManagedVault.service/index.ts +12 -12
  190. package/src/subsystems/Security/SecureConnector.class.ts +110 -110
  191. package/src/subsystems/Security/Vault.service/Vault.helper.ts +30 -30
  192. package/src/subsystems/Security/Vault.service/VaultConnector.ts +29 -29
  193. package/src/subsystems/Security/Vault.service/connectors/HashicorpVault.class.ts +46 -46
  194. package/src/subsystems/Security/Vault.service/connectors/JSONFileVault.class.ts +221 -221
  195. package/src/subsystems/Security/Vault.service/connectors/NullVault.class.ts +54 -54
  196. package/src/subsystems/Security/Vault.service/connectors/SecretsManager.class.ts +140 -140
  197. package/src/subsystems/Security/Vault.service/index.ts +12 -12
  198. package/src/types/ACL.types.ts +104 -104
  199. package/src/types/AWS.types.ts +10 -10
  200. package/src/types/Agent.types.ts +61 -61
  201. package/src/types/AgentLogger.types.ts +17 -17
  202. package/src/types/Cache.types.ts +1 -1
  203. package/src/types/Common.types.ts +2 -2
  204. package/src/types/LLM.types.ts +520 -520
  205. package/src/types/Redis.types.ts +8 -8
  206. package/src/types/SRE.types.ts +64 -64
  207. package/src/types/Security.types.ts +14 -14
  208. package/src/types/Storage.types.ts +5 -5
  209. package/src/types/VectorDB.types.ts +86 -86
  210. package/src/utils/base64.utils.ts +275 -275
  211. package/src/utils/cli.utils.ts +68 -68
  212. package/src/utils/data.utils.ts +322 -322
  213. package/src/utils/date-time.utils.ts +22 -22
  214. package/src/utils/general.utils.ts +238 -238
  215. package/src/utils/index.ts +12 -12
  216. package/src/utils/lazy-client.ts +261 -261
  217. package/src/utils/numbers.utils.ts +13 -13
  218. package/src/utils/oauth.utils.ts +35 -35
  219. package/src/utils/string.utils.ts +414 -414
  220. package/src/utils/url.utils.ts +19 -19
  221. package/src/utils/validation.utils.ts +74 -74
  222. package/dist/bundle-analysis-lazy.html +0 -4949
  223. package/dist/bundle-analysis.html +0 -4949
  224. package/dist/types/Components/Triggers/GmailTrigger.class.d.ts +0 -13
  225. package/dist/types/Components/Triggers/Trigger.class.d.ts +0 -3
  226. package/dist/types/helpers/AIPerformanceAnalyzer.helper.d.ts +0 -45
  227. package/dist/types/helpers/AIPerformanceCollector.helper.d.ts +0 -111
  228. package/dist/types/subsystems/IO/Storage.service/connectors/AzureBlobStorage.class.d.ts +0 -211
  229. package/dist/types/subsystems/IO/VectorDB.service/connectors/WeaviateVectorDB.class.d.ts +0 -187
  230. package/dist/types/subsystems/PerformanceManager/Performance.service/PerformanceConnector.d.ts +0 -102
  231. package/dist/types/subsystems/PerformanceManager/Performance.service/connectors/LocalPerformanceConnector.class.d.ts +0 -100
  232. package/dist/types/subsystems/PerformanceManager/Performance.service/index.d.ts +0 -22
  233. package/dist/types/types/Performance.types.d.ts +0 -468
  234. package/dist/types/utils/package-manager.utils.d.ts +0 -26
@@ -1,46 +1,46 @@
1
- const mimeTypeCategories = {
2
- binary: [
3
- 'image/',
4
- 'multipart/form-data',
5
- 'video/',
6
- 'application/msword',
7
- 'application/octet-stream',
8
- 'application/pdf',
9
- 'application/vnd.ms-excel',
10
- 'application/vnd.ms-powerpoint',
11
- 'application/vnd.oasis.opendocument.text',
12
- 'application/vnd.openxmlformats-officedocument',
13
- 'application/zip',
14
- 'application/x-7z-compressed',
15
- 'application/x-rar-compressed',
16
- 'application/x-tar',
17
- 'application/x-bzip',
18
- 'application/x-bzip2',
19
- 'application/x-gzip',
20
- 'application/vnd.android.package-archive',
21
- 'application/vnd.visio',
22
- 'application/x-deb',
23
- 'application/x-rpm',
24
- 'application/x-executable',
25
- 'font/ttf',
26
- 'font/otf',
27
- 'font/woff',
28
- 'font/woff2',
29
- 'model/',
30
- ],
31
- json: ['application/graphql', 'application/json', 'application/ld+json', 'application/vnd.api+json'],
32
- text: [
33
- 'text/', //all starting with text/
34
- 'application/xml',
35
- 'application/xhtml+xml',
36
- 'application/csv',
37
- 'application/x-www-form-urlencoded',
38
- 'application/x-yaml',
39
- 'application/yaml',
40
- 'application/javascript',
41
- 'application/sql',
42
- 'application/rtf',
43
- ],
44
- };
45
-
46
- export default mimeTypeCategories;
1
+ const mimeTypeCategories = {
2
+ binary: [
3
+ 'image/',
4
+ 'multipart/form-data',
5
+ 'video/',
6
+ 'application/msword',
7
+ 'application/octet-stream',
8
+ 'application/pdf',
9
+ 'application/vnd.ms-excel',
10
+ 'application/vnd.ms-powerpoint',
11
+ 'application/vnd.oasis.opendocument.text',
12
+ 'application/vnd.openxmlformats-officedocument',
13
+ 'application/zip',
14
+ 'application/x-7z-compressed',
15
+ 'application/x-rar-compressed',
16
+ 'application/x-tar',
17
+ 'application/x-bzip',
18
+ 'application/x-bzip2',
19
+ 'application/x-gzip',
20
+ 'application/vnd.android.package-archive',
21
+ 'application/vnd.visio',
22
+ 'application/x-deb',
23
+ 'application/x-rpm',
24
+ 'application/x-executable',
25
+ 'font/ttf',
26
+ 'font/otf',
27
+ 'font/woff',
28
+ 'font/woff2',
29
+ 'model/',
30
+ ],
31
+ json: ['application/graphql', 'application/json', 'application/ld+json', 'application/vnd.api+json'],
32
+ text: [
33
+ 'text/', //all starting with text/
34
+ 'application/xml',
35
+ 'application/xhtml+xml',
36
+ 'application/csv',
37
+ 'application/x-www-form-urlencoded',
38
+ 'application/x-yaml',
39
+ 'application/yaml',
40
+ 'application/javascript',
41
+ 'application/sql',
42
+ 'application/rtf',
43
+ ],
44
+ };
45
+
46
+ export default mimeTypeCategories;
@@ -1,167 +1,167 @@
1
- import { Agent } from '@sre/AgentManager/Agent.class';
2
- import { REQUEST_CONTENT_TYPES } from '@sre/constants';
3
- import { JSONContent } from '@sre/helpers/JsonContent.helper';
4
- import { TemplateString } from '@sre/helpers/TemplateString.helper';
5
- import { BinaryInput } from '@sre/helpers/BinaryInput.helper';
6
- import FormData from 'form-data';
7
- import { Readable } from 'stream';
8
-
9
- export async function parseData(input: any, config, agent: Agent): Promise<{ data: any; headers: any }> {
10
- const teamId = agent ? agent.teamId : null;
11
- const templateSettings = config?.template?.settings || {};
12
- const contentType = config?.data?.contentType || REQUEST_CONTENT_TYPES.none;
13
-
14
- let body = typeof config?.data?.body === 'string' ? config?.data?.body?.trim() : config?.data?.body;
15
- if (!body) {
16
- return { data: null, headers: {} };
17
- }
18
-
19
- // Parse component template variables
20
- if (config.data._templateVars && templateSettings) {
21
- //prettier-ignore
22
- body = await TemplateString(body)
23
- .parseComponentTemplateVarsAsync(templateSettings) // replace component template vars with their IDs (this turns the string parses into an async parser)
24
- .asyncResult; //returns parser promise
25
- }
26
-
27
- // Parse vault keys
28
- body = await TemplateString(body).parseTeamKeysAsync(teamId).asyncResult;
29
-
30
- // Handle different content types
31
- const handlers: Record<string, (parsedBody: any, input: any, config, agent: Agent) => any> = {
32
- [REQUEST_CONTENT_TYPES.json]: handleJson,
33
- [REQUEST_CONTENT_TYPES.urlEncodedFormData]: handleUrlEncoded,
34
- [REQUEST_CONTENT_TYPES.multipartFormData]: handleMultipartFormData,
35
- [REQUEST_CONTENT_TYPES.binary]: handleBinary,
36
- [REQUEST_CONTENT_TYPES.text]: handleText,
37
- [REQUEST_CONTENT_TYPES.none]: handleNone,
38
- };
39
-
40
- const handler = handlers[contentType] || handleNone;
41
- const { data = null, headers = {} } = await handler(body, input, config, agent);
42
-
43
- //const jsonBody: any = JSONContent(data).tryParse();
44
- return { data, headers };
45
- }
46
-
47
- async function handleJson(body: any, input: any, config, agent: Agent) {
48
- // Parse template and input variables
49
- //prettier-ignore
50
- const data = TemplateString(body)
51
- .parse(config.data._templateVars) //parse Template variables first (if any)
52
- .parse(input) //parse inputs
53
- .clean().result; //clean up the remaining unparsed values
54
-
55
- const jsonBody: any = JSONContent(data).tryParse();
56
- return { data: jsonBody };
57
- }
58
-
59
- async function handleUrlEncoded(body: any, input: any, config, agent: Agent) {
60
- const data = TemplateString(body)
61
- .parse(config.data._templateVars) //parse Template variables first (if any)
62
- .parse(input) //parse inputs
63
- .clean().result; //clean up the remaining unparsed values
64
-
65
- const jsonData: any = JSONContent(data).tryParse();
66
-
67
- if (typeof jsonData === 'object') {
68
- const params = new URLSearchParams();
69
- for (const key in jsonData) {
70
- params.append(key, String(jsonData[key]));
71
- }
72
- return { data: params.toString() };
73
- }
74
-
75
- return { data: jsonData };
76
- }
77
-
78
- async function handleMultipartFormData(body: any, input: any, config, agent: Agent) {
79
- const formData = new FormData();
80
-
81
- const _body = typeof body === 'string' ? JSON.parse(body) : body;
82
-
83
- for (const key in _body) {
84
- let value = _body[key];
85
- value = typeof value === 'boolean' ? String(value) : value;
86
-
87
- value = TemplateString(value).parseRaw(input).result;
88
-
89
- // * Note: It's important to check if the value is an instance of BinaryInput first.
90
- // Otherwise, condition like (value && typeof value === 'object' && value?.url)
91
- // might be true and lead to incorrect results.
92
- if (value instanceof BinaryInput) {
93
- const buffer = await value.getBuffer();
94
- const bufferStream = new Readable();
95
- bufferStream.push(buffer || null);
96
- bufferStream.push(null);
97
-
98
- const filename = (await value.getName()) || key;
99
- formData.append(key, bufferStream, {
100
- filename,
101
- contentType: value.mimetype,
102
- });
103
- } else if (value && typeof value === 'object' && value?.url) {
104
- // Retro compatibility with old binary data structure {url: '...', mimetype: '...', url: 'http(s)://...'}
105
- const binaryInput = await BinaryInput.from(value.url, '', value?.mimetype);
106
- const buffer = await binaryInput.getBuffer();
107
-
108
- const bufferStream = new Readable();
109
- bufferStream.push(buffer || null);
110
- bufferStream.push(null);
111
-
112
- const filename = (await binaryInput.getName()) || key;
113
- formData.append(key, bufferStream, {
114
- filename,
115
- contentType: binaryInput.mimetype,
116
- });
117
- } else {
118
- value = TemplateString(value)
119
- .parse(config.data._templateVars) //parse Template variables first (if any)
120
- .parse(input)
121
- .clean().result;
122
-
123
- if (value) {
124
- formData.append(key, value);
125
- }
126
-
127
- //formData.append(key, typeof value === 'boolean' ? String(value) : value);
128
- }
129
- }
130
- return { data: formData, headers: formData.getHeaders() };
131
- }
132
-
133
- async function handleBinary(body: any, input: any, config, agent: Agent) {
134
- const value: any = TemplateString(body).parseRaw(input).result;
135
-
136
- // * Note: It's important to check if the value is an instance of BinaryInput first.
137
- // Otherwise, condition like (value && typeof value === 'object' && value?.url)
138
- // might be true and lead to incorrect results.
139
- if (value && value instanceof BinaryInput) {
140
- const buffer = await value.getBuffer();
141
- return { data: buffer, headers: { 'Content-Type': value.mimetype } };
142
- } else if (value && typeof value === 'object' && value?.url) {
143
- // Retro compatibility with old binary data structure {url: '...', mimetype: '...', url: 'http(s)://...'}
144
- const binaryInput = await BinaryInput.from(value.url, '', value?.mimetype);
145
- const buffer = await binaryInput.getBuffer();
146
-
147
- return { data: buffer, headers: { 'Content-Type': binaryInput.mimetype } };
148
- }
149
-
150
- return { data: Buffer.from([]), headers: {} };
151
- }
152
-
153
- async function handleNone(body: any, input: any, config, agent: Agent) {
154
- //FIXME: try to guess the content type from headers content-type and data
155
-
156
- return { data: typeof body === 'string' ? body : JSON.stringify(body), headers: {} };
157
- }
158
- function handleText(body: any, input: any, config: any, agent: Agent) {
159
- // Parse template and input variables
160
- //prettier-ignore
161
- const data = TemplateString(body)
162
- .parse(config.data._templateVars) //parse Template variables first (if any)
163
- .parse(input) //parse inputs
164
- .clean().result; //clean up the remaining unparsed values
165
-
166
- return { data };
167
- }
1
+ import { Agent } from '@sre/AgentManager/Agent.class';
2
+ import { REQUEST_CONTENT_TYPES } from '@sre/constants';
3
+ import { JSONContent } from '@sre/helpers/JsonContent.helper';
4
+ import { TemplateString } from '@sre/helpers/TemplateString.helper';
5
+ import { BinaryInput } from '@sre/helpers/BinaryInput.helper';
6
+ import FormData from 'form-data';
7
+ import { Readable } from 'stream';
8
+
9
+ export async function parseData(input: any, config, agent: Agent): Promise<{ data: any; headers: any }> {
10
+ const teamId = agent ? agent.teamId : null;
11
+ const templateSettings = config?.template?.settings || {};
12
+ const contentType = config?.data?.contentType || REQUEST_CONTENT_TYPES.none;
13
+
14
+ let body = typeof config?.data?.body === 'string' ? config?.data?.body?.trim() : config?.data?.body;
15
+ if (!body) {
16
+ return { data: null, headers: {} };
17
+ }
18
+
19
+ // Parse component template variables
20
+ if (config.data._templateVars && templateSettings) {
21
+ //prettier-ignore
22
+ body = await TemplateString(body)
23
+ .parseComponentTemplateVarsAsync(templateSettings) // replace component template vars with their IDs (this turns the string parses into an async parser)
24
+ .asyncResult; //returns parser promise
25
+ }
26
+
27
+ // Parse vault keys
28
+ body = await TemplateString(body).parseTeamKeysAsync(teamId).asyncResult;
29
+
30
+ // Handle different content types
31
+ const handlers: Record<string, (parsedBody: any, input: any, config, agent: Agent) => any> = {
32
+ [REQUEST_CONTENT_TYPES.json]: handleJson,
33
+ [REQUEST_CONTENT_TYPES.urlEncodedFormData]: handleUrlEncoded,
34
+ [REQUEST_CONTENT_TYPES.multipartFormData]: handleMultipartFormData,
35
+ [REQUEST_CONTENT_TYPES.binary]: handleBinary,
36
+ [REQUEST_CONTENT_TYPES.text]: handleText,
37
+ [REQUEST_CONTENT_TYPES.none]: handleNone,
38
+ };
39
+
40
+ const handler = handlers[contentType] || handleNone;
41
+ const { data = null, headers = {} } = await handler(body, input, config, agent);
42
+
43
+ //const jsonBody: any = JSONContent(data).tryParse();
44
+ return { data, headers };
45
+ }
46
+
47
+ async function handleJson(body: any, input: any, config, agent: Agent) {
48
+ // Parse template and input variables
49
+ //prettier-ignore
50
+ const data = TemplateString(body)
51
+ .parse(config.data._templateVars) //parse Template variables first (if any)
52
+ .parse(input) //parse inputs
53
+ .clean().result; //clean up the remaining unparsed values
54
+
55
+ const jsonBody: any = JSONContent(data).tryParse();
56
+ return { data: jsonBody };
57
+ }
58
+
59
+ async function handleUrlEncoded(body: any, input: any, config, agent: Agent) {
60
+ const data = TemplateString(body)
61
+ .parse(config.data._templateVars) //parse Template variables first (if any)
62
+ .parse(input) //parse inputs
63
+ .clean().result; //clean up the remaining unparsed values
64
+
65
+ const jsonData: any = JSONContent(data).tryParse();
66
+
67
+ if (typeof jsonData === 'object') {
68
+ const params = new URLSearchParams();
69
+ for (const key in jsonData) {
70
+ params.append(key, String(jsonData[key]));
71
+ }
72
+ return { data: params.toString() };
73
+ }
74
+
75
+ return { data: jsonData };
76
+ }
77
+
78
+ async function handleMultipartFormData(body: any, input: any, config, agent: Agent) {
79
+ const formData = new FormData();
80
+
81
+ const _body = typeof body === 'string' ? JSON.parse(body) : body;
82
+
83
+ for (const key in _body) {
84
+ let value = _body[key];
85
+ value = typeof value === 'boolean' ? String(value) : value;
86
+
87
+ value = TemplateString(value).parseRaw(input).result;
88
+
89
+ // * Note: It's important to check if the value is an instance of BinaryInput first.
90
+ // Otherwise, condition like (value && typeof value === 'object' && value?.url)
91
+ // might be true and lead to incorrect results.
92
+ if (value instanceof BinaryInput) {
93
+ const buffer = await value.getBuffer();
94
+ const bufferStream = new Readable();
95
+ bufferStream.push(buffer || null);
96
+ bufferStream.push(null);
97
+
98
+ const filename = (await value.getName()) || key;
99
+ formData.append(key, bufferStream, {
100
+ filename,
101
+ contentType: value.mimetype,
102
+ });
103
+ } else if (value && typeof value === 'object' && value?.url) {
104
+ // Retro compatibility with old binary data structure {url: '...', mimetype: '...', url: 'http(s)://...'}
105
+ const binaryInput = await BinaryInput.from(value.url, '', value?.mimetype);
106
+ const buffer = await binaryInput.getBuffer();
107
+
108
+ const bufferStream = new Readable();
109
+ bufferStream.push(buffer || null);
110
+ bufferStream.push(null);
111
+
112
+ const filename = (await binaryInput.getName()) || key;
113
+ formData.append(key, bufferStream, {
114
+ filename,
115
+ contentType: binaryInput.mimetype,
116
+ });
117
+ } else {
118
+ value = TemplateString(value)
119
+ .parse(config.data._templateVars) //parse Template variables first (if any)
120
+ .parse(input)
121
+ .clean().result;
122
+
123
+ if (value) {
124
+ formData.append(key, value);
125
+ }
126
+
127
+ //formData.append(key, typeof value === 'boolean' ? String(value) : value);
128
+ }
129
+ }
130
+ return { data: formData, headers: formData.getHeaders() };
131
+ }
132
+
133
+ async function handleBinary(body: any, input: any, config, agent: Agent) {
134
+ const value: any = TemplateString(body).parseRaw(input).result;
135
+
136
+ // * Note: It's important to check if the value is an instance of BinaryInput first.
137
+ // Otherwise, condition like (value && typeof value === 'object' && value?.url)
138
+ // might be true and lead to incorrect results.
139
+ if (value && value instanceof BinaryInput) {
140
+ const buffer = await value.getBuffer();
141
+ return { data: buffer, headers: { 'Content-Type': value.mimetype } };
142
+ } else if (value && typeof value === 'object' && value?.url) {
143
+ // Retro compatibility with old binary data structure {url: '...', mimetype: '...', url: 'http(s)://...'}
144
+ const binaryInput = await BinaryInput.from(value.url, '', value?.mimetype);
145
+ const buffer = await binaryInput.getBuffer();
146
+
147
+ return { data: buffer, headers: { 'Content-Type': binaryInput.mimetype } };
148
+ }
149
+
150
+ return { data: Buffer.from([]), headers: {} };
151
+ }
152
+
153
+ async function handleNone(body: any, input: any, config, agent: Agent) {
154
+ //FIXME: try to guess the content type from headers content-type and data
155
+
156
+ return { data: typeof body === 'string' ? body : JSON.stringify(body), headers: {} };
157
+ }
158
+ function handleText(body: any, input: any, config: any, agent: Agent) {
159
+ // Parse template and input variables
160
+ //prettier-ignore
161
+ const data = TemplateString(body)
162
+ .parse(config.data._templateVars) //parse Template variables first (if any)
163
+ .parse(input) //parse inputs
164
+ .clean().result; //clean up the remaining unparsed values
165
+
166
+ return { data };
167
+ }
@@ -1,41 +1,41 @@
1
- import { Agent } from '@sre/AgentManager/Agent.class';
2
- import { REQUEST_CONTENT_TYPES } from '@sre/constants';
3
- import { JSONContent } from '@sre/helpers/JsonContent.helper';
4
- import { TemplateString } from '@sre/helpers/TemplateString.helper';
5
- import { AxiosHeaders } from 'axios';
6
-
7
- export async function parseHeaders(input, config, agent: Agent) {
8
- const teamId = agent ? agent.teamId : null;
9
- const templateSettings = config?.template?.settings || {};
10
- const contentType = config?.data?.contentType || REQUEST_CONTENT_TYPES.none;
11
- let headers = typeof config?.data?.headers == 'object' ? JSON.stringify(config?.data?.headers) : config?.data?.headers || '{}';
12
-
13
- //parse component template vars
14
- if (config.data._templateVars && templateSettings) {
15
- headers = await TemplateString(headers).parseComponentTemplateVarsAsync(templateSettings).asyncResult; // replaces component template vars with their IDs (this turns the string parses into an async parser) // replaces IDs with actual values then returns parser promise
16
-
17
- headers = await TemplateString(headers).parse(config.data._templateVars).result;
18
- }
19
-
20
- //parse vault keys
21
- headers = await TemplateString(headers).parseTeamKeysAsync(teamId).asyncResult;
22
-
23
- //parse input variables and clean up the remaining unparsed values
24
- headers = TemplateString(headers).parse(input).clean().result;
25
-
26
- //parse headers as json
27
- let jsonHeaders: any = JSONContent(headers).tryParse();
28
- if (typeof jsonHeaders !== 'object') {
29
- jsonHeaders = { 'x-smyth-error': 'Error parsing headers' };
30
- }
31
-
32
- //normalize headers key names to lowercase
33
- jsonHeaders = Object.fromEntries(Object.entries(jsonHeaders).map(([key, value]) => [key.toLowerCase(), value]));
34
-
35
- //if headers does not contain content-type, add it
36
- if (!jsonHeaders['content-type'] && contentType !== 'none') {
37
- jsonHeaders['content-type'] = contentType;
38
- }
39
-
40
- return new AxiosHeaders(jsonHeaders);
41
- }
1
+ import { Agent } from '@sre/AgentManager/Agent.class';
2
+ import { REQUEST_CONTENT_TYPES } from '@sre/constants';
3
+ import { JSONContent } from '@sre/helpers/JsonContent.helper';
4
+ import { TemplateString } from '@sre/helpers/TemplateString.helper';
5
+ import { AxiosHeaders } from 'axios';
6
+
7
+ export async function parseHeaders(input, config, agent: Agent) {
8
+ const teamId = agent ? agent.teamId : null;
9
+ const templateSettings = config?.template?.settings || {};
10
+ const contentType = config?.data?.contentType || REQUEST_CONTENT_TYPES.none;
11
+ let headers = typeof config?.data?.headers == 'object' ? JSON.stringify(config?.data?.headers) : config?.data?.headers || '{}';
12
+
13
+ //parse component template vars
14
+ if (config.data._templateVars && templateSettings) {
15
+ headers = await TemplateString(headers).parseComponentTemplateVarsAsync(templateSettings).asyncResult; // replaces component template vars with their IDs (this turns the string parses into an async parser) // replaces IDs with actual values then returns parser promise
16
+
17
+ headers = await TemplateString(headers).parse(config.data._templateVars).result;
18
+ }
19
+
20
+ //parse vault keys
21
+ headers = await TemplateString(headers).parseTeamKeysAsync(teamId).asyncResult;
22
+
23
+ //parse input variables and clean up the remaining unparsed values
24
+ headers = TemplateString(headers).parse(input).clean().result;
25
+
26
+ //parse headers as json
27
+ let jsonHeaders: any = JSONContent(headers).tryParse();
28
+ if (typeof jsonHeaders !== 'object') {
29
+ jsonHeaders = { 'x-smyth-error': 'Error parsing headers' };
30
+ }
31
+
32
+ //normalize headers key names to lowercase
33
+ jsonHeaders = Object.fromEntries(Object.entries(jsonHeaders).map(([key, value]) => [key.toLowerCase(), value]));
34
+
35
+ //if headers does not contain content-type, add it
36
+ if (!jsonHeaders['content-type'] && contentType !== 'none') {
37
+ jsonHeaders['content-type'] = contentType;
38
+ }
39
+
40
+ return new AxiosHeaders(jsonHeaders);
41
+ }
@@ -1,68 +1,68 @@
1
- import { AxiosProxyConfig } from 'axios';
2
- import { SocksProxyAgent } from 'socks-proxy-agent';
3
-
4
- import { Agent } from '@sre/AgentManager/Agent.class';
5
- import { TemplateString } from '@sre/helpers/TemplateString.helper';
6
-
7
- export async function parseProxy(input, config, agent: Agent): Promise<AxiosProxyConfig | SocksProxyAgent | false> {
8
- const teamId = agent ? agent.teamId : null;
9
- const templateSettings = config?.template?.settings || {};
10
-
11
- let proxy = config?.data?.proxy;
12
-
13
- if (!proxy) {
14
- return false;
15
- }
16
-
17
- proxy = decodeURIComponent(proxy); //decode the url in order to parse the template vars
18
-
19
- //parse component template vars
20
- if (config.data._templateVars && templateSettings) {
21
- proxy = await TemplateString(proxy)
22
- .parseComponentTemplateVarsAsync(templateSettings) // replaces component template vars with their IDs (this turns the string parses into an async parser)
23
- .parse(config.data._templateVars).asyncResult; // replaces IDs with actual values then returns parser promise
24
- }
25
-
26
- //parse vault keys
27
- proxy = await TemplateString(proxy).parseTeamKeysAsync(teamId).asyncResult;
28
-
29
- //parse input variables and clean up the remaining unparsed values
30
- proxy = TemplateString(proxy).parse(input).clean().result;
31
-
32
- const proxyList = proxy.split(/\n|\\n/).filter((p) => p) || [];
33
-
34
- const randomIdx = Math.floor(Math.random() * proxyList?.length);
35
- const proxyUrl = proxyList[randomIdx]?.trim();
36
-
37
- //URL will take care of encoding the url properly
38
- const urlObj = new URL(proxyUrl);
39
- const protocol = urlObj.protocol.replace(':', ''); // As urlObj.protocol is like 'http:'
40
-
41
- let proxyConfig: AxiosProxyConfig | SocksProxyAgent;
42
-
43
- if (urlObj.protocol.startsWith('socks')) {
44
- let proxyUrlString = `${protocol}://${urlObj.hostname}:${urlObj.port}`;
45
-
46
- if ((protocol === 'socks4' || protocol === 'socks4a') && urlObj.username) {
47
- proxyUrlString = `${protocol}://${urlObj.username}@${urlObj.hostname}:${urlObj.port}`;
48
- } else if (protocol === 'socks5' && urlObj.username) {
49
- proxyUrlString = `${protocol}://${urlObj.username}:${urlObj.password}@${urlObj.hostname}:${urlObj.port}`;
50
- }
51
-
52
- proxyConfig = new SocksProxyAgent(proxyUrlString);
53
- } else {
54
- proxyConfig = {
55
- protocol,
56
- host: urlObj.hostname,
57
- port: parseInt(urlObj.port),
58
- auth: urlObj.username
59
- ? {
60
- username: urlObj.username,
61
- password: urlObj.password,
62
- }
63
- : undefined,
64
- };
65
- }
66
-
67
- return proxyConfig;
68
- }
1
+ import { AxiosProxyConfig } from 'axios';
2
+ import { SocksProxyAgent } from 'socks-proxy-agent';
3
+
4
+ import { Agent } from '@sre/AgentManager/Agent.class';
5
+ import { TemplateString } from '@sre/helpers/TemplateString.helper';
6
+
7
+ export async function parseProxy(input, config, agent: Agent): Promise<AxiosProxyConfig | SocksProxyAgent | false> {
8
+ const teamId = agent ? agent.teamId : null;
9
+ const templateSettings = config?.template?.settings || {};
10
+
11
+ let proxy = config?.data?.proxy;
12
+
13
+ if (!proxy) {
14
+ return false;
15
+ }
16
+
17
+ proxy = decodeURIComponent(proxy); //decode the url in order to parse the template vars
18
+
19
+ //parse component template vars
20
+ if (config.data._templateVars && templateSettings) {
21
+ proxy = await TemplateString(proxy)
22
+ .parseComponentTemplateVarsAsync(templateSettings) // replaces component template vars with their IDs (this turns the string parses into an async parser)
23
+ .parse(config.data._templateVars).asyncResult; // replaces IDs with actual values then returns parser promise
24
+ }
25
+
26
+ //parse vault keys
27
+ proxy = await TemplateString(proxy).parseTeamKeysAsync(teamId).asyncResult;
28
+
29
+ //parse input variables and clean up the remaining unparsed values
30
+ proxy = TemplateString(proxy).parse(input).clean().result;
31
+
32
+ const proxyList = proxy.split(/\n|\\n/).filter((p) => p) || [];
33
+
34
+ const randomIdx = Math.floor(Math.random() * proxyList?.length);
35
+ const proxyUrl = proxyList[randomIdx]?.trim();
36
+
37
+ //URL will take care of encoding the url properly
38
+ const urlObj = new URL(proxyUrl);
39
+ const protocol = urlObj.protocol.replace(':', ''); // As urlObj.protocol is like 'http:'
40
+
41
+ let proxyConfig: AxiosProxyConfig | SocksProxyAgent;
42
+
43
+ if (urlObj.protocol.startsWith('socks')) {
44
+ let proxyUrlString = `${protocol}://${urlObj.hostname}:${urlObj.port}`;
45
+
46
+ if ((protocol === 'socks4' || protocol === 'socks4a') && urlObj.username) {
47
+ proxyUrlString = `${protocol}://${urlObj.username}@${urlObj.hostname}:${urlObj.port}`;
48
+ } else if (protocol === 'socks5' && urlObj.username) {
49
+ proxyUrlString = `${protocol}://${urlObj.username}:${urlObj.password}@${urlObj.hostname}:${urlObj.port}`;
50
+ }
51
+
52
+ proxyConfig = new SocksProxyAgent(proxyUrlString);
53
+ } else {
54
+ proxyConfig = {
55
+ protocol,
56
+ host: urlObj.hostname,
57
+ port: parseInt(urlObj.port),
58
+ auth: urlObj.username
59
+ ? {
60
+ username: urlObj.username,
61
+ password: urlObj.password,
62
+ }
63
+ : undefined,
64
+ };
65
+ }
66
+
67
+ return proxyConfig;
68
+ }