@smythos/sre 1.6.14 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/CHANGELOG +15 -0
  2. package/dist/index.js +52 -46
  3. package/dist/index.js.map +1 -1
  4. package/dist/types/Components/APIEndpoint.class.d.ts +2 -8
  5. package/dist/types/Components/Component.class.d.ts +9 -0
  6. package/dist/types/Components/Triggers/Gmail.trigger.d.ts +0 -17
  7. package/dist/types/Components/Triggers/JobScheduler.trigger.d.ts +10 -0
  8. package/dist/types/Components/Triggers/Trigger.class.d.ts +11 -0
  9. package/dist/types/Components/index.d.ts +6 -0
  10. package/dist/types/Core/Connector.class.d.ts +1 -0
  11. package/dist/types/Core/ConnectorsService.d.ts +2 -0
  12. package/dist/types/Core/HookService.d.ts +1 -1
  13. package/dist/types/helpers/Conversation.helper.d.ts +2 -0
  14. package/dist/types/helpers/Crypto.helper.d.ts +8 -0
  15. package/dist/types/index.d.ts +13 -0
  16. package/dist/types/subsystems/AgentManager/Agent.class.d.ts +4 -2
  17. package/dist/types/subsystems/AgentManager/AgentData.service/AgentDataConnector.d.ts +13 -0
  18. package/dist/types/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.d.ts +1 -4
  19. package/dist/types/subsystems/AgentManager/Scheduler.service/Job.class.d.ts +29 -6
  20. package/dist/types/subsystems/AgentManager/Scheduler.service/SchedulerConnector.d.ts +11 -3
  21. package/dist/types/subsystems/AgentManager/Scheduler.service/connectors/LocalScheduler.class.d.ts +31 -7
  22. package/dist/types/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.d.ts +2 -5
  23. package/dist/types/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.d.ts +3 -6
  24. package/dist/types/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.d.ts +7 -0
  25. package/dist/types/subsystems/LLMManager/LLM.service/connectors/xAI.class.d.ts +2 -5
  26. package/dist/types/types/Agent.types.d.ts +1 -0
  27. package/dist/types/types/LLM.types.d.ts +2 -5
  28. package/dist/types/types/SRE.types.d.ts +4 -1
  29. package/package.json +6 -2
  30. package/src/Components/APICall/OAuth.helper.ts +30 -35
  31. package/src/Components/APIEndpoint.class.ts +25 -6
  32. package/src/Components/Component.class.ts +11 -0
  33. package/src/Components/Triggers/Gmail.trigger.ts +282 -0
  34. package/src/Components/Triggers/JobScheduler.trigger.ts +45 -0
  35. package/src/Components/Triggers/README.md +3 -0
  36. package/src/Components/Triggers/Trigger.class.ts +101 -0
  37. package/src/Components/Triggers/WhatsApp.trigger.ts +219 -0
  38. package/src/Components/index.ts +8 -0
  39. package/src/Core/AgentProcess.helper.ts +4 -6
  40. package/src/Core/Connector.class.ts +11 -3
  41. package/src/Core/ConnectorsService.ts +5 -0
  42. package/src/Core/ExternalEventsReceiver.ts +317 -0
  43. package/src/Core/HookService.ts +20 -6
  44. package/src/Core/SmythRuntime.class.ts +7 -0
  45. package/src/Core/SystemEvents.ts +17 -0
  46. package/src/Core/boot.ts +2 -0
  47. package/src/helpers/Conversation.helper.ts +35 -11
  48. package/src/helpers/Crypto.helper.ts +28 -0
  49. package/src/index.ts +208 -195
  50. package/src/index.ts.bak +208 -195
  51. package/src/subsystems/AGENTS.md +594 -0
  52. package/src/subsystems/AgentManager/Agent.class.ts +71 -21
  53. package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts +24 -1
  54. package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts +2 -2
  55. package/src/subsystems/AgentManager/AgentRuntime.class.ts +34 -5
  56. package/src/subsystems/AgentManager/Scheduler.service/Job.class.ts +414 -0
  57. package/src/subsystems/AgentManager/Scheduler.service/Schedule.class.ts +200 -0
  58. package/src/subsystems/AgentManager/Scheduler.service/SchedulerConnector.ts +200 -0
  59. package/src/subsystems/AgentManager/Scheduler.service/connectors/LocalScheduler.class.ts +767 -0
  60. package/src/subsystems/AgentManager/Scheduler.service/index.ts +11 -0
  61. package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +1 -1
  62. package/src/subsystems/LLMManager/LLM.service/LLMCredentials.helper.ts +61 -2
  63. package/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts +3 -0
  64. package/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts +3 -1
  65. package/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts +5 -1
  66. package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts +247 -56
  67. package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts +3 -0
  68. package/src/subsystems/LLMManager/LLM.service/connectors/Ollama.class.ts +28 -21
  69. package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts +3 -0
  70. package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts +121 -33
  71. package/src/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.ts +38 -27
  72. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.ts +115 -18
  73. package/src/subsystems/LLMManager/LLM.service/connectors/xAI.class.ts +3 -0
  74. package/src/subsystems/LLMManager/ModelsProvider.service/ModelsProviderConnector.ts +1 -6
  75. package/src/subsystems/MemoryManager/LLMContext.ts +3 -8
  76. package/src/subsystems/MemoryManager/RuntimeContext.ts +10 -9
  77. package/src/subsystems/Security/Credentials/Credentials.class.ts +1 -0
  78. package/src/subsystems/Security/Credentials/ManagedOAuth2Credentials.class.ts +106 -0
  79. package/src/types/Agent.types.ts +1 -0
  80. package/src/types/LLM.types.ts +2 -2
  81. package/src/types/SRE.types.ts +3 -0
@@ -13,6 +13,7 @@ import { LogService } from '@sre/IO/Log.service';
13
13
  import { ComponentService } from '@sre/AgentManager/Component.service';
14
14
  import { ModelsProviderService } from '@sre/LLMManager/ModelsProvider.service';
15
15
  import { CodeService } from '@sre/ComputeManager/Code.service';
16
+ import { SchedulerService } from '@sre/AgentManager/Scheduler.service';
16
17
  export type TServiceRegistry = {
17
18
  Storage?: StorageService;
18
19
  VectorDB?: VectorDBService;
@@ -29,6 +30,7 @@ export type TServiceRegistry = {
29
30
  Component?: ComponentService;
30
31
  ModelsProvider?: ModelsProviderService;
31
32
  Code?: CodeService;
33
+ Scheduler?: SchedulerService;
32
34
  };
33
35
  export declare enum TConnectorService {
34
36
  Storage = "Storage",
@@ -45,7 +47,8 @@ export declare enum TConnectorService {
45
47
  Log = "Log",
46
48
  Component = "Component",
47
49
  ModelsProvider = "ModelsProvider",
48
- Code = "Code"
50
+ Code = "Code",
51
+ Scheduler = "Scheduler"
49
52
  }
50
53
  export type SREConnectorConfig = {
51
54
  Connector: string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@smythos/sre",
3
- "version": "1.6.14",
3
+ "version": "1.7.1",
4
4
  "description": "Smyth Runtime Environment",
5
5
  "author": "Alaa-eddine KADDOURI",
6
6
  "license": "MIT",
@@ -31,6 +31,7 @@
31
31
  "@types/express": "^4.17.23",
32
32
  "@types/lodash": "^4.17.10",
33
33
  "@types/node": "^20.19.0",
34
+ "@types/ws": "^8.18.1",
34
35
  "cross-env": "^7.0.3",
35
36
  "ctix": "^2.7.1",
36
37
  "dependency-cruiser": "^16.3.3",
@@ -59,11 +60,12 @@
59
60
  "@pinecone-database/pinecone": "^3.0.0",
60
61
  "@runware/sdk-js": "^1.1.36",
61
62
  "@smithy/smithy-client": "^4.4.3",
62
- "@zilliz/milvus2-sdk-node": "^2.5.11",
63
+ "@zilliz/milvus2-sdk-node": "^2.6.2",
63
64
  "acorn": "^8.14.1",
64
65
  "axios": "^1.7.2",
65
66
  "chokidar": "^4.0.3",
66
67
  "dayjs": "^1.11.11",
68
+ "domutils": "^3.2.2",
67
69
  "dotenv": "^16.4.5",
68
70
  "eventsource": "^3.0.2",
69
71
  "express": "^4.21.2",
@@ -71,6 +73,7 @@
71
73
  "form-data": "^4.0.3",
72
74
  "gpt-tokenizer": "^2.2.1",
73
75
  "groq-sdk": "^0.6.1",
76
+ "htmlparser2": "^10.0.0",
74
77
  "image-size": "^1.1.1",
75
78
  "ioredis": "^5.4.1",
76
79
  "isbinaryfile": "^5.0.2",
@@ -90,6 +93,7 @@
90
93
  "socks-proxy-agent": "^8.0.4",
91
94
  "winston": "^3.13.0",
92
95
  "winston-transport": "^4.7.0",
96
+ "ws": "^8.18.3",
93
97
  "xxhashjs": "^0.2.2",
94
98
  "zip-lib": "^1.1.2"
95
99
  },
@@ -7,8 +7,9 @@ import axios, { AxiosRequestConfig } from 'axios';
7
7
  import { Logger } from '@sre/helpers/Log.helper';
8
8
  import { ConnectorService } from '@sre/Core/ConnectorsService';
9
9
  import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
10
- import { TemplateString } from '@sre/helpers/TemplateString.helper';
10
+ import { Match, TemplateString } from '@sre/helpers/TemplateString.helper';
11
11
  import { SystemEvents } from '@sre/Core/SystemEvents';
12
+ import { cloneDeep } from 'lodash';
12
13
 
13
14
  const console = Logger('OAuth.helper');
14
15
  let managedVault: any;
@@ -49,11 +50,9 @@ export function extractAdditionalParamsForOAuth1(reqConfig: AxiosRequestConfig =
49
50
 
50
51
  // Headers might be an object or array of objects
51
52
  if (Array.isArray(headers)) {
52
- const contentTypeHeader = headers.find(h =>
53
- Object.keys(h).some(k => k.toLowerCase() === 'content-type')
54
- );
53
+ const contentTypeHeader = headers.find((h) => Object.keys(h).some((k) => k.toLowerCase() === 'content-type'));
55
54
  if (contentTypeHeader) {
56
- const key = Object.keys(contentTypeHeader).find(k => k.toLowerCase() === 'content-type');
55
+ const key = Object.keys(contentTypeHeader).find((k) => k.toLowerCase() === 'content-type');
57
56
  contentType = contentTypeHeader[key];
58
57
  }
59
58
  } else {
@@ -83,9 +82,7 @@ export function extractAdditionalParamsForOAuth1(reqConfig: AxiosRequestConfig =
83
82
  console.debug('OAuth1: Including form parameters in signature:', Object.keys(formParams));
84
83
  additionalParams = { ...additionalParams, ...formParams };
85
84
  }
86
- } else if (contentType.includes(REQUEST_CONTENT_TYPES.json) ||
87
- contentType.includes('application/') ||
88
- contentType.includes('text/')) {
85
+ } else if (contentType.includes(REQUEST_CONTENT_TYPES.json) || contentType.includes('application/') || contentType.includes('text/')) {
89
86
  // For JSON and other non-form data, use oauth_body_hash
90
87
  if (reqConfig.data && method !== 'GET' && method !== 'HEAD') {
91
88
  let bodyString = '';
@@ -110,8 +107,7 @@ export function extractAdditionalParamsForOAuth1(reqConfig: AxiosRequestConfig =
110
107
  // Only include string values, exclude Files/Blobs
111
108
  if (typeof value === 'string') {
112
109
  additionalParams[key] = value;
113
- } else if (typeof value === 'object' && value !== null &&
114
- ('size' in value || 'type' in value)) {
110
+ } else if (typeof value === 'object' && value !== null && ('size' in value || 'type' in value)) {
115
111
  // Skip binary data (Files, Blobs, etc.)
116
112
  continue;
117
113
  } else {
@@ -123,8 +119,7 @@ export function extractAdditionalParamsForOAuth1(reqConfig: AxiosRequestConfig =
123
119
  } else if (!contentType && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
124
120
  // No content type specified but has data
125
121
  if (reqConfig.data) {
126
- const bodyString = typeof reqConfig.data === 'string' ?
127
- reqConfig.data : JSON.stringify(reqConfig.data);
122
+ const bodyString = typeof reqConfig.data === 'string' ? reqConfig.data : JSON.stringify(reqConfig.data);
128
123
  const hash = crypto.createHash('sha1').update(bodyString).digest('base64');
129
124
  additionalParams['oauth_body_hash'] = hash;
130
125
  }
@@ -165,9 +160,10 @@ export const buildOAuth1Header = (url, method, oauth1Credentials, additionalPara
165
160
  data: additionalParams, // Parameters should be in data field for oauth-1.0a library
166
161
  };
167
162
 
168
- const token = oauth1Credentials.token && oauth1Credentials.token !== '' ?
169
- { key: oauth1Credentials.token, secret: oauth1Credentials.tokenSecret || '' } :
170
- null;
163
+ const token =
164
+ oauth1Credentials.token && oauth1Credentials.token !== ''
165
+ ? { key: oauth1Credentials.token, secret: oauth1Credentials.tokenSecret || '' }
166
+ : null;
171
167
 
172
168
  const signedRequest = oauth.authorize(requestData, token);
173
169
  return oauth.toHeader(signedRequest);
@@ -190,24 +186,25 @@ export const retrieveOAuthTokens = async (agent, config) => {
190
186
  // Check if it's new structure (has auth_data and auth_settings) or old structure
191
187
  const isNewStructure = tokensData.auth_data !== undefined && tokensData.auth_settings !== undefined;
192
188
 
189
+ //* Resolve vault keys inside auth_data
190
+ if (isNewStructure) {
191
+ await Promise.all(
192
+ Object.keys(tokensData.auth_settings).map(async (key) => {
193
+ if (typeof tokensData.auth_settings[key] !== 'string') return;
194
+ tokensData.auth_settings[key] = await TemplateString(tokensData.auth_settings[key]).parseTeamKeysAsync(agent.teamId)
195
+ .asyncResult;
196
+ })
197
+ );
198
+ }
199
+
193
200
  // Extract tokens based on structure
194
- const primaryToken = isNewStructure
195
- ? tokensData.auth_data?.primary
196
- : tokensData.primary;
197
- const secondaryToken = isNewStructure
198
- ? tokensData.auth_data?.secondary
199
- : tokensData.secondary;
200
- const expiresIn = isNewStructure
201
- ? tokensData.auth_data?.expires_in
202
- : tokensData.expires_in;
201
+ const primaryToken = isNewStructure ? tokensData.auth_data?.primary : tokensData.primary;
202
+ const secondaryToken = isNewStructure ? tokensData.auth_data?.secondary : tokensData.secondary;
203
+ const expiresIn = isNewStructure ? tokensData.auth_data?.expires_in : tokensData.expires_in;
203
204
 
204
205
  // Extract settings based on structure
205
- const type = isNewStructure
206
- ? tokensData.auth_settings?.type
207
- : (tokensData.type || tokensData.oauth_info?.type);
208
- const service = isNewStructure
209
- ? tokensData.auth_settings?.service
210
- : tokensData.oauth_info?.service;
206
+ const type = isNewStructure ? tokensData.auth_settings?.type : tokensData.type || tokensData.oauth_info?.type;
207
+ const service = isNewStructure ? tokensData.auth_settings?.service : tokensData.oauth_info?.service;
211
208
 
212
209
  // Add warning logs for OAuth2
213
210
  if (type === 'oauth2' && service !== 'oauth2_client_credentials') {
@@ -353,8 +350,6 @@ const getKeyIdsFromTemplateVars = (str: string): string[] => {
353
350
  };
354
351
 
355
352
  async function getClientCredentialToken(tokensData, logger, keyId, oauthTokens, config, agent, isNewStructure = false) {
356
-
357
-
358
353
  const logAndThrowError = (message) => {
359
354
  logger.debug(message);
360
355
  throw new Error(message);
@@ -399,7 +394,7 @@ async function getClientCredentialToken(tokensData, logger, keyId, oauthTokens,
399
394
  auth_data: {
400
395
  ...(tokensData?.auth_data || {}),
401
396
  primary: newAccessToken,
402
- expires_in: expirationTimestamp.toString()
397
+ expires_in: expirationTimestamp.toString(),
403
398
  },
404
399
  auth_settings: {
405
400
  ...(tokensData?.auth_settings || {}),
@@ -416,7 +411,7 @@ async function getClientCredentialToken(tokensData, logger, keyId, oauthTokens,
416
411
  updatedData = {
417
412
  ...tokensData,
418
413
  primary: newAccessToken,
419
- expires_in: expirationTimestamp.toString()
414
+ expires_in: expirationTimestamp.toString(),
420
415
  };
421
416
  // Ensure required fields are present for old structure
422
417
  if (!updatedData.type) updatedData.type = 'oauth2';
@@ -428,7 +423,7 @@ async function getClientCredentialToken(tokensData, logger, keyId, oauthTokens,
428
423
  service: 'oauth2_client_credentials',
429
424
  tokenURL,
430
425
  clientID,
431
- clientSecret
426
+ clientSecret,
432
427
  };
433
428
  }
434
429
  }
@@ -39,7 +39,7 @@ function parseKey(str: string = '', teamId: string): string {
39
39
  export class APIEndpoint extends Component {
40
40
  protected configSchema = Joi.object({
41
41
  endpoint: Joi.string()
42
- .pattern(/^[a-zA-Z0-9]+([-_][a-zA-Z0-9]+)*$/)
42
+ .pattern(/^[a-zA-Z0-9_]+([-_][a-zA-Z0-9_]+)*$/)
43
43
  .max(50)
44
44
  .required(),
45
45
  method: Joi.string().valid('POST', 'GET').allow(''), //we're accepting empty value because we consider it POST by default.
@@ -57,13 +57,25 @@ export class APIEndpoint extends Component {
57
57
  async process(input, config, agent: Agent) {
58
58
  await super.process(input, config, agent);
59
59
 
60
+ if (typeof config.process === 'function') {
61
+ //special case, APIEndpoint has a custom process method.
62
+ //the inputs are not passed directly to APIEndpoints, we need to read them from the context.
63
+ const contextData = agent?.agentRuntime?.getComponentData(config.id);
64
+
65
+ const inputs = Array.isArray(contextData?.input) ? contextData?.input : [contextData?.input];
66
+
67
+ const result = await config.process.apply(null, inputs);
68
+ return result;
69
+ }
70
+
60
71
  const req: AgentRequest = agent.agentRequest;
61
72
  const logger = this.createComponentLogger(agent, config);
62
73
 
74
+ const isTrigger = req.path.startsWith(agent.triggerBasePath);
63
75
  const headers = req ? req.headers : {};
64
- let body = req ? req.body : input; //handle debugger injection
65
- const params = req ? req.params : {};
66
- let query = req ? req.query : {};
76
+ let body = req && !isTrigger ? req.body : input; //handle debugger injection
77
+ const params = req && !isTrigger ? req.params : {};
78
+ let query = req && !isTrigger ? req.query : {};
67
79
  const _authInfo = req ? req._agent_authinfo : undefined;
68
80
 
69
81
  // parse template variables
@@ -87,7 +99,7 @@ export class APIEndpoint extends Component {
87
99
 
88
100
  // set default value and agent variables
89
101
  const inputsWithDefaultValue = config.inputs.filter(
90
- (input) => input.defaultVal !== undefined && input.defaultVal !== '' && input.defaultVal !== null,
102
+ (input) => input.defaultVal !== undefined && input.defaultVal !== '' && input.defaultVal !== null
91
103
  );
92
104
 
93
105
  const bodyInputNames: string[] = [];
@@ -218,7 +230,7 @@ export class APIEndpoint extends Component {
218
230
  return await binaryInput.getJsonData(AccessCandidate.agent(agent.id));
219
231
  }
220
232
  return null;
221
- }),
233
+ })
222
234
  );
223
235
 
224
236
  // Filter out null values and handle single/multiple results
@@ -231,4 +243,11 @@ export class APIEndpoint extends Component {
231
243
 
232
244
  return { headers, body, query, params, _authInfo, _debug: logger.output };
233
245
  }
246
+
247
+ async postProcess(output, config, agent: Agent): Promise<any> {
248
+ if (typeof config.process === 'function') {
249
+ return output?.result;
250
+ }
251
+ return output;
252
+ }
234
253
  }
@@ -144,4 +144,15 @@ export class Component {
144
144
  hasOutput(id, config, agent: Agent): any {
145
145
  return false;
146
146
  }
147
+
148
+ /**
149
+ * A generic registration function that can be overridden by child classes to register the trigger with providers
150
+ * this function is usually called outside of a workflow in order to register the trigger with providers
151
+ * @param componentId the id of the component to register (in SRE a single trigger instance handles all the triggers of the same type)
152
+ * @param agent the agent instance
153
+ * @param payload any additional payload to pass to the trigger
154
+ */
155
+ async register(componentId: string, componentSettings: any, payload?: any) {}
156
+
157
+ async unregister(componentId: string, componentSettings: any, payload?: any) {}
147
158
  }
@@ -0,0 +1,282 @@
1
+ import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
2
+ import { ManagedOAuth2Credentials } from '@sre/Security/Credentials/ManagedOAuth2Credentials.class';
3
+ import { IAgent as Agent } from '@sre/types/Agent.types';
4
+ import { findAll, innerText, isTag, removeElement } from 'domutils';
5
+ import { parseDocument } from 'htmlparser2';
6
+ import { Trigger } from './Trigger.class';
7
+ export type GmailTriggerAttachment = {
8
+ filename: string;
9
+ mimeType: string;
10
+ size: number;
11
+ attachmentId: string;
12
+ };
13
+ export type GmailTriggerMessage = {
14
+ id: string;
15
+ threadId: string;
16
+ labelIds: string[];
17
+ snippet: string;
18
+ sizeEstimate: number;
19
+ internalDate: string;
20
+ headers: {
21
+ from: string;
22
+ to: string;
23
+ cc: string;
24
+ bcc: string;
25
+ subject: string;
26
+ date: string;
27
+ messageId: string;
28
+ };
29
+ body: {
30
+ text: string;
31
+ html: string;
32
+ };
33
+ attachments: GmailTriggerAttachment[];
34
+ isUnread: true;
35
+ };
36
+ export class GmailTrigger extends Trigger {
37
+ async collectPayload(input, config, agent: Agent) {
38
+ const credentialsKey = config?.data?.oauth_cred_id;
39
+ const agentCandidate = AccessCandidate.agent(agent.id);
40
+ const oauth2Credentials = await ManagedOAuth2Credentials.load(credentialsKey, agentCandidate);
41
+
42
+ console.log(
43
+ oauth2Credentials.accessToken,
44
+ oauth2Credentials.refreshToken,
45
+ oauth2Credentials.expiresIn,
46
+ oauth2Credentials.scope,
47
+ oauth2Credentials.tokenUrl,
48
+ oauth2Credentials.service
49
+ );
50
+
51
+ await oauth2Credentials.refreshAccessToken();
52
+
53
+ const messages = await getMostRecentUnreadMessage(0, oauth2Credentials, 5);
54
+ return messages;
55
+ }
56
+ }
57
+
58
+ /**
59
+ * Make an authenticated Gmail API request
60
+ */
61
+ async function gmailApiRequest(endpoint, accessToken) {
62
+ const url = `https://www.googleapis.com/gmail/v1/users/me${endpoint}`;
63
+
64
+ try {
65
+ const result = await fetch(url, {
66
+ headers: {
67
+ Authorization: `Bearer ${accessToken}`,
68
+ },
69
+ });
70
+ const json = await result.json();
71
+ console.log(json);
72
+ return json;
73
+ } catch (error) {
74
+ throw new Error(`Gmail API request failed: ${error.message}`);
75
+ }
76
+ }
77
+
78
+ /**
79
+ * Get the most recent unread messages
80
+ */
81
+ async function getMostRecentUnreadMessage(
82
+ retryCount = 0,
83
+ oauth2Credentials: ManagedOAuth2Credentials,
84
+ numMessages: number = 1,
85
+ excludeMessageId?: string
86
+ ) {
87
+ const MAX_RETRIES = 1; // Only retry once to prevent infinite loops
88
+
89
+ let accessToken = oauth2Credentials.accessToken;
90
+
91
+ // If no access token provided, or if requests fail, refresh it
92
+ if (!accessToken) {
93
+ accessToken = await oauth2Credentials.refreshAccessToken();
94
+ }
95
+
96
+ try {
97
+ console.log('Fetching unread messages...');
98
+
99
+ const collectedIds: string[] = [];
100
+ let pageToken: string | undefined = undefined;
101
+ let reachedBoundary = false;
102
+
103
+ while (collectedIds.length < numMessages && !reachedBoundary) {
104
+ const pageSize = Math.min(Math.max(numMessages, 1), 100);
105
+ const pageQuery = `/messages?q=is:unread&maxResults=${pageSize}${pageToken ? `&pageToken=${pageToken}` : ''}`;
106
+ const listResponse = await gmailApiRequest(pageQuery, accessToken);
107
+
108
+ const messages = Array.isArray(listResponse.messages) ? listResponse.messages : [];
109
+ if (messages.length === 0) {
110
+ break;
111
+ }
112
+
113
+ for (const m of messages) {
114
+ if (excludeMessageId && m.id === excludeMessageId) {
115
+ reachedBoundary = true;
116
+ break;
117
+ }
118
+ collectedIds.push(m.id);
119
+ if (collectedIds.length >= numMessages) {
120
+ break;
121
+ }
122
+ }
123
+
124
+ if (collectedIds.length >= numMessages || reachedBoundary || !listResponse.nextPageToken) {
125
+ break;
126
+ }
127
+ pageToken = listResponse.nextPageToken;
128
+ }
129
+
130
+ if (collectedIds.length === 0) {
131
+ return [];
132
+ }
133
+
134
+ // Fetch full message details in parallel
135
+ const details = await Promise.all(collectedIds.map((id) => gmailApiRequest(`/messages/${id}?format=full`, accessToken)));
136
+ const parsed = details.map((d) => parseMessage(d));
137
+ return parsed;
138
+ } catch (error) {
139
+ // If token expired, try refreshing and retry once
140
+ if ((error.message.includes('401') || error.message.includes('unauthorized')) && retryCount < MAX_RETRIES) {
141
+ console.log('Access token expired, refreshing...');
142
+ try {
143
+ accessToken = await oauth2Credentials.refreshAccessToken();
144
+ return await getMostRecentUnreadMessage(retryCount + 1, oauth2Credentials, numMessages, excludeMessageId);
145
+ } catch (refreshError) {
146
+ throw new Error(`Authentication failed: ${refreshError.message}`);
147
+ }
148
+ }
149
+
150
+ throw error;
151
+ }
152
+ }
153
+
154
+ /**
155
+ * Parse Gmail message into structured format
156
+ */
157
+ function parseMessage(message) {
158
+ const headers: any = {};
159
+ const payload = message.payload || {};
160
+
161
+ // Extract headers
162
+ if (payload.headers) {
163
+ payload.headers.forEach((header) => {
164
+ headers[header.name.toLowerCase()] = header.value;
165
+ });
166
+ }
167
+
168
+ // Extract body content
169
+ let textBody = '';
170
+ let htmlBody = '';
171
+
172
+ function extractBody(part) {
173
+ if (part.mimeType === 'text/plain' && part.body && part.body.data) {
174
+ textBody = Buffer.from(part.body.data, 'base64').toString('utf-8');
175
+ } else if (part.mimeType === 'text/html' && part.body && part.body.data) {
176
+ htmlBody = Buffer.from(part.body.data, 'base64').toString('utf-8');
177
+ }
178
+
179
+ if (part.parts) {
180
+ part.parts.forEach(extractBody);
181
+ }
182
+ }
183
+
184
+ if (payload.parts) {
185
+ payload.parts.forEach(extractBody);
186
+ } else if (payload.body && payload.body.data) {
187
+ // Single part message
188
+ if (payload.mimeType === 'text/plain') {
189
+ textBody = Buffer.from(payload.body.data, 'base64').toString('utf-8');
190
+ } else if (payload.mimeType === 'text/html') {
191
+ htmlBody = Buffer.from(payload.body.data, 'base64').toString('utf-8');
192
+ }
193
+ }
194
+
195
+ if (textBody && !htmlBody) {
196
+ htmlBody = textBody;
197
+ }
198
+
199
+ if (htmlBody && !textBody) {
200
+ textBody = htmlToPlainText(htmlBody);
201
+ }
202
+
203
+ // Get attachments info
204
+ const attachments = [];
205
+ function findAttachments(part) {
206
+ if (part.filename && part.filename.length > 0) {
207
+ attachments.push({
208
+ filename: part.filename,
209
+ mimeType: part.mimeType,
210
+ size: part.body ? part.body.size : 0,
211
+ attachmentId: part.body ? part.body.attachmentId : null,
212
+ });
213
+ }
214
+
215
+ if (part.parts) {
216
+ part.parts.forEach(findAttachments);
217
+ }
218
+ }
219
+
220
+ if (payload.parts) {
221
+ payload.parts.forEach(findAttachments);
222
+ }
223
+
224
+ return {
225
+ email: {
226
+ id: message.id,
227
+ threadId: message.threadId,
228
+ labelIds: message.labelIds || [],
229
+ snippet: message.snippet || '',
230
+ sizeEstimate: message.sizeEstimate || 0,
231
+ internalDate: message.internalDate ? new Date(parseInt(message.internalDate)).toISOString() : null,
232
+ headers: {
233
+ from: headers.from || '',
234
+ to: headers.to || '',
235
+ cc: headers.cc || '',
236
+ bcc: headers.bcc || '',
237
+ subject: headers.subject || '',
238
+ date: headers.date || '',
239
+ messageId: headers['message-id'] || '',
240
+ },
241
+ body: {
242
+ text: textBody,
243
+ html: htmlBody,
244
+ },
245
+ attachments: attachments,
246
+ isUnread: message.labelIds ? message.labelIds.includes('UNREAD') : false,
247
+ },
248
+ };
249
+ }
250
+
251
+ var populateChar = function (ch, amount) {
252
+ var result = '';
253
+ for (var i = 0; i < amount; i += 1) {
254
+ result += ch;
255
+ }
256
+ return result;
257
+ };
258
+
259
+ function htmlToPlainText(htmlText, _styleConfig?) {
260
+ try {
261
+ const document = parseDocument(String(htmlText));
262
+ const nodesToRemove = findAll((node) => isTag(node) && (node.name === 'script' || node.name === 'style'), document.children);
263
+ nodesToRemove.forEach((node) => removeElement(node));
264
+
265
+ let text = innerText(document);
266
+
267
+ text = text.replace(/\u00A0/g, ' ');
268
+ text = text.replace(/\r\n?/g, '\n');
269
+ text = text.replace(/\t+/g, ' ');
270
+ text = text.replace(/[ \t\f]+\n/g, '\n');
271
+ text = text.replace(/\n{3,}/g, '\n\n');
272
+ text = text.replace(/[ ]{2,}/g, ' ');
273
+ text = text.replace(/^\s+|\s+$/g, '');
274
+
275
+ if (text.length === 0 || text.lastIndexOf('\n') !== text.length - 1) {
276
+ text += '\n';
277
+ }
278
+ return text;
279
+ } catch (err) {
280
+ return String(htmlText);
281
+ }
282
+ }
@@ -0,0 +1,45 @@
1
+ import { IAgent as Agent, IAgent } from '@sre/types/Agent.types';
2
+ import { Trigger } from './Trigger.class';
3
+ import { AgentRequest } from '@sre/AgentManager/AgentRequest.class';
4
+ import express from 'express';
5
+ import { ConnectorService } from '@sre/Core/ConnectorsService';
6
+ import { ISchedulerRequest } from '@sre/AgentManager/Scheduler.service/SchedulerConnector';
7
+ import { Schedule } from '@sre/AgentManager/Scheduler.service/Schedule.class';
8
+ import { Job } from '@sre/AgentManager/Scheduler.service/Job.class';
9
+ import { JSONContent } from '@sre/helpers/JsonContent.helper';
10
+
11
+ export class JobSchedulerTrigger extends Trigger {
12
+ async collectPayload(input, settings, agent: Agent) {
13
+ const req = agent.agentRequest;
14
+
15
+ let payload: any = JSONContent(settings?.data?.payload).tryParse();
16
+ if (typeof payload !== 'object') {
17
+ payload = {};
18
+ }
19
+
20
+ return [
21
+ {
22
+ ...payload,
23
+ },
24
+ ];
25
+ }
26
+
27
+ async register(componentId: string, componentSettings: any, payload?: { agentData: IAgent; triggerUrl: string }) {
28
+ try {
29
+ const agent: any = payload?.agentData;
30
+ const triggerName = componentSettings.triggerEndpoint;
31
+ const schedulerConnector = ConnectorService.getSchedulerConnector();
32
+ if (!schedulerConnector) {
33
+ throw new Error('Scheduler connector not found');
34
+ }
35
+
36
+ const schedulerRequester: ISchedulerRequest = schedulerConnector.agent(agent.id);
37
+ const jobId = `job-${agent.id}-${triggerName}`;
38
+ await schedulerRequester.add(jobId, new Job({ agentId: agent.id, type: 'trigger', triggerName }), Schedule.every('10s'));
39
+ } catch (error) {
40
+ throw new Error('Failed to schedule job');
41
+ }
42
+ }
43
+
44
+ async unregister(componentId: string, agent: Agent, payload?: any) {}
45
+ }
@@ -0,0 +1,3 @@
1
+ # Triggers System
2
+
3
+ ### !!!Work in progress!!! This is still a work in progress, the system is not yet fully implemented.