@smythos/sre 1.6.14 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG +15 -0
- package/dist/index.js +52 -46
- package/dist/index.js.map +1 -1
- package/dist/types/Components/APIEndpoint.class.d.ts +2 -8
- package/dist/types/Components/Component.class.d.ts +9 -0
- package/dist/types/Components/Triggers/Gmail.trigger.d.ts +0 -17
- package/dist/types/Components/Triggers/JobScheduler.trigger.d.ts +10 -0
- package/dist/types/Components/Triggers/Trigger.class.d.ts +11 -0
- package/dist/types/Components/index.d.ts +6 -0
- package/dist/types/Core/Connector.class.d.ts +1 -0
- package/dist/types/Core/ConnectorsService.d.ts +2 -0
- package/dist/types/Core/HookService.d.ts +1 -1
- package/dist/types/helpers/Conversation.helper.d.ts +2 -0
- package/dist/types/helpers/Crypto.helper.d.ts +8 -0
- package/dist/types/index.d.ts +13 -0
- package/dist/types/subsystems/AgentManager/Agent.class.d.ts +4 -2
- package/dist/types/subsystems/AgentManager/AgentData.service/AgentDataConnector.d.ts +13 -0
- package/dist/types/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.d.ts +1 -4
- package/dist/types/subsystems/AgentManager/Scheduler.service/Job.class.d.ts +29 -6
- package/dist/types/subsystems/AgentManager/Scheduler.service/SchedulerConnector.d.ts +11 -3
- package/dist/types/subsystems/AgentManager/Scheduler.service/connectors/LocalScheduler.class.d.ts +31 -7
- package/dist/types/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.d.ts +2 -5
- package/dist/types/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.d.ts +3 -6
- package/dist/types/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.d.ts +7 -0
- package/dist/types/subsystems/LLMManager/LLM.service/connectors/xAI.class.d.ts +2 -5
- package/dist/types/types/Agent.types.d.ts +1 -0
- package/dist/types/types/LLM.types.d.ts +2 -5
- package/dist/types/types/SRE.types.d.ts +4 -1
- package/package.json +6 -2
- package/src/Components/APICall/OAuth.helper.ts +30 -35
- package/src/Components/APIEndpoint.class.ts +25 -6
- package/src/Components/Component.class.ts +11 -0
- package/src/Components/Triggers/Gmail.trigger.ts +282 -0
- package/src/Components/Triggers/JobScheduler.trigger.ts +45 -0
- package/src/Components/Triggers/README.md +3 -0
- package/src/Components/Triggers/Trigger.class.ts +101 -0
- package/src/Components/Triggers/WhatsApp.trigger.ts +219 -0
- package/src/Components/index.ts +8 -0
- package/src/Core/AgentProcess.helper.ts +4 -6
- package/src/Core/Connector.class.ts +11 -3
- package/src/Core/ConnectorsService.ts +5 -0
- package/src/Core/ExternalEventsReceiver.ts +317 -0
- package/src/Core/HookService.ts +20 -6
- package/src/Core/SmythRuntime.class.ts +7 -0
- package/src/Core/SystemEvents.ts +17 -0
- package/src/Core/boot.ts +2 -0
- package/src/helpers/Conversation.helper.ts +35 -11
- package/src/helpers/Crypto.helper.ts +28 -0
- package/src/index.ts +208 -195
- package/src/index.ts.bak +208 -195
- package/src/subsystems/AGENTS.md +594 -0
- package/src/subsystems/AgentManager/Agent.class.ts +71 -21
- package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts +24 -1
- package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts +2 -2
- package/src/subsystems/AgentManager/AgentRuntime.class.ts +34 -5
- package/src/subsystems/AgentManager/Scheduler.service/Job.class.ts +414 -0
- package/src/subsystems/AgentManager/Scheduler.service/Schedule.class.ts +200 -0
- package/src/subsystems/AgentManager/Scheduler.service/SchedulerConnector.ts +200 -0
- package/src/subsystems/AgentManager/Scheduler.service/connectors/LocalScheduler.class.ts +767 -0
- package/src/subsystems/AgentManager/Scheduler.service/index.ts +11 -0
- package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +1 -1
- package/src/subsystems/LLMManager/LLM.service/LLMCredentials.helper.ts +61 -2
- package/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts +3 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts +3 -1
- package/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts +5 -1
- package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts +247 -56
- package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts +3 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/Ollama.class.ts +28 -21
- package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts +3 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts +121 -33
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.ts +38 -27
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.ts +115 -18
- package/src/subsystems/LLMManager/LLM.service/connectors/xAI.class.ts +3 -0
- package/src/subsystems/LLMManager/ModelsProvider.service/ModelsProviderConnector.ts +1 -6
- package/src/subsystems/MemoryManager/LLMContext.ts +3 -8
- package/src/subsystems/MemoryManager/RuntimeContext.ts +10 -9
- package/src/subsystems/Security/Credentials/Credentials.class.ts +1 -0
- package/src/subsystems/Security/Credentials/ManagedOAuth2Credentials.class.ts +106 -0
- package/src/types/Agent.types.ts +1 -0
- package/src/types/LLM.types.ts +2 -2
- package/src/types/SRE.types.ts +3 -0
|
@@ -13,6 +13,7 @@ import { LogService } from '@sre/IO/Log.service';
|
|
|
13
13
|
import { ComponentService } from '@sre/AgentManager/Component.service';
|
|
14
14
|
import { ModelsProviderService } from '@sre/LLMManager/ModelsProvider.service';
|
|
15
15
|
import { CodeService } from '@sre/ComputeManager/Code.service';
|
|
16
|
+
import { SchedulerService } from '@sre/AgentManager/Scheduler.service';
|
|
16
17
|
export type TServiceRegistry = {
|
|
17
18
|
Storage?: StorageService;
|
|
18
19
|
VectorDB?: VectorDBService;
|
|
@@ -29,6 +30,7 @@ export type TServiceRegistry = {
|
|
|
29
30
|
Component?: ComponentService;
|
|
30
31
|
ModelsProvider?: ModelsProviderService;
|
|
31
32
|
Code?: CodeService;
|
|
33
|
+
Scheduler?: SchedulerService;
|
|
32
34
|
};
|
|
33
35
|
export declare enum TConnectorService {
|
|
34
36
|
Storage = "Storage",
|
|
@@ -45,7 +47,8 @@ export declare enum TConnectorService {
|
|
|
45
47
|
Log = "Log",
|
|
46
48
|
Component = "Component",
|
|
47
49
|
ModelsProvider = "ModelsProvider",
|
|
48
|
-
Code = "Code"
|
|
50
|
+
Code = "Code",
|
|
51
|
+
Scheduler = "Scheduler"
|
|
49
52
|
}
|
|
50
53
|
export type SREConnectorConfig = {
|
|
51
54
|
Connector: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smythos/sre",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.1",
|
|
4
4
|
"description": "Smyth Runtime Environment",
|
|
5
5
|
"author": "Alaa-eddine KADDOURI",
|
|
6
6
|
"license": "MIT",
|
|
@@ -31,6 +31,7 @@
|
|
|
31
31
|
"@types/express": "^4.17.23",
|
|
32
32
|
"@types/lodash": "^4.17.10",
|
|
33
33
|
"@types/node": "^20.19.0",
|
|
34
|
+
"@types/ws": "^8.18.1",
|
|
34
35
|
"cross-env": "^7.0.3",
|
|
35
36
|
"ctix": "^2.7.1",
|
|
36
37
|
"dependency-cruiser": "^16.3.3",
|
|
@@ -59,11 +60,12 @@
|
|
|
59
60
|
"@pinecone-database/pinecone": "^3.0.0",
|
|
60
61
|
"@runware/sdk-js": "^1.1.36",
|
|
61
62
|
"@smithy/smithy-client": "^4.4.3",
|
|
62
|
-
"@zilliz/milvus2-sdk-node": "^2.
|
|
63
|
+
"@zilliz/milvus2-sdk-node": "^2.6.2",
|
|
63
64
|
"acorn": "^8.14.1",
|
|
64
65
|
"axios": "^1.7.2",
|
|
65
66
|
"chokidar": "^4.0.3",
|
|
66
67
|
"dayjs": "^1.11.11",
|
|
68
|
+
"domutils": "^3.2.2",
|
|
67
69
|
"dotenv": "^16.4.5",
|
|
68
70
|
"eventsource": "^3.0.2",
|
|
69
71
|
"express": "^4.21.2",
|
|
@@ -71,6 +73,7 @@
|
|
|
71
73
|
"form-data": "^4.0.3",
|
|
72
74
|
"gpt-tokenizer": "^2.2.1",
|
|
73
75
|
"groq-sdk": "^0.6.1",
|
|
76
|
+
"htmlparser2": "^10.0.0",
|
|
74
77
|
"image-size": "^1.1.1",
|
|
75
78
|
"ioredis": "^5.4.1",
|
|
76
79
|
"isbinaryfile": "^5.0.2",
|
|
@@ -90,6 +93,7 @@
|
|
|
90
93
|
"socks-proxy-agent": "^8.0.4",
|
|
91
94
|
"winston": "^3.13.0",
|
|
92
95
|
"winston-transport": "^4.7.0",
|
|
96
|
+
"ws": "^8.18.3",
|
|
93
97
|
"xxhashjs": "^0.2.2",
|
|
94
98
|
"zip-lib": "^1.1.2"
|
|
95
99
|
},
|
|
@@ -7,8 +7,9 @@ import axios, { AxiosRequestConfig } from 'axios';
|
|
|
7
7
|
import { Logger } from '@sre/helpers/Log.helper';
|
|
8
8
|
import { ConnectorService } from '@sre/Core/ConnectorsService';
|
|
9
9
|
import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
|
|
10
|
-
import { TemplateString } from '@sre/helpers/TemplateString.helper';
|
|
10
|
+
import { Match, TemplateString } from '@sre/helpers/TemplateString.helper';
|
|
11
11
|
import { SystemEvents } from '@sre/Core/SystemEvents';
|
|
12
|
+
import { cloneDeep } from 'lodash';
|
|
12
13
|
|
|
13
14
|
const console = Logger('OAuth.helper');
|
|
14
15
|
let managedVault: any;
|
|
@@ -49,11 +50,9 @@ export function extractAdditionalParamsForOAuth1(reqConfig: AxiosRequestConfig =
|
|
|
49
50
|
|
|
50
51
|
// Headers might be an object or array of objects
|
|
51
52
|
if (Array.isArray(headers)) {
|
|
52
|
-
const contentTypeHeader = headers.find(h =>
|
|
53
|
-
Object.keys(h).some(k => k.toLowerCase() === 'content-type')
|
|
54
|
-
);
|
|
53
|
+
const contentTypeHeader = headers.find((h) => Object.keys(h).some((k) => k.toLowerCase() === 'content-type'));
|
|
55
54
|
if (contentTypeHeader) {
|
|
56
|
-
const key = Object.keys(contentTypeHeader).find(k => k.toLowerCase() === 'content-type');
|
|
55
|
+
const key = Object.keys(contentTypeHeader).find((k) => k.toLowerCase() === 'content-type');
|
|
57
56
|
contentType = contentTypeHeader[key];
|
|
58
57
|
}
|
|
59
58
|
} else {
|
|
@@ -83,9 +82,7 @@ export function extractAdditionalParamsForOAuth1(reqConfig: AxiosRequestConfig =
|
|
|
83
82
|
console.debug('OAuth1: Including form parameters in signature:', Object.keys(formParams));
|
|
84
83
|
additionalParams = { ...additionalParams, ...formParams };
|
|
85
84
|
}
|
|
86
|
-
} else if (contentType.includes(REQUEST_CONTENT_TYPES.json) ||
|
|
87
|
-
contentType.includes('application/') ||
|
|
88
|
-
contentType.includes('text/')) {
|
|
85
|
+
} else if (contentType.includes(REQUEST_CONTENT_TYPES.json) || contentType.includes('application/') || contentType.includes('text/')) {
|
|
89
86
|
// For JSON and other non-form data, use oauth_body_hash
|
|
90
87
|
if (reqConfig.data && method !== 'GET' && method !== 'HEAD') {
|
|
91
88
|
let bodyString = '';
|
|
@@ -110,8 +107,7 @@ export function extractAdditionalParamsForOAuth1(reqConfig: AxiosRequestConfig =
|
|
|
110
107
|
// Only include string values, exclude Files/Blobs
|
|
111
108
|
if (typeof value === 'string') {
|
|
112
109
|
additionalParams[key] = value;
|
|
113
|
-
} else if (typeof value === 'object' && value !== null &&
|
|
114
|
-
('size' in value || 'type' in value)) {
|
|
110
|
+
} else if (typeof value === 'object' && value !== null && ('size' in value || 'type' in value)) {
|
|
115
111
|
// Skip binary data (Files, Blobs, etc.)
|
|
116
112
|
continue;
|
|
117
113
|
} else {
|
|
@@ -123,8 +119,7 @@ export function extractAdditionalParamsForOAuth1(reqConfig: AxiosRequestConfig =
|
|
|
123
119
|
} else if (!contentType && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
|
|
124
120
|
// No content type specified but has data
|
|
125
121
|
if (reqConfig.data) {
|
|
126
|
-
const bodyString = typeof reqConfig.data === 'string' ?
|
|
127
|
-
reqConfig.data : JSON.stringify(reqConfig.data);
|
|
122
|
+
const bodyString = typeof reqConfig.data === 'string' ? reqConfig.data : JSON.stringify(reqConfig.data);
|
|
128
123
|
const hash = crypto.createHash('sha1').update(bodyString).digest('base64');
|
|
129
124
|
additionalParams['oauth_body_hash'] = hash;
|
|
130
125
|
}
|
|
@@ -165,9 +160,10 @@ export const buildOAuth1Header = (url, method, oauth1Credentials, additionalPara
|
|
|
165
160
|
data: additionalParams, // Parameters should be in data field for oauth-1.0a library
|
|
166
161
|
};
|
|
167
162
|
|
|
168
|
-
const token =
|
|
169
|
-
|
|
170
|
-
|
|
163
|
+
const token =
|
|
164
|
+
oauth1Credentials.token && oauth1Credentials.token !== ''
|
|
165
|
+
? { key: oauth1Credentials.token, secret: oauth1Credentials.tokenSecret || '' }
|
|
166
|
+
: null;
|
|
171
167
|
|
|
172
168
|
const signedRequest = oauth.authorize(requestData, token);
|
|
173
169
|
return oauth.toHeader(signedRequest);
|
|
@@ -190,24 +186,25 @@ export const retrieveOAuthTokens = async (agent, config) => {
|
|
|
190
186
|
// Check if it's new structure (has auth_data and auth_settings) or old structure
|
|
191
187
|
const isNewStructure = tokensData.auth_data !== undefined && tokensData.auth_settings !== undefined;
|
|
192
188
|
|
|
189
|
+
//* Resolve vault keys inside auth_data
|
|
190
|
+
if (isNewStructure) {
|
|
191
|
+
await Promise.all(
|
|
192
|
+
Object.keys(tokensData.auth_settings).map(async (key) => {
|
|
193
|
+
if (typeof tokensData.auth_settings[key] !== 'string') return;
|
|
194
|
+
tokensData.auth_settings[key] = await TemplateString(tokensData.auth_settings[key]).parseTeamKeysAsync(agent.teamId)
|
|
195
|
+
.asyncResult;
|
|
196
|
+
})
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
|
|
193
200
|
// Extract tokens based on structure
|
|
194
|
-
const primaryToken = isNewStructure
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
const secondaryToken = isNewStructure
|
|
198
|
-
? tokensData.auth_data?.secondary
|
|
199
|
-
: tokensData.secondary;
|
|
200
|
-
const expiresIn = isNewStructure
|
|
201
|
-
? tokensData.auth_data?.expires_in
|
|
202
|
-
: tokensData.expires_in;
|
|
201
|
+
const primaryToken = isNewStructure ? tokensData.auth_data?.primary : tokensData.primary;
|
|
202
|
+
const secondaryToken = isNewStructure ? tokensData.auth_data?.secondary : tokensData.secondary;
|
|
203
|
+
const expiresIn = isNewStructure ? tokensData.auth_data?.expires_in : tokensData.expires_in;
|
|
203
204
|
|
|
204
205
|
// Extract settings based on structure
|
|
205
|
-
const type = isNewStructure
|
|
206
|
-
|
|
207
|
-
: (tokensData.type || tokensData.oauth_info?.type);
|
|
208
|
-
const service = isNewStructure
|
|
209
|
-
? tokensData.auth_settings?.service
|
|
210
|
-
: tokensData.oauth_info?.service;
|
|
206
|
+
const type = isNewStructure ? tokensData.auth_settings?.type : tokensData.type || tokensData.oauth_info?.type;
|
|
207
|
+
const service = isNewStructure ? tokensData.auth_settings?.service : tokensData.oauth_info?.service;
|
|
211
208
|
|
|
212
209
|
// Add warning logs for OAuth2
|
|
213
210
|
if (type === 'oauth2' && service !== 'oauth2_client_credentials') {
|
|
@@ -353,8 +350,6 @@ const getKeyIdsFromTemplateVars = (str: string): string[] => {
|
|
|
353
350
|
};
|
|
354
351
|
|
|
355
352
|
async function getClientCredentialToken(tokensData, logger, keyId, oauthTokens, config, agent, isNewStructure = false) {
|
|
356
|
-
|
|
357
|
-
|
|
358
353
|
const logAndThrowError = (message) => {
|
|
359
354
|
logger.debug(message);
|
|
360
355
|
throw new Error(message);
|
|
@@ -399,7 +394,7 @@ async function getClientCredentialToken(tokensData, logger, keyId, oauthTokens,
|
|
|
399
394
|
auth_data: {
|
|
400
395
|
...(tokensData?.auth_data || {}),
|
|
401
396
|
primary: newAccessToken,
|
|
402
|
-
expires_in: expirationTimestamp.toString()
|
|
397
|
+
expires_in: expirationTimestamp.toString(),
|
|
403
398
|
},
|
|
404
399
|
auth_settings: {
|
|
405
400
|
...(tokensData?.auth_settings || {}),
|
|
@@ -416,7 +411,7 @@ async function getClientCredentialToken(tokensData, logger, keyId, oauthTokens,
|
|
|
416
411
|
updatedData = {
|
|
417
412
|
...tokensData,
|
|
418
413
|
primary: newAccessToken,
|
|
419
|
-
expires_in: expirationTimestamp.toString()
|
|
414
|
+
expires_in: expirationTimestamp.toString(),
|
|
420
415
|
};
|
|
421
416
|
// Ensure required fields are present for old structure
|
|
422
417
|
if (!updatedData.type) updatedData.type = 'oauth2';
|
|
@@ -428,7 +423,7 @@ async function getClientCredentialToken(tokensData, logger, keyId, oauthTokens,
|
|
|
428
423
|
service: 'oauth2_client_credentials',
|
|
429
424
|
tokenURL,
|
|
430
425
|
clientID,
|
|
431
|
-
clientSecret
|
|
426
|
+
clientSecret,
|
|
432
427
|
};
|
|
433
428
|
}
|
|
434
429
|
}
|
|
@@ -39,7 +39,7 @@ function parseKey(str: string = '', teamId: string): string {
|
|
|
39
39
|
export class APIEndpoint extends Component {
|
|
40
40
|
protected configSchema = Joi.object({
|
|
41
41
|
endpoint: Joi.string()
|
|
42
|
-
.pattern(/^[a-zA-Z0-
|
|
42
|
+
.pattern(/^[a-zA-Z0-9_]+([-_][a-zA-Z0-9_]+)*$/)
|
|
43
43
|
.max(50)
|
|
44
44
|
.required(),
|
|
45
45
|
method: Joi.string().valid('POST', 'GET').allow(''), //we're accepting empty value because we consider it POST by default.
|
|
@@ -57,13 +57,25 @@ export class APIEndpoint extends Component {
|
|
|
57
57
|
async process(input, config, agent: Agent) {
|
|
58
58
|
await super.process(input, config, agent);
|
|
59
59
|
|
|
60
|
+
if (typeof config.process === 'function') {
|
|
61
|
+
//special case, APIEndpoint has a custom process method.
|
|
62
|
+
//the inputs are not passed directly to APIEndpoints, we need to read them from the context.
|
|
63
|
+
const contextData = agent?.agentRuntime?.getComponentData(config.id);
|
|
64
|
+
|
|
65
|
+
const inputs = Array.isArray(contextData?.input) ? contextData?.input : [contextData?.input];
|
|
66
|
+
|
|
67
|
+
const result = await config.process.apply(null, inputs);
|
|
68
|
+
return result;
|
|
69
|
+
}
|
|
70
|
+
|
|
60
71
|
const req: AgentRequest = agent.agentRequest;
|
|
61
72
|
const logger = this.createComponentLogger(agent, config);
|
|
62
73
|
|
|
74
|
+
const isTrigger = req.path.startsWith(agent.triggerBasePath);
|
|
63
75
|
const headers = req ? req.headers : {};
|
|
64
|
-
let body = req ? req.body : input; //handle debugger injection
|
|
65
|
-
const params = req ? req.params : {};
|
|
66
|
-
let query = req ? req.query : {};
|
|
76
|
+
let body = req && !isTrigger ? req.body : input; //handle debugger injection
|
|
77
|
+
const params = req && !isTrigger ? req.params : {};
|
|
78
|
+
let query = req && !isTrigger ? req.query : {};
|
|
67
79
|
const _authInfo = req ? req._agent_authinfo : undefined;
|
|
68
80
|
|
|
69
81
|
// parse template variables
|
|
@@ -87,7 +99,7 @@ export class APIEndpoint extends Component {
|
|
|
87
99
|
|
|
88
100
|
// set default value and agent variables
|
|
89
101
|
const inputsWithDefaultValue = config.inputs.filter(
|
|
90
|
-
(input) => input.defaultVal !== undefined && input.defaultVal !== '' && input.defaultVal !== null
|
|
102
|
+
(input) => input.defaultVal !== undefined && input.defaultVal !== '' && input.defaultVal !== null
|
|
91
103
|
);
|
|
92
104
|
|
|
93
105
|
const bodyInputNames: string[] = [];
|
|
@@ -218,7 +230,7 @@ export class APIEndpoint extends Component {
|
|
|
218
230
|
return await binaryInput.getJsonData(AccessCandidate.agent(agent.id));
|
|
219
231
|
}
|
|
220
232
|
return null;
|
|
221
|
-
})
|
|
233
|
+
})
|
|
222
234
|
);
|
|
223
235
|
|
|
224
236
|
// Filter out null values and handle single/multiple results
|
|
@@ -231,4 +243,11 @@ export class APIEndpoint extends Component {
|
|
|
231
243
|
|
|
232
244
|
return { headers, body, query, params, _authInfo, _debug: logger.output };
|
|
233
245
|
}
|
|
246
|
+
|
|
247
|
+
async postProcess(output, config, agent: Agent): Promise<any> {
|
|
248
|
+
if (typeof config.process === 'function') {
|
|
249
|
+
return output?.result;
|
|
250
|
+
}
|
|
251
|
+
return output;
|
|
252
|
+
}
|
|
234
253
|
}
|
|
@@ -144,4 +144,15 @@ export class Component {
|
|
|
144
144
|
hasOutput(id, config, agent: Agent): any {
|
|
145
145
|
return false;
|
|
146
146
|
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* A generic registration function that can be overridden by child classes to register the trigger with providers
|
|
150
|
+
* this function is usually called outside of a workflow in order to register the trigger with providers
|
|
151
|
+
* @param componentId the id of the component to register (in SRE a single trigger instance handles all the triggers of the same type)
|
|
152
|
+
* @param agent the agent instance
|
|
153
|
+
* @param payload any additional payload to pass to the trigger
|
|
154
|
+
*/
|
|
155
|
+
async register(componentId: string, componentSettings: any, payload?: any) {}
|
|
156
|
+
|
|
157
|
+
async unregister(componentId: string, componentSettings: any, payload?: any) {}
|
|
147
158
|
}
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
|
|
2
|
+
import { ManagedOAuth2Credentials } from '@sre/Security/Credentials/ManagedOAuth2Credentials.class';
|
|
3
|
+
import { IAgent as Agent } from '@sre/types/Agent.types';
|
|
4
|
+
import { findAll, innerText, isTag, removeElement } from 'domutils';
|
|
5
|
+
import { parseDocument } from 'htmlparser2';
|
|
6
|
+
import { Trigger } from './Trigger.class';
|
|
7
|
+
export type GmailTriggerAttachment = {
|
|
8
|
+
filename: string;
|
|
9
|
+
mimeType: string;
|
|
10
|
+
size: number;
|
|
11
|
+
attachmentId: string;
|
|
12
|
+
};
|
|
13
|
+
export type GmailTriggerMessage = {
|
|
14
|
+
id: string;
|
|
15
|
+
threadId: string;
|
|
16
|
+
labelIds: string[];
|
|
17
|
+
snippet: string;
|
|
18
|
+
sizeEstimate: number;
|
|
19
|
+
internalDate: string;
|
|
20
|
+
headers: {
|
|
21
|
+
from: string;
|
|
22
|
+
to: string;
|
|
23
|
+
cc: string;
|
|
24
|
+
bcc: string;
|
|
25
|
+
subject: string;
|
|
26
|
+
date: string;
|
|
27
|
+
messageId: string;
|
|
28
|
+
};
|
|
29
|
+
body: {
|
|
30
|
+
text: string;
|
|
31
|
+
html: string;
|
|
32
|
+
};
|
|
33
|
+
attachments: GmailTriggerAttachment[];
|
|
34
|
+
isUnread: true;
|
|
35
|
+
};
|
|
36
|
+
export class GmailTrigger extends Trigger {
|
|
37
|
+
async collectPayload(input, config, agent: Agent) {
|
|
38
|
+
const credentialsKey = config?.data?.oauth_cred_id;
|
|
39
|
+
const agentCandidate = AccessCandidate.agent(agent.id);
|
|
40
|
+
const oauth2Credentials = await ManagedOAuth2Credentials.load(credentialsKey, agentCandidate);
|
|
41
|
+
|
|
42
|
+
console.log(
|
|
43
|
+
oauth2Credentials.accessToken,
|
|
44
|
+
oauth2Credentials.refreshToken,
|
|
45
|
+
oauth2Credentials.expiresIn,
|
|
46
|
+
oauth2Credentials.scope,
|
|
47
|
+
oauth2Credentials.tokenUrl,
|
|
48
|
+
oauth2Credentials.service
|
|
49
|
+
);
|
|
50
|
+
|
|
51
|
+
await oauth2Credentials.refreshAccessToken();
|
|
52
|
+
|
|
53
|
+
const messages = await getMostRecentUnreadMessage(0, oauth2Credentials, 5);
|
|
54
|
+
return messages;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Make an authenticated Gmail API request
|
|
60
|
+
*/
|
|
61
|
+
async function gmailApiRequest(endpoint, accessToken) {
|
|
62
|
+
const url = `https://www.googleapis.com/gmail/v1/users/me${endpoint}`;
|
|
63
|
+
|
|
64
|
+
try {
|
|
65
|
+
const result = await fetch(url, {
|
|
66
|
+
headers: {
|
|
67
|
+
Authorization: `Bearer ${accessToken}`,
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
const json = await result.json();
|
|
71
|
+
console.log(json);
|
|
72
|
+
return json;
|
|
73
|
+
} catch (error) {
|
|
74
|
+
throw new Error(`Gmail API request failed: ${error.message}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Get the most recent unread messages
|
|
80
|
+
*/
|
|
81
|
+
async function getMostRecentUnreadMessage(
|
|
82
|
+
retryCount = 0,
|
|
83
|
+
oauth2Credentials: ManagedOAuth2Credentials,
|
|
84
|
+
numMessages: number = 1,
|
|
85
|
+
excludeMessageId?: string
|
|
86
|
+
) {
|
|
87
|
+
const MAX_RETRIES = 1; // Only retry once to prevent infinite loops
|
|
88
|
+
|
|
89
|
+
let accessToken = oauth2Credentials.accessToken;
|
|
90
|
+
|
|
91
|
+
// If no access token provided, or if requests fail, refresh it
|
|
92
|
+
if (!accessToken) {
|
|
93
|
+
accessToken = await oauth2Credentials.refreshAccessToken();
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
try {
|
|
97
|
+
console.log('Fetching unread messages...');
|
|
98
|
+
|
|
99
|
+
const collectedIds: string[] = [];
|
|
100
|
+
let pageToken: string | undefined = undefined;
|
|
101
|
+
let reachedBoundary = false;
|
|
102
|
+
|
|
103
|
+
while (collectedIds.length < numMessages && !reachedBoundary) {
|
|
104
|
+
const pageSize = Math.min(Math.max(numMessages, 1), 100);
|
|
105
|
+
const pageQuery = `/messages?q=is:unread&maxResults=${pageSize}${pageToken ? `&pageToken=${pageToken}` : ''}`;
|
|
106
|
+
const listResponse = await gmailApiRequest(pageQuery, accessToken);
|
|
107
|
+
|
|
108
|
+
const messages = Array.isArray(listResponse.messages) ? listResponse.messages : [];
|
|
109
|
+
if (messages.length === 0) {
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
for (const m of messages) {
|
|
114
|
+
if (excludeMessageId && m.id === excludeMessageId) {
|
|
115
|
+
reachedBoundary = true;
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
collectedIds.push(m.id);
|
|
119
|
+
if (collectedIds.length >= numMessages) {
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
if (collectedIds.length >= numMessages || reachedBoundary || !listResponse.nextPageToken) {
|
|
125
|
+
break;
|
|
126
|
+
}
|
|
127
|
+
pageToken = listResponse.nextPageToken;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (collectedIds.length === 0) {
|
|
131
|
+
return [];
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// Fetch full message details in parallel
|
|
135
|
+
const details = await Promise.all(collectedIds.map((id) => gmailApiRequest(`/messages/${id}?format=full`, accessToken)));
|
|
136
|
+
const parsed = details.map((d) => parseMessage(d));
|
|
137
|
+
return parsed;
|
|
138
|
+
} catch (error) {
|
|
139
|
+
// If token expired, try refreshing and retry once
|
|
140
|
+
if ((error.message.includes('401') || error.message.includes('unauthorized')) && retryCount < MAX_RETRIES) {
|
|
141
|
+
console.log('Access token expired, refreshing...');
|
|
142
|
+
try {
|
|
143
|
+
accessToken = await oauth2Credentials.refreshAccessToken();
|
|
144
|
+
return await getMostRecentUnreadMessage(retryCount + 1, oauth2Credentials, numMessages, excludeMessageId);
|
|
145
|
+
} catch (refreshError) {
|
|
146
|
+
throw new Error(`Authentication failed: ${refreshError.message}`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
throw error;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Parse Gmail message into structured format
|
|
156
|
+
*/
|
|
157
|
+
function parseMessage(message) {
|
|
158
|
+
const headers: any = {};
|
|
159
|
+
const payload = message.payload || {};
|
|
160
|
+
|
|
161
|
+
// Extract headers
|
|
162
|
+
if (payload.headers) {
|
|
163
|
+
payload.headers.forEach((header) => {
|
|
164
|
+
headers[header.name.toLowerCase()] = header.value;
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Extract body content
|
|
169
|
+
let textBody = '';
|
|
170
|
+
let htmlBody = '';
|
|
171
|
+
|
|
172
|
+
function extractBody(part) {
|
|
173
|
+
if (part.mimeType === 'text/plain' && part.body && part.body.data) {
|
|
174
|
+
textBody = Buffer.from(part.body.data, 'base64').toString('utf-8');
|
|
175
|
+
} else if (part.mimeType === 'text/html' && part.body && part.body.data) {
|
|
176
|
+
htmlBody = Buffer.from(part.body.data, 'base64').toString('utf-8');
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
if (part.parts) {
|
|
180
|
+
part.parts.forEach(extractBody);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (payload.parts) {
|
|
185
|
+
payload.parts.forEach(extractBody);
|
|
186
|
+
} else if (payload.body && payload.body.data) {
|
|
187
|
+
// Single part message
|
|
188
|
+
if (payload.mimeType === 'text/plain') {
|
|
189
|
+
textBody = Buffer.from(payload.body.data, 'base64').toString('utf-8');
|
|
190
|
+
} else if (payload.mimeType === 'text/html') {
|
|
191
|
+
htmlBody = Buffer.from(payload.body.data, 'base64').toString('utf-8');
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
if (textBody && !htmlBody) {
|
|
196
|
+
htmlBody = textBody;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
if (htmlBody && !textBody) {
|
|
200
|
+
textBody = htmlToPlainText(htmlBody);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// Get attachments info
|
|
204
|
+
const attachments = [];
|
|
205
|
+
function findAttachments(part) {
|
|
206
|
+
if (part.filename && part.filename.length > 0) {
|
|
207
|
+
attachments.push({
|
|
208
|
+
filename: part.filename,
|
|
209
|
+
mimeType: part.mimeType,
|
|
210
|
+
size: part.body ? part.body.size : 0,
|
|
211
|
+
attachmentId: part.body ? part.body.attachmentId : null,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
if (part.parts) {
|
|
216
|
+
part.parts.forEach(findAttachments);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
if (payload.parts) {
|
|
221
|
+
payload.parts.forEach(findAttachments);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
return {
|
|
225
|
+
email: {
|
|
226
|
+
id: message.id,
|
|
227
|
+
threadId: message.threadId,
|
|
228
|
+
labelIds: message.labelIds || [],
|
|
229
|
+
snippet: message.snippet || '',
|
|
230
|
+
sizeEstimate: message.sizeEstimate || 0,
|
|
231
|
+
internalDate: message.internalDate ? new Date(parseInt(message.internalDate)).toISOString() : null,
|
|
232
|
+
headers: {
|
|
233
|
+
from: headers.from || '',
|
|
234
|
+
to: headers.to || '',
|
|
235
|
+
cc: headers.cc || '',
|
|
236
|
+
bcc: headers.bcc || '',
|
|
237
|
+
subject: headers.subject || '',
|
|
238
|
+
date: headers.date || '',
|
|
239
|
+
messageId: headers['message-id'] || '',
|
|
240
|
+
},
|
|
241
|
+
body: {
|
|
242
|
+
text: textBody,
|
|
243
|
+
html: htmlBody,
|
|
244
|
+
},
|
|
245
|
+
attachments: attachments,
|
|
246
|
+
isUnread: message.labelIds ? message.labelIds.includes('UNREAD') : false,
|
|
247
|
+
},
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
var populateChar = function (ch, amount) {
|
|
252
|
+
var result = '';
|
|
253
|
+
for (var i = 0; i < amount; i += 1) {
|
|
254
|
+
result += ch;
|
|
255
|
+
}
|
|
256
|
+
return result;
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
function htmlToPlainText(htmlText, _styleConfig?) {
|
|
260
|
+
try {
|
|
261
|
+
const document = parseDocument(String(htmlText));
|
|
262
|
+
const nodesToRemove = findAll((node) => isTag(node) && (node.name === 'script' || node.name === 'style'), document.children);
|
|
263
|
+
nodesToRemove.forEach((node) => removeElement(node));
|
|
264
|
+
|
|
265
|
+
let text = innerText(document);
|
|
266
|
+
|
|
267
|
+
text = text.replace(/\u00A0/g, ' ');
|
|
268
|
+
text = text.replace(/\r\n?/g, '\n');
|
|
269
|
+
text = text.replace(/\t+/g, ' ');
|
|
270
|
+
text = text.replace(/[ \t\f]+\n/g, '\n');
|
|
271
|
+
text = text.replace(/\n{3,}/g, '\n\n');
|
|
272
|
+
text = text.replace(/[ ]{2,}/g, ' ');
|
|
273
|
+
text = text.replace(/^\s+|\s+$/g, '');
|
|
274
|
+
|
|
275
|
+
if (text.length === 0 || text.lastIndexOf('\n') !== text.length - 1) {
|
|
276
|
+
text += '\n';
|
|
277
|
+
}
|
|
278
|
+
return text;
|
|
279
|
+
} catch (err) {
|
|
280
|
+
return String(htmlText);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { IAgent as Agent, IAgent } from '@sre/types/Agent.types';
|
|
2
|
+
import { Trigger } from './Trigger.class';
|
|
3
|
+
import { AgentRequest } from '@sre/AgentManager/AgentRequest.class';
|
|
4
|
+
import express from 'express';
|
|
5
|
+
import { ConnectorService } from '@sre/Core/ConnectorsService';
|
|
6
|
+
import { ISchedulerRequest } from '@sre/AgentManager/Scheduler.service/SchedulerConnector';
|
|
7
|
+
import { Schedule } from '@sre/AgentManager/Scheduler.service/Schedule.class';
|
|
8
|
+
import { Job } from '@sre/AgentManager/Scheduler.service/Job.class';
|
|
9
|
+
import { JSONContent } from '@sre/helpers/JsonContent.helper';
|
|
10
|
+
|
|
11
|
+
export class JobSchedulerTrigger extends Trigger {
|
|
12
|
+
async collectPayload(input, settings, agent: Agent) {
|
|
13
|
+
const req = agent.agentRequest;
|
|
14
|
+
|
|
15
|
+
let payload: any = JSONContent(settings?.data?.payload).tryParse();
|
|
16
|
+
if (typeof payload !== 'object') {
|
|
17
|
+
payload = {};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
return [
|
|
21
|
+
{
|
|
22
|
+
...payload,
|
|
23
|
+
},
|
|
24
|
+
];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async register(componentId: string, componentSettings: any, payload?: { agentData: IAgent; triggerUrl: string }) {
|
|
28
|
+
try {
|
|
29
|
+
const agent: any = payload?.agentData;
|
|
30
|
+
const triggerName = componentSettings.triggerEndpoint;
|
|
31
|
+
const schedulerConnector = ConnectorService.getSchedulerConnector();
|
|
32
|
+
if (!schedulerConnector) {
|
|
33
|
+
throw new Error('Scheduler connector not found');
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const schedulerRequester: ISchedulerRequest = schedulerConnector.agent(agent.id);
|
|
37
|
+
const jobId = `job-${agent.id}-${triggerName}`;
|
|
38
|
+
await schedulerRequester.add(jobId, new Job({ agentId: agent.id, type: 'trigger', triggerName }), Schedule.every('10s'));
|
|
39
|
+
} catch (error) {
|
|
40
|
+
throw new Error('Failed to schedule job');
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async unregister(componentId: string, agent: Agent, payload?: any) {}
|
|
45
|
+
}
|