@smythos/sre 1.5.0 → 1.5.2

package/src/Components/APICall/OAuth.helper.ts

@@ -0,0 +1,294 @@
+// helper.ts
+import crypto from 'crypto';
+import OAuth from 'oauth-1.0a';
+import AccessTokenManager from './AccessTokenManager';
+import { REQUEST_CONTENT_TYPES } from '@sre/constants';
+import axios, { AxiosRequestConfig } from 'axios';
+import { Logger } from '@sre/helpers/Log.helper';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { SystemEvents } from '@sre/Core/SystemEvents';
+
+const console = Logger('OAuth.helper');
+let managedVault: any;
+
+SystemEvents.on('SRE:Booted', () => {
+    try {
+        managedVault = ConnectorService.getManagedVaultConnector();
+    } catch (error) {
+        console.warn('Could not find a compatible ManagedVault connector, OAuth APICalls will not work');
+    }
+});
+
+export function extractAdditionalParamsForOAuth1(reqConfig: AxiosRequestConfig = {}) {
+    let additionalParams = {};
+    // Parse URL parameters using URL and URLSearchParams
+    const url = new URL(reqConfig.url);
+    const searchParams = url.searchParams;
+    additionalParams = Object.fromEntries(searchParams.entries());
+
+    // Check content type and add required parameters for OAuth 1 signature
+    const contentType = reqConfig.headers?.['Content-Type'] || '';
+    if (contentType === REQUEST_CONTENT_TYPES.urlEncodedFormData) {
+        // For form data, include the form parameters in the signature
+        if (typeof reqConfig.data === 'string') {
+            const formData = new URLSearchParams(reqConfig.data);
+            additionalParams = { ...additionalParams, ...Object.fromEntries(formData) };
+        }
+    } else if (contentType === REQUEST_CONTENT_TYPES.json) {
+        // For JSON data, include a hash of the request body
+        if (reqConfig.data) {
+            const hash = crypto.createHash('sha1').update(JSON.stringify(reqConfig.data)).digest('base64');
+            additionalParams['oauth_body_hash'] = hash;
+        }
+    } else if (contentType === REQUEST_CONTENT_TYPES.multipartFormData) {
+        const formData = reqConfig.data as FormData;
+        for (const [key, value] of formData.entries()) {
+            // Exclude binary form data (File, Blob, etc.)
+            if (typeof value === 'object' && value !== null && 'size' in value && 'type' in value) {
+                continue;
+            }
+
+            additionalParams[key] = value;
+        }
+    }
+
+    return additionalParams;
+}
+
+export const buildOAuth1Header = (url, method, oauth1Credentials, additionalParams = {}) => {
+    const oauth = new OAuth({
+        consumer: {
+            key: oauth1Credentials.consumerKey,
+            secret: oauth1Credentials.consumerSecret,
+        },
+        signature_method: 'HMAC-SHA1',
+        hash_function(base_string, key) {
+            return crypto.createHmac('sha1', key).update(base_string).digest('base64');
+        },
+    });
+
+    // Include additional parameters if necessary (e.g., for non-standard providers)
+    const requestData = {
+        url,
+        method,
+        ...additionalParams,
+    };
+    const signedRequest = oauth.authorize(requestData, { key: oauth1Credentials.token, secret: oauth1Credentials.tokenSecret });
+    return oauth.toHeader(signedRequest);
+};
+
+export const retrieveOAuthTokens = async (agent, config) => {
+    let tokenKey: any = null;
+    try {
+        tokenKey = `OAUTH_${config.componentId ?? config.id}_TOKENS`;
+
+        try {
+            const result: any = await managedVault.user(AccessCandidate.agent(agent.id)).get(tokenKey);
+            const tokensData = typeof result === 'object' ? result : JSON.parse(result || '{}');
+
+            if (!tokensData) {
+                throw new Error('Failed to retrieve OAuth tokens from vault. Please authenticate ...');
+            }
+
+            const primaryToken = tokensData.primary; // accessToken or token
+            const secondaryToken = tokensData.secondary; // refreshToken or tokenSecret
+            const type = tokensData.type; // oauth || oauth2
+
+            // Add warning logs for OAuth2
+            if (type === 'oauth2' && config.data.oauthService !== 'OAuth2 Client Credentials') {
+                if (!secondaryToken) {
+                    console.warn('Warning: refresh_token is missing for OAuth2');
+                }
+                if (!tokensData.expires_in) {
+                    console.warn('Warning: expires_in is missing for OAuth2.');
+                }
+            }
+
+            // sometimes refreshToken is not available . e.g in case of linkedIn. so only add check for primary token
+            if (config.data.oauthService !== 'OAuth2 Client Credentials') {
+                if (!primaryToken) {
+                    throw new Error('Retrieved OAuth tokens do not exist, invalid OR incomplete. Please authenticate ...');
+                }
+            }
+
+            const responseData: any = {
+                primaryToken,
+                secondaryToken,
+                type,
+            };
+
+            if (type === 'oauth') {
+                // Check and assign if present
+                if ('consumerKey' in tokensData) responseData.consumerKey = tokensData.consumerKey;
+                if ('consumerSecret' in tokensData) responseData.consumerSecret = tokensData.consumerSecret;
+                responseData.team = tokensData.team;
+            } else if (type === 'oauth2') {
+                // Check and assign if present
+                responseData.tokenURL = tokensData.tokenURL;
+                if ('clientID' in tokensData) responseData.clientID = tokensData.clientID;
+                if ('clientSecret' in tokensData) responseData.clientSecret = tokensData.clientSecret;
+                responseData.expiresIn = tokensData.expires_in ?? 0; // Optional property, default to 0 if not present. time to expire access token
+                responseData.team = tokensData.team;
+            }
+
+            return { responseData, data: tokensData, keyId: tokenKey };
+        } catch (error) {
+            throw new Error(`Failed to parse retrieved tokens: ${error}`);
+        }
+    } catch (error) {
+        console.error('Error retrieving OAuth tokens:', error);
+        throw error; // rethrow for potential handling by the calling code
+    }
+};
+
+export const handleOAuthHeaders = async (agent, config, reqConfig, logger, additionalParams = {}, rootUrl) => {
+    let headers = {}; // Initialize headers as an empty object
+    const { responseData: oauthTokens, data, keyId } = await retrieveOAuthTokens(agent, config);
+
+    try {
+        // Extract template variable key IDs for consumerKey, consumerSecret, clientID, and clientSecret
+        const keys = ['consumerKey', 'consumerSecret', 'clientID', 'clientSecret'];
+        let oAuthConfigString = JSON.stringify({
+            consumerKey: config.data.consumerKey,
+            consumerSecret: config.data.consumerSecret,
+            clientID: config.data.clientID,
+            clientSecret: config.data.clientSecret,
+            tokenURL: config.data.tokenURL,
+        });
+
+        oAuthConfigString = await TemplateString(oAuthConfigString).parseTeamKeysAsync(oauthTokens.team || agent.teamId).asyncResult;
+
+        const oAuthConfig = JSON.parse(oAuthConfigString);
+
+        if (oAuthConfig.oauthService === 'OAuth2 Client Credentials') {
+            const accessToken = await getClientCredentialToken(data, logger, keyId, oauthTokens, config, agent);
+            headers['Authorization'] = `Bearer ${accessToken}`;
+        } else {
+            if (oauthTokens.type === 'oauth') {
+                // For OAuth1, generate and replace the signature in headers
+                const oauthHeader = buildOAuth1Header(
+                    rootUrl,
+                    reqConfig.method,
+                    {
+                        consumerKey: oAuthConfig.consumerKey,
+                        consumerSecret: oAuthConfig.consumerSecret,
+                        token: oauthTokens.primaryToken,
+                        tokenSecret: oauthTokens.secondaryToken,
+                    },
+                    additionalParams,
+                );
+
+                headers = { ...reqConfig.headers, ...oauthHeader };
+                logger.debug('OAuth1 access token check success.');
+            } else if (oauthTokens.type === 'oauth2') {
+                // For OAuth2, add the 'Authorization' header with the bearer token
+                const accessTokenManager = new AccessTokenManager(
+                    oAuthConfig.clientID,
+                    oAuthConfig.clientSecret,
+                    oauthTokens.secondaryToken,
+                    oAuthConfig.tokenURL,
+                    oauthTokens.expiresIn,
+                    oauthTokens.primaryToken,
+                    data,
+                    keyId,
+                    logger,
+                    agent,
+                );
+
+                const accessToken = await accessTokenManager.getAccessToken();
+                headers['Authorization'] = `Bearer ${accessToken}`;
+            }
+        }
+        return headers;
+    } catch (error) {
+        logger.error(`Access token check failed: ${error}`);
+        throw error;
+    }
+};
+
+const getKeyIdsFromTemplateVars = (str: string): string[] => {
+    if (!str) return [];
+
+    const pattern = /{{KEY\((.*?)\)}}/g;
+    const keyIds: any = [];
+    let match: any = [];
+
+    while ((match = pattern.exec(str)) !== null) {
+        if (match?.length < 2) continue;
+        keyIds.push(match[1]);
+    }
+
+    return keyIds;
+};
+
+async function getClientCredentialToken(data, logger, keyId, oauthTokens, config, agent) {
+    const logAndThrowError = (message) => {
+        logger.debug(message);
+        throw new Error(message);
+    };
+
+    try {
+        data = data[keyId] || {};
+        const { clientID, clientSecret, tokenURL } = config.data;
+        const currentTime = new Date().getTime();
+        // Check for token expiration
+        if (!oauthTokens.expiresIn || currentTime >= Number(oauthTokens.expiresIn)) {
+            // Verify required parameters
+            if (!clientID || !clientSecret || !tokenURL) {
+                logAndThrowError('Missing client_id, client_secret OR token_url');
+            }
+
+            const params = new URLSearchParams({
+                grant_type: 'client_credentials',
+                client_id: clientID,
+                client_secret: clientSecret,
+            });
+
+            const response = await axios.post(tokenURL, params.toString(), {
+                headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+            });
+
+            console.log('Access token refreshed successfully.');
+            logger.debug('Access token refreshed successfully.');
+
+            const newAccessToken = response.data.access_token;
+            const expiresInMilliseconds = response.data.expires_in * 1000;
+            const expirationTimestamp = currentTime + expiresInMilliseconds;
+
+            // Set data if it's empty
+            if (Object.keys(data).length === 0) {
+                data = {
+                    primary: '',
+                    secondary: '',
+                    type: 'oauth2',
+                    tokenURL,
+                    expires_in: '',
+                    team: agent.teamId,
+                    oauth_info: {
+                        oauth_keys_prefix: `OAUTH_${config.componentId ?? config.id}`,
+                        service: 'oauth2_client_credentials',
+                        tokenURL,
+                        clientID,
+                        clientSecret,
+                    },
+                };
+            }
+
+            data.primary = newAccessToken;
+            data.expires_in = expirationTimestamp.toString();
+            //const oauthTeamSettings = new OauthTeamSettings();
+            //const save = await oauthTeamSettings.update({ keyId: keyId, data: data });
+            await managedVault.user(AccessCandidate.agent(agent.id)).set(keyId, data);
+
+            return newAccessToken;
+        } else {
+            console.log('Access token value is still valid.');
+            logger.debug('Access token value is still valid.');
+            return oauthTokens.primaryToken;
+        }
+    } catch (error) {
+        logAndThrowError(`Failed to refresh access token: ${error}`);
+    }
+}

package/src/Components/APICall/mimeTypeCategories.ts

@@ -0,0 +1,46 @@
+const mimeTypeCategories = {
+    binary: [
+        'image/',
+        'multipart/form-data',
+        'video/',
+        'application/msword',
+        'application/octet-stream',
+        'application/pdf',
+        'application/vnd.ms-excel',
+        'application/vnd.ms-powerpoint',
+        'application/vnd.oasis.opendocument.text',
+        'application/vnd.openxmlformats-officedocument',
+        'application/zip',
+        'application/x-7z-compressed',
+        'application/x-rar-compressed',
+        'application/x-tar',
+        'application/x-bzip',
+        'application/x-bzip2',
+        'application/x-gzip',
+        'application/vnd.android.package-archive',
+        'application/vnd.visio',
+        'application/x-deb',
+        'application/x-rpm',
+        'application/x-executable',
+        'font/ttf',
+        'font/otf',
+        'font/woff',
+        'font/woff2',
+        'model/',
+    ],
+    json: ['application/graphql', 'application/json', 'application/ld+json', 'application/vnd.api+json'],
+    text: [
+        'text/', //all starting with text/
+        'application/xml',
+        'application/xhtml+xml',
+        'application/csv',
+        'application/x-www-form-urlencoded',
+        'application/x-yaml',
+        'application/yaml',
+        'application/javascript',
+        'application/sql',
+        'application/rtf',
+    ],
+};
+
+export default mimeTypeCategories;

package/src/Components/APICall/parseData.ts

@@ -0,0 +1,167 @@
+import { Agent } from '@sre/AgentManager/Agent.class';
+import { REQUEST_CONTENT_TYPES } from '@sre/constants';
+import { JSONContent } from '@sre/helpers/JsonContent.helper';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { BinaryInput } from '@sre/helpers/BinaryInput.helper';
+import FormData from 'form-data';
+import { Readable } from 'stream';
+
+export async function parseData(input: any, config, agent: Agent): Promise<{ data: any; headers: any }> {
+    const teamId = agent ? agent.teamId : null;
+    const templateSettings = config?.template?.settings || {};
+    const contentType = config?.data?.contentType || REQUEST_CONTENT_TYPES.none;
+
+    let body = typeof config?.data?.body === 'string' ? config?.data?.body?.trim() : config?.data?.body;
+    if (!body) {
+        return { data: null, headers: {} };
+    }
+
+    // Parse component template variables
+    if (config.data._templateVars && templateSettings) {
+        //prettier-ignore
+        body = await TemplateString(body) 
+            .parseComponentTemplateVarsAsync(templateSettings) // replace component template vars with their IDs (this turns the string parses into an async parser)            
+            .asyncResult; //returns parser promise
+    }
+
+    // Parse vault keys
+    body = await TemplateString(body).parseTeamKeysAsync(teamId).asyncResult;
+
+    // Handle different content types
+    const handlers: Record<string, (parsedBody: any, input: any, config, agent: Agent) => any> = {
+        [REQUEST_CONTENT_TYPES.json]: handleJson,
+        [REQUEST_CONTENT_TYPES.urlEncodedFormData]: handleUrlEncoded,
+        [REQUEST_CONTENT_TYPES.multipartFormData]: handleMultipartFormData,
+        [REQUEST_CONTENT_TYPES.binary]: handleBinary,
+        [REQUEST_CONTENT_TYPES.text]: handleText,
+        [REQUEST_CONTENT_TYPES.none]: handleNone,
+    };
+
+    const handler = handlers[contentType] || handleNone;
+    const { data = null, headers = {} } = await handler(body, input, config, agent);
+
+    //const jsonBody: any = JSONContent(data).tryParse();
+    return { data, headers };
+}
+
+async function handleJson(body: any, input: any, config, agent: Agent) {
+    // Parse template and input variables
+    //prettier-ignore
+    const data = TemplateString(body)
+        .parse(config.data._templateVars) //parse Template variables first (if any)
+        .parse(input) //parse inputs
+        .clean().result; //clean up the remaining unparsed values
+
+    const jsonBody: any = JSONContent(data).tryParse();
+    return { data: jsonBody };
+}
+
+async function handleUrlEncoded(body: any, input: any, config, agent: Agent) {
+    const data = TemplateString(body)
+        .parse(config.data._templateVars) //parse Template variables first (if any)
+        .parse(input) //parse inputs
+        .clean().result; //clean up the remaining unparsed values
+
+    const jsonData: any = JSONContent(data).tryParse();
+
+    if (typeof jsonData === 'object') {
+        const params = new URLSearchParams();
+        for (const key in jsonData) {
+            params.append(key, String(jsonData[key]));
+        }
+        return { data: params.toString() };
+    }
+
+    return { data: jsonData };
+}
+
+async function handleMultipartFormData(body: any, input: any, config, agent: Agent) {
+    const formData = new FormData();
+
+    const _body = typeof body === 'string' ? JSON.parse(body) : body;
+
+    for (const key in _body) {
+        let value = _body[key];
+        value = typeof value === 'boolean' ? String(value) : value;
+
+        value = TemplateString(value).parseRaw(input).result;
+
+        // * Note: It's important to check if the value is an instance of BinaryInput first.
+        // Otherwise, condition like (value && typeof value === 'object' && value?.url)
+        // might be true and lead to incorrect results.
+        if (value instanceof BinaryInput) {
+            const buffer = await value.getBuffer();
+            const bufferStream = new Readable();
+            bufferStream.push(buffer || null);
+            bufferStream.push(null);
+
+            const filename = (await value.getName()) || key;
+            formData.append(key, bufferStream, {
+                filename,
+                contentType: value.mimetype,
+            });
+        } else if (value && typeof value === 'object' && value?.url) {
+            // Retro compatibility with old binary data structure {url: '...', mimetype: '...', url: 'http(s)://...'}
+            const binaryInput = await BinaryInput.from(value.url, '', value?.mimetype);
+            const buffer = await binaryInput.getBuffer();
+
+            const bufferStream = new Readable();
+            bufferStream.push(buffer || null);
+            bufferStream.push(null);
+
+            const filename = (await binaryInput.getName()) || key;
+            formData.append(key, bufferStream, {
+                filename,
+                contentType: binaryInput.mimetype,
+            });
+        } else {
+            value = TemplateString(value)
+                .parse(config.data._templateVars) //parse Template variables first (if any)
+                .parse(input)
+                .clean().result;
+
+            if (value) {
+                formData.append(key, value);
+            }
+
+            //formData.append(key, typeof value === 'boolean' ? String(value) : value);
+        }
+    }
+    return { data: formData, headers: formData.getHeaders() };
+}
+
+async function handleBinary(body: any, input: any, config, agent: Agent) {
+    const value: any = TemplateString(body).parseRaw(input).result;
+
+    // * Note: It's important to check if the value is an instance of BinaryInput first.
+    // Otherwise, condition like (value && typeof value === 'object' && value?.url)
+    // might be true and lead to incorrect results.
+    if (value && value instanceof BinaryInput) {
+        const buffer = await value.getBuffer();
+        return { data: buffer, headers: { 'Content-Type': value.mimetype } };
+    } else if (value && typeof value === 'object' && value?.url) {
+        // Retro compatibility with old binary data structure {url: '...', mimetype: '...', url: 'http(s)://...'}
+        const binaryInput = await BinaryInput.from(value.url, '', value?.mimetype);
+        const buffer = await binaryInput.getBuffer();
+
+        return { data: buffer, headers: { 'Content-Type': binaryInput.mimetype } };
+    }
+
+    return { data: Buffer.from([]), headers: {} };
+}
+
+async function handleNone(body: any, input: any, config, agent: Agent) {
+    //FIXME: try to guess the content type from headers content-type and data
+
+    return { data: typeof body === 'string' ? body : JSON.stringify(body), headers: {} };
+}
+function handleText(body: any, input: any, config: any, agent: Agent) {
+    // Parse template and input variables
+    //prettier-ignore
+    const data = TemplateString(body)
+        .parse(config.data._templateVars) //parse Template variables first (if any)
+        .parse(input) //parse inputs
+        .clean().result; //clean up the remaining unparsed values
+
+    return { data };
+}

package/src/Components/APICall/parseHeaders.ts

@@ -0,0 +1,41 @@
+import { Agent } from '@sre/AgentManager/Agent.class';
+import { REQUEST_CONTENT_TYPES } from '@sre/constants';
+import { JSONContent } from '@sre/helpers/JsonContent.helper';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { AxiosHeaders } from 'axios';
+
+export async function parseHeaders(input, config, agent: Agent) {
+    const teamId = agent ? agent.teamId : null;
+    const templateSettings = config?.template?.settings || {};
+    const contentType = config?.data?.contentType || REQUEST_CONTENT_TYPES.none;
+    let headers = typeof config?.data?.headers == 'object' ? JSON.stringify(config?.data?.headers) : config?.data?.headers || '{}';
+
+    //parse component template vars
+    if (config.data._templateVars && templateSettings) {
+        headers = await TemplateString(headers).parseComponentTemplateVarsAsync(templateSettings).asyncResult; // replaces component template vars with their IDs (this turns the string parses into an async parser) // replaces IDs with actual values then returns parser promise
+
+        headers = await TemplateString(headers).parse(config.data._templateVars).result;
+    }
+
+    //parse vault keys
+    headers = await TemplateString(headers).parseTeamKeysAsync(teamId).asyncResult;
+
+    //parse input variables and clean up the remaining unparsed values
+    headers = TemplateString(headers).parse(input).clean().result;
+
+    //parse headers as json
+    let jsonHeaders: any = JSONContent(headers).tryParse();
+    if (typeof jsonHeaders !== 'object') {
+        jsonHeaders = { 'x-smyth-error': 'Error parsing headers' };
+    }
+
+    //normalize headers key names to lowercase
+    jsonHeaders = Object.fromEntries(Object.entries(jsonHeaders).map(([key, value]) => [key.toLowerCase(), value]));
+
+    //if headers does not contain content-type, add it
+    if (!jsonHeaders['content-type'] && contentType !== 'none') {
+        jsonHeaders['content-type'] = contentType;
+    }
+
+    return new AxiosHeaders(jsonHeaders);
+}

package/src/Components/APICall/parseProxy.ts

@@ -0,0 +1,68 @@
+import { AxiosProxyConfig } from 'axios';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+
+import { Agent } from '@sre/AgentManager/Agent.class';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+
+export async function parseProxy(input, config, agent: Agent): Promise<AxiosProxyConfig | SocksProxyAgent | false> {
+    const teamId = agent ? agent.teamId : null;
+    const templateSettings = config?.template?.settings || {};
+
+    let proxy = config?.data?.proxy;
+
+    if (!proxy) {
+        return false;
+    }
+
+    proxy = decodeURIComponent(proxy); //decode the url in order to parse the template vars
+
+    //parse component template vars
+    if (config.data._templateVars && templateSettings) {
+        proxy = await TemplateString(proxy)
+            .parseComponentTemplateVarsAsync(templateSettings) // replaces component template vars with their IDs (this turns the string parses into an async parser)
+            .parse(config.data._templateVars).asyncResult; // replaces IDs with actual values then returns parser promise
+    }
+
+    //parse vault keys
+    proxy = await TemplateString(proxy).parseTeamKeysAsync(teamId).asyncResult;
+
+    //parse input variables and clean up the remaining unparsed values
+    proxy = TemplateString(proxy).parse(input).clean().result;
+
+    const proxyList = proxy.split(/\n|\\n/).filter((p) => p) || [];
+
+    const randomIdx = Math.floor(Math.random() * proxyList?.length);
+    const proxyUrl = proxyList[randomIdx]?.trim();
+
+    //URL will take care of encoding the url properly
+    const urlObj = new URL(proxyUrl);
+    const protocol = urlObj.protocol.replace(':', ''); // As urlObj.protocol is like 'http:'
+
+    let proxyConfig: AxiosProxyConfig | SocksProxyAgent;
+
+    if (urlObj.protocol.startsWith('socks')) {
+        let proxyUrlString = `${protocol}://${urlObj.hostname}:${urlObj.port}`;
+
+        if ((protocol === 'socks4' || protocol === 'socks4a') && urlObj.username) {
+            proxyUrlString = `${protocol}://${urlObj.username}@${urlObj.hostname}:${urlObj.port}`;
+        } else if (protocol === 'socks5' && urlObj.username) {
+            proxyUrlString = `${protocol}://${urlObj.username}:${urlObj.password}@${urlObj.hostname}:${urlObj.port}`;
+        }
+
+        proxyConfig = new SocksProxyAgent(proxyUrlString);
+    } else {
+        proxyConfig = {
+            protocol,
+            host: urlObj.hostname,
+            port: parseInt(urlObj.port),
+            auth: urlObj.username
+                ? {
+                      username: urlObj.username,
+                      password: urlObj.password,
+                  }
+                : undefined,
+        };
+    }
+
+    return proxyConfig;
+}