@smythos/sre 1.5.1 → 1.5.2

package/LICENSE

@@ -0,0 +1,18 @@
+The MIT License
+
+Copyright 2025, INK Content, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -1,128 +1,129 @@
 {
-    "name": "@smythos/sre",
-    "version": "1.5.1",
-    "description": "Smyth Runtime Environment",
-    "author": "Alaa-eddine KADDOURI",
-    "license": "MIT",
-    "main": "dist/index.js",
-    "types": "dist/types/index.d.ts",
-    "exports": {
-        ".": {
-            "import": "./dist/index.js",
-            "types": "./dist/types/index.d.ts"
-        }
-    },
-    "files": [
-        "dist",
-        "README.md",
-        "CHANGELOG"
-    ],
-    "type": "module",
-    "scripts": {
-        "gen:barrel": "ctix build",
-        "build:types": "tsc --emitDeclarationOnly --declaration --outDir dist/types -p tsconfig.dts.json",
-        "build:jsbundle": "cross-env rollup -c",
-        "build": "pnpm run build:jsbundle && pnpm run build:types",
-        "doc:gen": "typedoc",
-        "doc:graphgen": "npx depcruise src --config ./doc/.dep-minimal.json --output-type dot > ./doc/dep-graph.dot && dot -Tpng ./doc/dep-graph.dot -o ./doc/dep-graph.png",
-        "knip": "knip"
-    },
-    "devDependencies": {
-        "@faker-js/faker": "^8.4.1",
-        "@istanbuljs/nyc-config-typescript": "^1.0.2",
-        "@rollup/plugin-json": "^6.1.0",
-        "@rollup/plugin-node-resolve": "^15.2.3",
-        "@rollup/pluginutils": "^5.1.0",
-        "@types/cookie-parser": "^1.4.7",
-        "@types/express": "^4.17.23",
-        "@types/express-session": "^1.18.0",
-        "@types/lodash": "^4.17.10",
-        "@types/node": "^20.19.0",
-        "cross-env": "^7.0.3",
-        "ctix": "^2.7.1",
-        "dependency-cruiser": "^16.3.3",
-        "esbuild": "^0.25.0",
-        "knip": "^5.23.1",
-        "nyc": "^17.0.0",
-        "pkg": "^5.8.1",
-        "rollup-plugin-copy": "^3.5.0",
-        "rollup-plugin-esbuild": "^6.1.1",
-        "rollup-plugin-sourcemaps": "^0.6.3",
-        "rollup-plugin-terser": "^7.0.2",
-        "rollup-plugin-typescript-paths": "^1.5.0",
-        "rollup-plugin-typescript2": "^0.36.0",
-        "source-map-support": "^0.5.21",
-        "typedoc": "^0.28.5",
-        "typedoc-github-theme": "^0.3.0",
-        "typescript": "^5.4.5"
-    },
-    "dependencies": {
-        "@anthropic-ai/sdk": "^0.39.0",
-        "@apidevtools/json-schema-ref-parser": "^11.6.4",
-        "@aws-sdk/client-bedrock-runtime": "^3.826.0",
-        "@aws-sdk/client-iam": "^3.826.0",
-        "@aws-sdk/client-lambda": "^3.826.0",
-        "@aws-sdk/client-s3": "^3.826.0",
-        "@aws-sdk/client-secrets-manager": "^3.826.0",
-        "@google-cloud/vertexai": "^1.7.0",
-        "@google/generative-ai": "^0.14.1",
-        "@huggingface/inference": "^2.8.0",
-        "@modelcontextprotocol/sdk": "^1.10.1",
-        "@pinecone-database/pinecone": "^3.0.0",
-        "@runware/sdk-js": "^1.1.36",
-        "@smithy/smithy-client": "^4.4.3",
-        "@zilliz/milvus2-sdk-node": "^2.5.11",
-        "acorn": "^8.14.1",
-        "axios": "^1.7.2",
-        "chokidar": "^4.0.3",
-        "commander": "^11.1.0",
-        "cookie-parser": "^1.4.7",
-        "cors": "^2.8.5",
-        "dayjs": "^1.11.11",
-        "dotenv": "^16.4.5",
-        "eventsource": "^3.0.2",
-        "express": "^4.21.2",
-        "express-async-handler": "^1.2.0",
-        "express-session": "^1.18.1",
-        "file-type": "^19.0.0",
-        "form-data": "^4.0.3",
-        "gpt-tokenizer": "^2.2.1",
-        "groq-sdk": "^0.6.1",
-        "image-size": "^1.1.1",
-        "ioredis": "^5.4.1",
-        "isbinaryfile": "^5.0.2",
-        "joi": "^17.13.1",
-        "js-yaml": "^4.1.0",
-        "jsonrepair": "^3.8.0",
-        "jsonwebtoken": "^9.0.2",
-        "jwks-rsa": "^3.1.0",
-        "lodash": "^4.17.21",
-        "memorystore": "^1.6.7",
-        "mime": "^4.0.3",
-        "multer": "^1.4.5-lts.1",
-        "mysql2": "^3.11.3",
-        "oauth-1.0a": "^2.2.6",
-        "openai": "^4.103.0",
-        "p-limit": "^6.1.0",
-        "pkce-challenge": "^5.0.0",
-        "qs": "^6.13.0",
-        "rate-limiter-flexible": "^5.0.3",
-        "readline-sync": "^1.4.10",
-        "socket.io-client": "^4.8.1",
-        "socks-proxy-agent": "^8.0.4",
-        "uuid": "^10.0.0",
-        "winston": "^3.13.0",
-        "winston-transport": "^4.7.0",
-        "xxhashjs": "^0.2.2",
-        "zip-lib": "^1.0.5"
-    },
-    "repository": {
-        "type": "git",
-        "url": "https://github.com/smythos/sre.git",
-        "directory": "packages/core"
-    },
-    "bugs": {
-        "url": "https://github.com/smythos/sre/issues"
-    },
-    "homepage": "https://github.com/smythos/sre/tree/main/packages/core#readme"
-}
+  "name": "@smythos/sre",
+  "version": "1.5.2",
+  "description": "Smyth Runtime Environment",
+  "author": "Alaa-eddine KADDOURI",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "types": "dist/types/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/types/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "README.md",
+    "CHANGELOG"
+  ],
+  "type": "module",
+  "devDependencies": {
+    "@faker-js/faker": "^8.4.1",
+    "@istanbuljs/nyc-config-typescript": "^1.0.2",
+    "@rollup/plugin-json": "^6.1.0",
+    "@rollup/plugin-node-resolve": "^15.2.3",
+    "@rollup/pluginutils": "^5.1.0",
+    "@types/cookie-parser": "^1.4.7",
+    "@types/express": "^4.17.23",
+    "@types/express-session": "^1.18.0",
+    "@types/lodash": "^4.17.10",
+    "@types/node": "^20.19.0",
+    "cross-env": "^7.0.3",
+    "ctix": "^2.7.1",
+    "dependency-cruiser": "^16.3.3",
+    "esbuild": "^0.25.0",
+    "knip": "^5.23.1",
+    "nyc": "^17.0.0",
+    "pkg": "^5.8.1",
+    "rollup-plugin-copy": "^3.5.0",
+    "rollup-plugin-esbuild": "^6.1.1",
+    "rollup-plugin-sourcemaps": "^0.6.3",
+    "rollup-plugin-terser": "^7.0.2",
+    "rollup-plugin-typescript-paths": "^1.5.0",
+    "rollup-plugin-typescript2": "^0.36.0",
+    "source-map-support": "^0.5.21",
+    "typedoc": "^0.28.5",
+    "typedoc-github-theme": "^0.3.0",
+    "typescript": "^5.4.5"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.39.0",
+    "@apidevtools/json-schema-ref-parser": "^11.6.4",
+    "@aws-sdk/client-bedrock-runtime": "^3.826.0",
+    "@aws-sdk/client-iam": "^3.826.0",
+    "@aws-sdk/client-lambda": "^3.826.0",
+    "@aws-sdk/client-s3": "^3.826.0",
+    "@aws-sdk/client-secrets-manager": "^3.826.0",
+    "@google-cloud/vertexai": "^1.7.0",
+    "@google/generative-ai": "^0.14.1",
+    "@huggingface/inference": "^2.8.0",
+    "@modelcontextprotocol/sdk": "^1.10.1",
+    "@pinecone-database/pinecone": "^3.0.0",
+    "@runware/sdk-js": "^1.1.36",
+    "@smithy/smithy-client": "^4.4.3",
+    "@zilliz/milvus2-sdk-node": "^2.5.11",
+    "acorn": "^8.14.1",
+    "axios": "^1.7.2",
+    "chokidar": "^4.0.3",
+    "commander": "^11.1.0",
+    "cookie-parser": "^1.4.7",
+    "cors": "^2.8.5",
+    "dayjs": "^1.11.11",
+    "dotenv": "^16.4.5",
+    "eventsource": "^3.0.2",
+    "express": "^4.21.2",
+    "express-async-handler": "^1.2.0",
+    "express-session": "^1.18.1",
+    "file-type": "^19.0.0",
+    "form-data": "^4.0.3",
+    "gpt-tokenizer": "^2.2.1",
+    "groq-sdk": "^0.6.1",
+    "image-size": "^1.1.1",
+    "ioredis": "^5.4.1",
+    "isbinaryfile": "^5.0.2",
+    "joi": "^17.13.1",
+    "js-yaml": "^4.1.0",
+    "jsonrepair": "^3.8.0",
+    "jsonwebtoken": "^9.0.2",
+    "jwks-rsa": "^3.1.0",
+    "lodash": "^4.17.21",
+    "memorystore": "^1.6.7",
+    "mime": "^4.0.3",
+    "multer": "^1.4.5-lts.1",
+    "mysql2": "^3.11.3",
+    "oauth-1.0a": "^2.2.6",
+    "openai": "^4.103.0",
+    "p-limit": "^6.1.0",
+    "pkce-challenge": "^5.0.0",
+    "qs": "^6.13.0",
+    "rate-limiter-flexible": "^5.0.3",
+    "readline-sync": "^1.4.10",
+    "socket.io-client": "^4.8.1",
+    "socks-proxy-agent": "^8.0.4",
+    "uuid": "^10.0.0",
+    "winston": "^3.13.0",
+    "winston-transport": "^4.7.0",
+    "xxhashjs": "^0.2.2",
+    "zip-lib": "^1.0.5"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/smythos/sre.git",
+    "directory": "packages/core"
+  },
+  "bugs": {
+    "url": "https://github.com/smythos/sre/issues"
+  },
+  "homepage": "https://github.com/smythos/sre/tree/main/packages/core#readme",
+  "scripts": {
+    "gen:barrel": "ctix build",
+    "build:types": "tsc --emitDeclarationOnly --declaration --outDir dist/types -p tsconfig.dts.json",
+    "build:jsbundle": "cross-env rollup -c",
+    "build": "pnpm run build:jsbundle && pnpm run build:types",
+    "doc:gen": "typedoc",
+    "doc:graphgen": "npx depcruise src --config ./doc/.dep-minimal.json --output-type dot > ./doc/dep-graph.dot && dot -Tpng ./doc/dep-graph.dot -o ./doc/dep-graph.png",
+    "knip": "knip"
+  }
+}

package/src/Components/APICall/APICall.class.ts

@@ -0,0 +1,155 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import Joi from 'joi';
+
+import { Agent } from '@sre/AgentManager/Agent.class';
+import { Component } from '../Component.class';
+import { parseHeaders } from './parseHeaders';
+import { parseUrl, parseSmythFsUrl, destroyPublicUrls } from './parseUrl';
+import { parseData } from './parseData';
+import { parseProxy } from './parseProxy';
+import { parseArrayBufferResponse } from './ArrayBufferResponse.helper';
+import { extractAdditionalParamsForOAuth1, handleOAuthHeaders as generateOAuthHeaders } from './OAuth.helper';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import { formatDataForDebug } from '@sre/utils/data.utils';
+
+export class APICall extends Component {
+    protected schema = {
+        name: 'APICall',
+        description: 'Use this component to make an API call',
+        inputs: {},
+        outputs: {
+            Headers: {
+                description: 'The headers of the API call response',
+                default: true,
+            },
+            Response: {
+                description: 'The response of the API call',
+                default: true,
+            },
+        },
+    };
+
+    protected configSchema = Joi.object({
+        method: Joi.string().valid('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS').required().label('Method'),
+        url: Joi.string()
+            .max(8192) /*.custom(isUrlValid, 'URL validation')*/
+            .required()
+            .label('URL'),
+        headers: Joi.any().allow('').label('Headers'),
+        contentType: Joi.string()
+            .valid('none', 'application/json', 'multipart/form-data', 'binary', 'application/x-www-form-urlencoded', 'text/plain', 'application/xml')
+            .label('Content-Type'),
+        body: Joi.any().allow('').label('Body'),
+        _templateSettings: Joi.object().allow(null).label('Template Settings'),
+        _templateVars: Joi.object().allow(null).label('Template Variables'),
+        proxy: Joi.string().allow('').label('Proxy'),
+        oauthService: Joi.string().allow('').label('OAuth Service'),
+        scope: Joi.string().allow('').label('Scope'),
+        authorizationURL: Joi.string().allow('').label('Authorization URL'),
+        tokenURL: Joi.string().allow('').label('Token URL'),
+        clientID: Joi.string().allow('').label('Client ID'),
+        clientSecret: Joi.string().allow('').label('Client Secret'),
+        oauth2CallbackURL: Joi.string().allow('').label('OAuth2 Callback URL'),
+        callbackURL: Joi.string().allow('').label('Callback URL'), // !TEMP: prevent validation error
+        requestTokenURL: Joi.string().allow('').label('Request Token URL'),
+        accessTokenURL: Joi.string().allow('').label('Access Token URL'),
+        userAuthorizationURL: Joi.string().allow('').label('User Authorization URL'),
+        consumerKey: Joi.string().allow('').label('Consumer Key'),
+        consumerSecret: Joi.string().allow('').label('Consumer Secret'),
+        oauth1CallbackURL: Joi.string().allow('').label('OAuth1 Callback URL'),
+        authenticate: Joi.string().allow('').label('Authenticate'),
+    });
+    constructor() {
+        super();
+    }
+
+    init() {}
+
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+
+        const logger = this.createComponentLogger(agent, config);
+
+        let publicUrls: string[] = [];
+
+        try {
+            logger.debug(`=== API Call Log ===`);
+
+            const method = config?.data?.method || 'get';
+
+            const reqConfig: AxiosRequestConfig = {};
+            reqConfig.method = method;
+
+            reqConfig.url = await parseUrl(input, config, agent);
+
+            // We generate public URLs for any resources specified with the smythfs protocol in the request URL.
+            ({ url: reqConfig.url, publicUrls } = await parseSmythFsUrl(reqConfig.url, agent));
+
+            const { data, headers } = await parseData(input, config, agent);
+
+            // If the data is null, the request may fail. We encountered an issue where a request failed due to null data being provided.
+            let dataForDebug;
+            if (data) {
+                reqConfig.data = data;
+
+                dataForDebug = await formatDataForDebug(data);
+            }
+
+            reqConfig.headers = (await parseHeaders(input, config, agent)).concat({ ...headers });
+
+            const proxyConfig = await parseProxy(input, config, agent);
+
+            if (proxyConfig) {
+                if (proxyConfig instanceof SocksProxyAgent) {
+                    const isSecureEndpoint = reqConfig.url?.startsWith('https://');
+                    reqConfig[isSecureEndpoint ? 'httpsAgent' : 'httpAgent'] = proxyConfig;
+                } else {
+                    reqConfig.proxy = proxyConfig;
+                }
+            }
+
+            let Response: any = {};
+            let Headers: any = {};
+            let _error: any = undefined;
+            try {
+                if (config?.data?.oauthService && config?.data?.oauthService !== 'None') {
+                    const rootUrl = new URL(reqConfig.url).origin;
+                    const additionalParams = extractAdditionalParamsForOAuth1(reqConfig);
+                    const oauthHeaders = await generateOAuthHeaders(agent, config, reqConfig, logger, additionalParams, rootUrl);
+                    //reqConfig.headers = { ...reqConfig.headers, ...oauthHeaders };
+                    reqConfig.headers = reqConfig.headers.concat({ ...oauthHeaders });
+                }
+
+                // in order to handle binary data automatically, we need to set responseType to 'arraybuffer' for all requests, then parse the response data based on content-type
+                reqConfig.responseType = 'arraybuffer';
+
+                logger.debug('Making API call', { ...reqConfig, data: dataForDebug });
+
+                const response = await axios.request(reqConfig);
+
+                const parsedRes = await parseArrayBufferResponse(response, agent);
+
+                // log response headers
+                logger.debug('API call Response Headers', response.headers);
+                Response = parsedRes;
+
+                logger.debug('API call Response\n', Response);
+
+                Headers = Object.fromEntries(Object.entries(response.headers));
+            } catch (error) {
+                logger.debug(`Error making API call: ${error.message}`);
+                Headers = error?.response?.headers ? Object.fromEntries(Object.entries(error.response.headers)) : {};
+                Response = await parseArrayBufferResponse(error.response, agent);
+                _error = error.message;
+            }
+
+            return { Response, Headers, _error, _debug: logger.output };
+        } catch (error) {
+            return { _error: error.message, _debug: logger.output };
+        } finally {
+            if (publicUrls.length > 0) {
+                await destroyPublicUrls(publicUrls);
+            }
+        }
+    }
+}

package/src/Components/APICall/AccessTokenManager.ts

@@ -0,0 +1,130 @@
+// accessTokenManager.ts
+import { Agent } from '@sre/AgentManager/Agent.class';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { SystemEvents } from '@sre/Core/SystemEvents';
+import { Logger } from '@sre/helpers/Log.helper';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import axios from 'axios';
+
+const console = Logger('AccessTokenManager');
+let managedVault: any;
+
+SystemEvents.on('SRE:Booted', () => {
+    managedVault = ConnectorService.getManagedVaultConnector();
+});
+class AccessTokenManager {
+    private clientId: string;
+    private clientSecret: string;
+    private primaryToken: string; // accessToken || token
+    private secondaryToken: string; // refreshToken || tokenSecret
+    private tokenUrl: string; // tokenURL to refresh accessToken
+    private expires_in: any;
+    private data: any; // value of key(keyId) in teamSettings that needs to be updated if required
+    private keyId: any; // key of object  in teamSettings
+    private logger: any; // Use to log console in debugger
+    private agent: Agent;
+    constructor(
+        clientId: string,
+        clientSecret: string,
+        secondaryToken: string,
+        tokenUrl: string,
+        expires_in: any,
+        primaryToken: string,
+        data: any,
+        keyId: any,
+        logger: any,
+        agent: Agent,
+    ) {
+        this.clientId = clientId;
+        this.clientSecret = clientSecret;
+        this.primaryToken = primaryToken;
+        this.secondaryToken = secondaryToken;
+        this.tokenUrl = tokenUrl;
+        this.expires_in = expires_in;
+        this.data = data;
+        this.keyId = keyId;
+        this.logger = logger;
+        this.agent = agent;
+    }
+
+    async getAccessToken(): Promise<string> {
+        try {
+            const currentTime: any = new Date().getTime();
+
+            // If there's no secondaryToken (refresh token) and no expires_in,
+            // assume it's a long-lived token and return the primaryToken directly
+            if (!this.secondaryToken && !this.expires_in) {
+                console.log('Using long-lived access token');
+                this.logger.debug('Using long-lived access token. If authentication failes, please re-authenticate and try again');
+                return this.primaryToken;
+            }
+
+            // Regular token expiration check for tokens with expiration
+            // should be alway currentTime >= Number(this.expires_in)
+            if (!this.expires_in || currentTime >= Number(this.expires_in)) {
+                if (!this.secondaryToken) {
+                    this.logger.debug('Refresh token is missing. Please re authenticate');
+                    console.log('Refresh token is missing. Please re authenticate...');
+                    // Redirect the user to the OAuth authorization URL or initiate the reauthentication flow
+                    throw new Error('Reauthentication required');
+                }
+                this.logger.debug('Access token is expired or missing. Refreshing access token...');
+                console.log('Access token is expired or missing. Refreshing access token...');
+                return await this.refreshAccessToken();
+            } else {
+                console.log('Access token is still valid');
+                this.logger.debug('Access token is still valid.');
+                return this.primaryToken;
+            }
+        } catch (error) {
+            console.error('Error fetching access token:', error);
+            this.logger.debug('Error fetching access token');
+            throw error;
+        }
+    }
+
+    async refreshAccessToken(): Promise<string> {
+        try {
+            const response = await axios.post(
+                this.tokenUrl,
+                new URLSearchParams({
+                    client_id: this.clientId,
+                    client_secret: this.clientSecret,
+                    refresh_token: this.secondaryToken,
+                    grant_type: 'refresh_token',
+                }).toString(),
+                {
+                    headers: {
+                        'Content-Type': 'application/x-www-form-urlencoded',
+                    },
+                },
+            );
+
+            const newAccessToken: string = response?.data?.access_token;
+            console.log('Access token refreshed successfully.');
+            this.logger.debug('Access token refreshed successfully.');
+            const expiresInMilliseconds: number = response?.data?.expires_in ? response?.data?.expires_in * 1000 : response?.data?.expires_in;
+            const expirationTimestamp: number = expiresInMilliseconds ? new Date().getTime() + expiresInMilliseconds : expiresInMilliseconds;
+            this.data.primary = newAccessToken;
+            this.data.expires_in = expirationTimestamp ? expirationTimestamp?.toString() : expirationTimestamp;
+            //const oauthTeamSettings = new OauthTeamSettings();
+            //const save: any = await oauthTeamSettings.update({ keyId: this.keyId, data: this.data });
+
+            const save: any = await managedVault.user(AccessCandidate.agent(this.agent.id)).set(this.keyId, JSON.stringify(this.data));
+            if (save && save.status === 200) {
+                console.log('Access token value is updated successfully.');
+                this.logger.debug('Access token value is updated successfully.');
+            } else {
+                console.log('Warning: new access token value is not updated.');
+                this.logger.debug('Warning: new access token value is not updated.');
+            }
+            return newAccessToken;
+        } catch (error) {
+            console.error('Failed to refresh access token:', error);
+            this.logger.debug(`Failed to refresh access token: ${error}`);
+            throw new Error('Failed to refresh access token.');
+        }
+    }
+}
+
+export default AccessTokenManager;

package/src/Components/APICall/ArrayBufferResponse.helper.ts

@@ -0,0 +1,58 @@
+import { Agent } from '@sre/AgentManager/Agent.class';
+import { BinaryInput } from '@sre/helpers/BinaryInput.helper';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { isBinaryData, isBinaryMimeType } from '@sre/utils/data.utils';
+import { AxiosResponse } from 'axios';
+import mimeTypeCategories from './mimeTypeCategories';
+
+const contentHandlers = {
+    json: parseJson,
+    text: parseText,
+    binary: parseBinary,
+};
+
+function parseJson(data) {
+    return JSON.parse(Buffer.from(data).toString('utf8') || '{}');
+}
+
+function parseText(data) {
+    return Buffer.from(data).toString('utf8');
+}
+
+async function parseBinary(data, contentType, agentId) {
+    const binaryInput = BinaryInput.from(data, null, contentType);
+    const smythFile = await binaryInput.getJsonData(AccessCandidate.agent(agentId));
+
+    return smythFile;
+}
+
+export async function parseArrayBufferResponse(response: AxiosResponse, agent: Agent): Promise<any> {
+    if (!response?.data) {
+        return null;
+    }
+    const data = response.data;
+    const contentType = response.headers['content-type'];
+    const cleanContentType = contentType?.split(';')[0];
+
+    // Try to find an exact match first,
+    let handlerType = Object.keys(mimeTypeCategories).find((type) => mimeTypeCategories[type].includes(cleanContentType));
+
+    // If no exact match, try to find a match for the first part of the handlerTypes, some handlers are generic like text/ in that case we check if the handler is a substring of the contentType
+    if (!handlerType) {
+        handlerType = Object.keys(mimeTypeCategories).find((type) => mimeTypeCategories[type].some((prefix) => cleanContentType?.startsWith(prefix)));
+    }
+
+    const handler = contentHandlers[handlerType];
+
+    if (handler) {
+        return handler(data, contentType, agent.id);
+    }
+
+    // Fallback: if no content type matches then check if the data is binary
+    // If so then parse it as binary, otherwise parse it as text
+    if (isBinaryMimeType(contentType) || isBinaryData(data)) {
+        return parseBinary(data, contentType, agent.id);
+    } else {
+        return parseText(data);
+    }
+}