@smythos/sre 1.7.20 → 1.7.41

package/src/subsystems/Security/Vault.service/connectors/SecretsManager.class.ts

@@ -13,6 +13,7 @@ import {
     GetSecretValueCommandOutput,
 } from '@aws-sdk/client-secrets-manager';
 
+const defaultPrefix = 'smythos';
 const console = Logger('SecretsManager');
 
 export type SecretsManagerConfig = {
@@ -28,8 +29,7 @@ export class SecretsManager extends VaultConnector {
 
     constructor(protected _settings: SecretsManagerConfig) {
         super(_settings);
-        //if (!SmythRuntime.Instance) throw new Error('SRE not initialized');
-        this.prefix = _settings.prefix || '';
+        this.prefix = _settings.prefix || defaultPrefix;
         this.secretsManager = new SecretsManagerClient({
             region: _settings.region,
             ...(_settings.awsAccessKeyId && _settings.awsSecretAccessKey
@@ -46,11 +46,7 @@ export class SecretsManager extends VaultConnector {
         try {
             const accountConnector = ConnectorService.getAccountConnector();
             const teamId = await accountConnector.getCandidateTeam(acRequest.candidate);
-            // try fetchting by Id, if not found, try fetching by name
             let secret = await this.getSecretById(teamId, secretName);
-            if (!secret) {
-                secret = await this.getSecretByName(acRequest, secretName);
-            }
             return secret;
         } catch (error) {
             console.error(error);
@@ -73,51 +69,18 @@ export class SecretsManager extends VaultConnector {
 
         do {
             const listResponse: ListSecretsCommandOutput = await this.secretsManager.send(
-                new ListSecretsCommand({ NextToken: nextToken, Filters: [{ Key: 'tag-key', Values: ['smyth-vault'] }, { Key: 'name', Values: [this.getVaultKey(teamId, '')] }] })
+                new ListSecretsCommand({ NextToken: nextToken, Filters: [{ Key: 'name', Values: [this.getVaultKey(teamId, '')] }] })
             );
             if (listResponse.SecretList) {
                 for (const secret of listResponse.SecretList) {
                     if (secret.Name) {
-                        secrets.push({
-                            ARN: secret.ARN,
-                            Name: secret.Name,
-                            CreatedDate: secret.CreatedDate,
-                        });
+                        secrets.push(this.extractSecretName(secret.Name, teamId, this.prefix));
                     }
                 }
             }
             nextToken = listResponse.NextToken;
         } while (nextToken);
-
-      const $promises = [];
-        for (const secret of secrets) {
-            $promises.push(getSpecificSecret(secret, this.secretsManager));
-        }
-        const formattedSecrets = await Promise.all($promises);
-        return formattedSecrets;
-
-        async function getSpecificSecret(secret, secretsManager: SecretsManagerClient) {
-            const data: GetSecretValueCommandOutput = await secretsManager.send(new GetSecretValueCommand({ SecretId: secret.ARN }));
-            let secretString = data.SecretString;
-    
-            if (secretString) {
-              try {
-                let parsedSecret = JSON.parse(secretString);
-                const secretId = secret.Name?.split('/').pop();
-                const key = parsedSecret.key;
-                const value = parsedSecret.value;
-                const metadata = parsedSecret.metadata;
-                return {
-                  id: secretId,
-                  key,
-                  value,
-                  metadata,
-                };
-              } catch (error) {
-              }
-            }
-            return null;
-        }
+        return secrets;
     }
 
     public async getResourceACL(resourceId: string, candidate: IAccessCandidate) {
@@ -133,17 +96,6 @@ export class SecretsManager extends VaultConnector {
         return acl;
     }
 
-    private async getSecretByName(acRequest: AccessRequest, secretName: string) {
-        try {
-            const secrets = await this.listKeys(acRequest);
-            const secret = secrets.find((s) => s.key === secretName);
-            return secret?.value;
-        } catch (error) {
-            console.error(error);
-            return null;
-        }      
-    }
-
     private getVaultKey(teamId: string, secretName: string) {
         return `${this.prefix.length ? `${this.prefix}/` : ''}${teamId}/${secretName}`;
     }
@@ -151,20 +103,13 @@ export class SecretsManager extends VaultConnector {
     private async getSecretById(teamId: string, secretId: string) {
         try {
             const secret: GetSecretValueCommandOutput = await this.secretsManager.send(new GetSecretValueCommand({ SecretId: this.getVaultKey(teamId, secretId) }));
-            return this.getSecretValue(secret);
+            return secret.SecretString;
         } catch (error) {
-            console.error(error);
             return null;
         }
     }
 
-    private getSecretValue(secret: GetSecretValueCommandOutput) {
-        try {
-            const parsedSecret = typeof secret.SecretString === 'string' ? JSON.parse(secret.SecretString) : secret.SecretString;
-            return parsedSecret.value;
-        } catch (error) {
-            return null;
-        }
-    
+    private extractSecretName(secretKey: string, teamId: string, prefix: string) {
+        return secretKey.replace(`${prefix}/${teamId}/`, '');
     }
 }

package/src/types/LLM.types.ts

@@ -237,6 +237,7 @@ export type TLLMModel = {
     isCustomLLM?: boolean;
     isUserCustomLLM?: boolean;
     modelId?: string;
+    modelEntryName?: string;
     tokens?: number;
     completionTokens?: number;
     components?: string[];
@@ -446,11 +447,34 @@ export type TLLMInputMessage = {
 };
 
 export interface ILLMContextStore {
+    id: string;
     save(messages: any[]): Promise<void>;
     load(count?: number): Promise<any[]>;
     getMessage(message_id: string): Promise<any[]>;
 }
 
+/**
+ * Configuration options for Conversation helper
+ */
+export interface IConversationSettings {
+    maxContextSize?: number;
+    maxOutputTokens?: number;
+    systemPrompt?: string;
+    toolChoice?: string;
+    store?: ILLMContextStore;
+    experimentalCache?: boolean;
+    toolsStrategy?: (toolsConfig: any) => any;
+    agentId?: string;
+    agentVersion?: string;
+    baseUrl?: string;
+    /**
+     * Maximum number of tool calls allowed in a single conversation session.
+     * Prevents infinite loops in tool calling scenarios.
+     * @default 100
+     */
+    maxToolCalls?: number;
+}
+
 export enum APIKeySource {
     Smyth = 'smyth-managed',
     User = 'user-managed',

package/src/types/VectorDB.types.ts

@@ -64,6 +64,10 @@ export interface IStorageVectorDataSource {
     id: string;
     candidateId: string;
     candidateRole: string;
+    datasourceSizeMb?: number;
+    createdAt?: Date;
+    chunkSize?: number;
+    chunkOverlap?: number;
 }
 
 export interface IStorageVectorNamespace {

package/src/utils/array.utils.ts

@@ -0,0 +1,11 @@
+/**
+ * Utility method to chunk arrays into smaller batches
+ */
+export function chunkArr<T>(arr: T[], sizePerChunk: number): T[][] {
+    return arr.reduce((chunks, elem, index) => {
+        const chunkIndex = Math.floor(index / sizePerChunk);
+        const chunk = chunks[chunkIndex] || [];
+        chunks[chunkIndex] = chunk.concat([elem]);
+        return chunks;
+    }, [] as T[][]);
+}

package/src/utils/base64.utils.ts

@@ -110,7 +110,7 @@ export function isBase64(str: string): boolean {
     const cleanedBase64Data = cleanBase64(str);
 
     // Sometimes words like 'male' and hashes like md5, sha1, sha256, sha512 are detected as base64
-    if (cleanedBase64Data.length < 128) return false;
+    //if (cleanedBase64Data.length < 128) return false;
 
     try {
         const buffer = Buffer.from(cleanedBase64Data, 'base64');

package/src/utils/data.utils.ts

@@ -161,12 +161,12 @@ function isValidPathFormat(path: string): boolean {
     const windowsAbsolute = /^[a-zA-Z]:[\\\/]/; // C:\ or C:/
     const windowsUNC = /^\\\\[^\\]+\\[^\\]+/; // \\server\share
     const windowsRelative = /^\.{1,2}[\\\/]/; // .\ or ..\ or ./ or ../
-    
+
     // Unix path patterns
     const unixAbsolute = /^\//; // /path/to/file
     const unixHome = /^~[\/]/; // ~/path/to/file
     const unixRelative = /^\.{1,2}\//; // ./ or ../
-    
+
     // Relative paths without leading ./ or .\
     const genericRelative = /^[^\\\/]/; // path/to/file or path\to\file
 
@@ -259,12 +259,14 @@ export async function formatDataForDebug(data: any, candidate: IAccessCandidate)
     }
 
     try {
-        if (data.constructor?.name === 'BinaryInput') {
+        // We use .includes() instead of === because constructor.name can be 'BinaryInput$1', 'FormData$1', etc.
+        // This happens when the same class is loaded in different module contexts (bundling, HMR, circular deps)
+        if (data.constructor?.name.includes('BinaryInput')) {
             const jsonData = await data.getJsonData(candidate);
             dataForDebug = `[BinaryInput size=${jsonData?.size}]`;
         } else if (isBuffer(data)) {
             dataForDebug = `[Buffer size=${data.byteLength}]`;
-        } else if (data.constructor?.name === 'FormData') {
+        } else if (data.constructor?.name.includes('FormData')) {
             dataForDebug = `[FormData]`;
         } else if (isBase64(data) || isBase64DataUrl(data)) {
             dataForDebug = `[Base64 size=${getBase64FileSize(data)}]`;

package/src/utils/string.utils.ts

@@ -222,195 +222,6 @@ export const identifyMimetypeFromString = (input: string) => {
     return 'text/plain';
 };
 
-// export function chunkText(
-//     text: string,
-//     {
-//         chunkSize,
-//         chunkOverlap,
-//     }: {
-//         chunkSize: number;
-//         chunkOverlap: number;
-//     }
-// ): string[] {
-//     const textSplitter = new TextSplitter({
-//         chunkSize,
-//         chunkOverlap,
-//         separators: ['\n\n', '\n', ' ', ''],
-//         keepSeparator: true,
-//     });
-//     let output = textSplitter.splitText(text);
-
-//     return output;
-// }
-// class TextSplitter {
-//     private chunkSize: number;
-//     private chunkOverlap: number;
-//     private separators: string[] = ['\n\n', '\n', ' ', ''];
-//     private keepSeparator: boolean = true;
-
-//     constructor({
-//         chunkSize = 1000,
-//         chunkOverlap = 200,
-//         separators,
-//         keepSeparator,
-//     }: {
-//         chunkSize?: number;
-//         chunkOverlap?: number;
-//         separators?: string[];
-//         keepSeparator?: boolean;
-//     } = {}) {
-//         this.chunkSize = chunkSize;
-//         this.chunkOverlap = chunkOverlap;
-
-//         if (separators) {
-//             this.separators = separators;
-//         }
-
-//         if (keepSeparator !== undefined) {
-//             this.keepSeparator = keepSeparator;
-//         }
-
-//         if (this.chunkOverlap >= this.chunkSize) {
-//             throw new Error('Cannot have chunkOverlap >= chunkSize');
-//         }
-//     }
-
-//     public splitText(text: string): string[] {
-//         return this._splitText(text, this.separators);
-//     }
-
-//     private _splitText(text: string, separators: string[]): string[] {
-//         const finalChunks: string[] = [];
-
-//         // Get appropriate separator to use
-//         let separator: string = separators[separators.length - 1];
-//         let newSeparators: string[] | undefined;
-
-//         for (let i = 0; i < separators.length; i += 1) {
-//             const s = separators[i];
-//             if (s === '') {
-//                 separator = s;
-//                 break;
-//             }
-//             if (text.includes(s)) {
-//                 separator = s;
-//                 newSeparators = separators.slice(i + 1);
-//                 break;
-//             }
-//         }
-
-//         // Split the text using the identified separator
-//         const splits = this.splitOnSeparator(text, separator);
-
-//         // Process splits, recursively splitting longer texts
-//         let goodSplits: string[] = [];
-//         const _separator = this.keepSeparator ? '' : separator;
-
-//         for (const s of splits) {
-//             if (this.lengthFunction(s) < this.chunkSize) {
-//                 goodSplits.push(s);
-//             } else {
-//                 if (goodSplits.length) {
-//                     const mergedText = this.mergeSplits(goodSplits, _separator);
-//                     finalChunks.push(...mergedText);
-//                     goodSplits = [];
-//                 }
-
-//                 if (!newSeparators) {
-//                     finalChunks.push(s);
-//                 } else {
-//                     const otherInfo = this._splitText(s, newSeparators);
-//                     finalChunks.push(...otherInfo);
-//                 }
-//             }
-//         }
-
-//         if (goodSplits.length) {
-//             const mergedText = this.mergeSplits(goodSplits, _separator);
-//             finalChunks.push(...mergedText);
-//         }
-
-//         return finalChunks;
-//     }
-
-//     private splitOnSeparator(text: string, separator: string): string[] {
-//         let splits: string[];
-
-//         if (separator) {
-//             if (this.keepSeparator) {
-//                 const regexEscapedSeparator = separator.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&');
-//                 splits = text.split(new RegExp(`(?=${regexEscapedSeparator})`));
-//             } else {
-//                 splits = text.split(separator);
-//             }
-//         } else {
-//             splits = text.split('');
-//         }
-
-//         return splits.filter((s) => s !== '');
-//     }
-
-//     private lengthFunction(text: string): number {
-//         return text.length;
-//     }
-
-//     private joinDocs(docs: string[], separator: string): string | null {
-//         const text = docs.join(separator).trim();
-//         return text === '' ? null : text;
-//     }
-
-//     private mergeSplits(splits: string[], separator: string): string[] {
-//         const docs: string[] = [];
-//         const currentDoc: string[] = [];
-//         let total = 0;
-
-//         for (const d of splits) {
-//             const _len = this.lengthFunction(d);
-
-//             if (total + _len + currentDoc.length * separator.length > this.chunkSize) {
-//                 if (total > this.chunkSize) {
-//                     console.warn(`Created a chunk of size ${total}, which is longer than the specified ${this.chunkSize}`);
-//                 }
-
-//                 if (currentDoc.length > 0) {
-//                     const doc = this.joinDocs(currentDoc, separator);
-//                     if (doc !== null) {
-//                         docs.push(doc);
-//                     }
-
-//                     // Keep popping if conditions are met
-//                     while (total > this.chunkOverlap || (total + _len + currentDoc.length * separator.length > this.chunkSize && total > 0)) {
-//                         total -= this.lengthFunction(currentDoc[0]);
-//                         currentDoc.shift();
-//                     }
-//                 }
-//             }
-
-//             currentDoc.push(d);
-//             total += _len;
-//         }
-
-//         const doc = this.joinDocs(currentDoc, separator);
-//         if (doc !== null) {
-//             docs.push(doc);
-//         }
-
-//         return docs;
-//     }
-// }
-
-// class RecursiveTextSplitter extends TextSplitter {
-//     constructor({
-//         chunkSize = 1000,
-//         chunkOverlap = 200,
-//         separators = ['\n\n', '\n', ' ', ''],
-//         keepSeparator = true,
-//     }: {
-//         chunkSize?: number;
-//         chunkOverlap?: number;
-//         separators?: string[];
-//         keepSeparator?: boolean;
-//     } = {}) {
-//         super({ chunkSize, chunkOverlap, separators, keepSeparator });
-//     }
-// }
+export function calcSizeMb(text: string): number {
+    return Buffer.byteLength(text, 'utf-8') / 1024 / 1024;
+}

package/src/Components/DataSourceCleaner.class.ts

@@ -1,92 +0,0 @@
-import { IAgent as Agent } from '@sre/types/Agent.types';
-import { Component } from './Component.class';
-import { TemplateString } from '@sre/helpers/TemplateString.helper';
-import Joi from 'joi';
-import { validateCharacterSet } from '../utils';
-import { ConnectorService } from '@sre/Core/ConnectorsService';
-import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
-
-import { DataSourceIndexer } from './DataSourceIndexer.class';
-
-export class DataSourceCleaner extends Component {
-    protected configSchema = Joi.object({
-        namespaceId: Joi.string().max(50).allow('').label('namespace'),
-        id: Joi.string().custom(validateCharacterSet, 'custom validation characterSet').allow('').label('source identifier'),
-    });
-    constructor() {
-        super();
-    }
-    init() {}
-    async process(input, config, agent: Agent) {
-        await super.process(input, config, agent);
-        const teamId = agent.teamId;
-        const agentId = agent.id;
-        let debugOutput = agent.agentRuntime?.debug ? '== Source Indexer Log ==\n' : null;
-
-        try {
-            const configSchema = this.validateConfigData(config.data);
-            if (configSchema.error) {
-                throw new Error(`Config data validation error: ${configSchema.error}\n EXITING...`);
-            }
-
-            const outputs = {};
-            for (let con of config.outputs) {
-                if (con.default) continue;
-                outputs[con.name] = con?.description ? `<${con?.description}>` : '';
-            }
-
-            const inputSchema = this.validateInput(input);
-            if (inputSchema.error) {
-                throw new Error(`Input validation error: ${inputSchema.error}\n EXITING...`);
-            }
-
-            const namespaceId = configSchema.value.namespaceId.split('_')?.slice(1).join('_') || configSchema.value.namespaceId;
-
-            let vectorDbConnector = ConnectorService.getVectorDBConnector();
-
-            let existingnamespace = await vectorDbConnector.requester(AccessCandidate.team(teamId)).namespaceExists(namespaceId);
-            if (!existingnamespace) {
-                throw new Error(`Namespace ${namespaceId} does not exist`);
-            }
-
-            const providedId = TemplateString(config.data.id).parse(input).result;
-            const idRegex = /^[a-zA-Z0-9\-\_\.]+$/;
-            if (!idRegex.test(providedId)) {
-                throw new Error(`Invalid id. Accepted characters: 'a-z', 'A-Z', '0-9', '-', '_', '.'`);
-            }
-            debugOutput += `Searching for data source with id: ${providedId}\n`;
-
-            const dsId = DataSourceIndexer.genDsId(providedId, teamId, namespaceId);
-
-            await vectorDbConnector.requester(AccessCandidate.team(teamId)).deleteDatasource(namespaceId, dsId);
-
-            debugOutput += `Deleted data source with id: ${providedId}\n`;
-
-            return {
-                _debug: debugOutput,
-                Success: true,
-                // _error,
-            };
-        } catch (err: any) {
-            debugOutput += `Failed to delete data source: \n Error: ${err?.message}\n`;
-
-            return {
-                _debug: debugOutput,
-                _error: err?.message || "Couldn't delete data source",
-            };
-        }
-    }
-
-    validateInput(input: any) {
-        return Joi.object({}).unknown(true).validate(input);
-    }
-
-    validateConfigData(data: any) {
-        return Joi.object({
-            namespaceId: Joi.string().required(),
-            id: Joi.string().optional().allow('').allow(null),
-        })
-            .unknown(true)
-            .validate(data);
-    }
-}