@smythos/sre 1.7.20 → 1.7.41

package/dist/types/Components/AgentPlugin.class.d.ts

@@ -9,7 +9,7 @@ export declare class AgentPlugin extends Component {
         _debug: string;
         Response?: undefined;
     } | {
-        Response: string;
+        Response: any;
         _debug: string;
         _error?: undefined;
     }>;

package/dist/types/Components/DataSourceIndexer.class.d.ts

@@ -6,18 +6,10 @@ export declare class DataSourceIndexer extends Component {
     protected configSchema: Joi.ObjectSchema<any>;
     constructor();
     init(): void;
-    process(input: any, config: any, agent: Agent): Promise<{
-        _debug: string;
-        Success: {
-            result: any;
-            id: any;
-        };
-        _error?: undefined;
-    } | {
-        _debug: string;
-        _error: any;
-        Success?: undefined;
-    }>;
+    process(input: any, config: any, agent: Agent): Promise<any>;
+    private processV1;
+    private processV2;
+    private transformEmbedding;
     validateInput(input: any): Joi.ValidationResult<any>;
     private addDSFromText;
     static genDsId(providedId: string, teamId: string, namespaceId: string): string;

package/dist/types/Components/GenAILLM.class.d.ts

@@ -262,19 +262,19 @@ export declare class GenAILLM extends Component {
     process(input: any, config: any, agent: Agent): Promise<{
         Reply: any;
     } | {
-        Reply: any;
+        Output: any;
         _error: string;
         _debug: string;
-        Output?: undefined;
+        Reply?: undefined;
     } | {
-        Output: any;
+        Reply: any;
         _error: string;
         _debug: string;
-        Reply?: undefined;
+        Output?: undefined;
     } | {
         _error: any;
         _debug: string;
-        Reply?: undefined;
         Output?: undefined;
+        Reply?: undefined;
     }>;
 }

package/dist/types/Components/RAG/DataSourceCleaner.class.d.ts

@@ -6,13 +6,13 @@ export declare class DataSourceCleaner extends DataSourceComponent {
     constructor();
     init(): void;
     process(input: any, config: any, agent: Agent): Promise<{
-        _debug: string;
-        Success: boolean;
-        _error?: undefined;
-    } | {
         _debug: string;
         _error: any;
         Success?: undefined;
+    } | {
+        _debug: string;
+        Success: boolean;
+        _error?: undefined;
     }>;
     processV1(input: any, config: any, agent: Agent): Promise<{
         _debug: string;

package/dist/types/Components/RAG/DataSourceComponent.class.d.ts

@@ -21,7 +21,11 @@ export declare class DataSourceCompError extends Error {
 }
 export declare class DataSourceComponent extends Component {
     constructor();
-    resolveVectorDbConnector(namespace: string | NsRecord, teamId: string): Promise<VectorDBConnector>;
+    resolveVectorDbConnector(namespace: string | NsRecord, teamId: string): Promise<{
+        vecDbConnector: VectorDBConnector;
+        namespaceRecord: NsRecord;
+    }>;
+    private resolveNamespaceRecord;
     buildEmbeddingConfig(embedding: {
         dimensions: string;
         modelId: string;

package/dist/types/Components/index.d.ts

@@ -8,9 +8,9 @@ import { FSleep } from './FSleep.class';
 import { FHash } from './FHash.class';
 import { FEncDec } from './FEncDec.class';
 import { FTimestamp } from './FTimestamp.class';
-import { DataSourceLookup } from './DataSourceLookup.class';
-import { DataSourceIndexer } from './DataSourceIndexer.class';
-import { DataSourceCleaner } from './DataSourceCleaner.class';
+import { DataSourceLookup } from './RAG/DataSourceLookup.class';
+import { DataSourceIndexer } from './RAG/DataSourceIndexer.class';
+import { DataSourceCleaner } from './RAG/DataSourceCleaner.class';
 import { JSONFilter } from './JSONFilter.class';
 import { LogicAND } from './LogicAND.class';
 import { LogicOR } from './LogicOR.class';

package/dist/types/config.d.ts

@@ -3,6 +3,7 @@ declare const config: {
         LOG_LEVEL: string;
         LOG_FILTER: string;
         NODE_ENV: string;
+        ROLLOUT_RAG_V2: string;
     };
     agent: {
         ENDPOINT_PREFIX: string;

package/dist/types/helpers/Conversation.helper.d.ts

@@ -1,6 +1,6 @@
 import { LLMInference } from '@sre/LLMManager/LLM.inference';
 import { LLMContext } from '@sre/MemoryManager/LLMContext';
-import { ILLMContextStore, TLLMModel } from '@sre/types/LLM.types';
+import { IConversationSettings, TLLMModel } from '@sre/types/LLM.types';
 import EventEmitter from 'events';
 export declare class Conversation extends EventEmitter {
     private _model;
@@ -30,7 +30,15 @@ export declare class Conversation extends EventEmitter {
     agentData: any;
     private _id;
     get id(): string;
+    private _toolCallCount;
+    private _maxToolCallsPerSession;
+    private _disableToolsForNextCall;
     get context(): LLMContext;
+    get storeId(): string;
+    /**
+     * Headers to be added to all tool call requests
+     */
+    headers: Record<string, string>;
     private _lastError;
     private _spec;
     private _customToolsDeclarations;
@@ -41,18 +49,7 @@ export declare class Conversation extends EventEmitter {
     get model(): string | TLLMModel;
     private _llmInference;
     get llmInference(): LLMInference;
-    constructor(_model: string | TLLMModel, _specSource?: string | Record<string, any>, _settings?: {
-        maxContextSize?: number;
-        maxOutputTokens?: number;
-        systemPrompt?: string;
-        toolChoice?: string;
-        store?: ILLMContextStore;
-        experimentalCache?: boolean;
-        toolsStrategy?: (toolsConfig: any) => any;
-        agentId?: string;
-        agentVersion?: string;
-        baseUrl?: string;
-    });
+    constructor(_model: string | TLLMModel, _specSource?: string | Record<string, any>, _settings?: IConversationSettings);
     get ready(): any;
     prompt(message?: string | any, toolHeaders?: {}, concurrentToolCalls?: number, abortSignal?: AbortSignal): Promise<string>;
     streamPrompt(message?: string | any, toolHeaders?: {}, concurrentToolCalls?: number, abortSignal?: AbortSignal): Promise<string>;

package/dist/types/helpers/TemplateString.helper.d.ts

@@ -33,7 +33,7 @@ export declare class TemplateStringHelper {
      * unmatched placeholders will be left as is
      * Recursively resolves nested template variables until no more variables are found
      */
-    parse(data: Record<string, string>, regex?: TemplateStringMatch, maxDepth?: number): this;
+    parse(data: Record<string, unknown>, regex?: TemplateStringMatch, maxDepth?: number): this;
     /**
      * Parses a template string by replacing placeholders with values from the provided data object, keeping the original raw values intact. This is particularly important for BinaryInput instances, as they include buffer data.
      * unmatched placeholders will be left as is

package/dist/types/index.d.ts

@@ -9,9 +9,6 @@ export * from './Components/Await.class';
 export * from './Components/Classifier.class';
 export * from './Components/Component.class';
 export * from './Components/ComponentHost.class';
-export * from './Components/DataSourceCleaner.class';
-export * from './Components/DataSourceIndexer.class';
-export * from './Components/DataSourceLookup.class';
 export * from './Components/FEncDec.class';
 export * from './Components/FHash.class';
 export * from './Components/FileStore.class';
@@ -88,6 +85,10 @@ export * from './Components/APICall/parseHeaders';
 export * from './Components/APICall/parseProxy';
 export * from './Components/APICall/parseUrl';
 export * from './Components/Image/imageSettings.config';
+export * from './Components/RAG/DataSourceCleaner.class';
+export * from './Components/RAG/DataSourceComponent.class';
+export * from './Components/RAG/DataSourceIndexer.class';
+export * from './Components/RAG/DataSourceLookup.class';
 export * from './Components/Triggers/Gmail.trigger';
 export * from './Components/Triggers/JobScheduler.trigger';
 export * from './Components/Triggers/Trigger.class';

package/dist/types/subsystems/IO/VectorDB.service/VectorDBConnector.d.ts

@@ -22,6 +22,7 @@ export interface IVectorDBRequest {
 }
 export declare abstract class VectorDBConnector extends SecureConnector<IVectorDBRequest> {
     protected readonly USER_METADATA_KEY = "user_metadata";
+    protected readonly LEGACY_USER_METADATA_KEY = "metadata";
     abstract id: string;
     abstract getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL>;
     requester(candidate: AccessCandidate): IVectorDBRequest;

package/dist/types/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.d.ts

@@ -34,6 +34,7 @@ export declare class MilvusVectorDB extends VectorDBConnector {
     embedder: BaseEmbedding;
     private SCHEMA_DEFINITION;
     private INDEX_PARAMS;
+    private nkvConnector;
     constructor(_settings: MilvusConfig);
     protected createNamespace(acRequest: AccessRequest, namespace: string, metadata?: {
         [key: string]: any;

package/dist/types/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.d.ts

@@ -7,13 +7,20 @@ import { DatasourceDto, IStorageVectorDataSource, IVectorDataSourceDto, QueryOpt
 import { BaseEmbedding, TEmbeddings } from '../embed/BaseEmbedding';
 export type PineconeConfig = {
     /**
-     * The Pinecone API key
+     * The Pinecone API key [LEGACY]
      */
-    apiKey: string;
+    apiKey?: string;
     /**
-     * The Pinecone index name
+     * The Pinecone index name [LEGACY]
      */
-    indexName: string;
+    indexName?: string;
+    /**
+     * The Pinecone credentials [New unified format]
+     */
+    credentials?: {
+        apiKey: string;
+        indexName: string;
+    };
     /**
      * The embeddings model to use
      */

package/dist/types/subsystems/IO/VectorDB.service/embed/index.d.ts

@@ -20,5 +20,10 @@ export declare class EmbeddingsFactory {
     static create(provider?: SupportedProviders, config?: TEmbeddings & {
         model?: SupportedModels[SupportedProviders] | TLLMModel;
     }): OpenAIEmbeds | GoogleEmbeds;
+    static getProviderByModel(model: SupportedModels): SupportedProviders;
+    static getModels(): {
+        provider: SupportedProviders;
+        model: SupportedModels[SupportedProviders];
+    }[];
 }
 export {};

package/dist/types/subsystems/LLMManager/LLM.helper.d.ts

@@ -133,4 +133,23 @@ export declare class LLMHelper {
      * @returns {TLLMMessageBlock[]} The modified array of message objects.
      */
     static removeDuplicateUserMessages(messages: TLLMMessageBlock[]): TLLMMessageBlock[];
+    /**
+     * Checks if the given model is part of the Claude 4 family.
+     *
+     * @param {string} modelId - The model identifier to check.
+     * @returns {boolean} True if the model is Claude 4 family, false otherwise.
+     *
+     * @example
+     * const isClaude4 = LLMHelper.isClaude4Family('claude-sonnet-4-20250514');
+     * console.log(isClaude4); // true
+     *
+     * @example
+     * const isClaude4 = LLMHelper.isClaude4Family('claude-opus-4-5');
+     * console.log(isClaude4); // true
+     *
+     * @example
+     * const isClaude4 = LLMHelper.isClaude4Family('gpt-4-turbo');
+     * console.log(isClaude4); // false
+     */
+    static isClaude4Family(modelId: string): boolean;
 }

package/dist/types/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.d.ts

@@ -1,6 +1,6 @@
 import EventEmitter from 'events';
 import { type GenerateContentResponseUsageMetadata } from '@google/genai/node';
-import { TLLMMessageBlock, ToolData, TLLMToolResultMessageBlock, APIKeySource, ILLMRequestFuncParams, TLLMChatResponse, TGoogleAIRequestBody, ILLMRequestContext, TLLMPreparedParams } from '@sre/types/LLM.types';
+import { TLLMMessageBlock, ToolData, TLLMToolResultMessageBlock, APIKeySource, ILLMRequestFuncParams, TLLMChatResponse, TGoogleAIRequestBody, TLLMPreparedParams } from '@sre/types/LLM.types';
 import { LLMConnector } from '../LLMConnector';
 type UsageMetadataWithThoughtsToken = GenerateContentResponseUsageMetadata & {
     thoughtsTokenCount?: number;
@@ -26,6 +26,7 @@ export declare class GoogleAIConnector extends LLMConnector {
         sourceId: string;
         input_tokens: number;
         output_tokens: number;
+        output_tokens_image: number;
         input_tokens_audio: number;
         input_tokens_cache_read: number;
         input_tokens_cache_read_audio: number;
@@ -33,22 +34,26 @@ export declare class GoogleAIConnector extends LLMConnector {
         keySource: APIKeySource;
         agentId: string;
         teamId: string;
-        inTier: string;
-        outTier: string;
-        crTier: string;
+        tier: string;
     };
     /**
      * Extract text and image tokens from Google AI usage metadata
      */
     private extractTokenCounts;
-    protected reportImageUsage({ usage, context, numberOfImages, }: {
-        usage: {
-            cost?: number;
-            usageMetadata?: UsageMetadataWithThoughtsToken;
-        };
-        context: ILLMRequestContext;
+    protected reportImageCost({ cost, context, numberOfImages }: {
+        cost: any;
+        context: any;
         numberOfImages?: number;
     }): void;
+    /**
+     * Normalizes function response values to ensure they conform to Google AI's STRUCT requirement.
+     * Gemini expects functionResponse.response to be a STRUCT (JSON object format), not a list or scalar.
+     */
+    private normalizeFunctionResponse;
+    /**
+     * Parses and normalizes function response values, handling string JSON and various data types.
+     */
+    private parseFunctionResponse;
     formatToolsConfig({ toolDefinitions, toolChoice }: {
         toolDefinitions: any;
         toolChoice?: string;

package/dist/types/subsystems/LLMManager/ModelsProvider.service/connectors/JSONModelsProvider.class.d.ts

@@ -35,6 +35,41 @@ export declare class JSONModelsProvider extends ModelsProviderConnector {
     private getValidModels;
     private isValidSingleModel;
     private isValidModel;
+    /**
+     * Determines whether a file path should be ignored by the directory watcher.
+     *
+     * This method implements a sophisticated filtering strategy for dot-segment paths
+     * (paths containing directories that start with a dot, like .git, .env, .cache).
+     *
+     * **Filtering Strategy:**
+     * 1. Paths WITHOUT dot segments: Never ignored
+     * 2. Paths WITH dot segments:
+     *    - If SMYTH_PATH is not configured: All ignored
+     *    - If SMYTH_PATH is configured:
+     *      - Allow the watched directory even if SMYTH_PATH contains dot-segments
+     *        (e.g., /home/user/.smyth/models/OpenAI/default.json is allowed)
+     *      - Ignore dot-segments INSIDE the models directory
+     *        (e.g., /home/user/.smyth/models/.hidden/model.json is ignored)
+     *      - Paths outside watched directory: Ignored
+     *
+     * @param filePath - The file path to check
+     * @param watchedDir - The absolute path of the directory being watched (models folder)
+     * @param smythPath - The resolved SMYTH_PATH, or null if not configured
+     * @returns true if the path should be ignored, false if it should be watched
+     *
+     * @example
+     * ```typescript
+     * // Path without dot-segment (allowed)
+     * shouldIgnorePath('/models/OpenAI/default.json', '/models', '/home/.smyth') // => false
+     *
+     * // Dot-segment inside models directory (ignored)
+     * shouldIgnorePath('/models/.git/config', '/models', '/home/.smyth') // => true
+     *
+     * // Dot-segment in parent path only (allowed)
+     * shouldIgnorePath('/home/.smyth/models/OpenAI/default.json', '/home/.smyth/models', '/home/.smyth') // => false
+     * ```
+     */
+    private shouldIgnorePath;
     private initDirWatcher;
 }
 export {};

package/dist/types/subsystems/Security/Account.service/AccountConnector.d.ts

@@ -9,7 +9,7 @@ export interface ISmythAccountRequest {
     getCandidateTeam(): Promise<string | undefined>;
     getAllTeamSettings(): Promise<KeyValueObject>;
     getAllUserSettings(): Promise<KeyValueObject>;
-    getTeamSetting(settingKey: string): Promise<string>;
+    getTeamSetting(settingKey: string, group?: string): Promise<string>;
     getUserSetting(settingKey: string): Promise<string>;
     getAgentSetting(settingKey: string): Promise<string>;
     getTeam(): Promise<string>;
@@ -23,7 +23,7 @@ export declare abstract class AccountConnector extends Connector {
     abstract getCandidateTeam(candidate: IAccessCandidate): Promise<string | undefined>;
     abstract getAllTeamSettings(acRequest: AccessRequest, teamId: string): Promise<KeyValueObject>;
     abstract getAllUserSettings(acRequest: AccessRequest, accountId: string): Promise<KeyValueObject>;
-    abstract getTeamSetting(acRequest: AccessRequest, teamId: string, settingKey: string): Promise<string>;
+    abstract getTeamSetting(acRequest: AccessRequest, teamId: string, settingKey: string, group?: string): Promise<string>;
     abstract getUserSetting(acRequest: AccessRequest, accountId: string, settingKey: string): Promise<string>;
     abstract getAgentSetting(acRequest: AccessRequest, agentId: string, settingKey: string): Promise<string>;
 }

package/dist/types/subsystems/Security/Vault.service/connectors/SecretsManager.class.d.ts

@@ -14,12 +14,11 @@ export declare class SecretsManager extends VaultConnector {
     private secretsManager;
     private prefix;
     constructor(_settings: SecretsManagerConfig);
-    protected get(acRequest: AccessRequest, secretName: string): Promise<any>;
+    protected get(acRequest: AccessRequest, secretName: string): Promise<string>;
     protected exists(acRequest: AccessRequest, keyId: string): Promise<boolean>;
     protected listKeys(acRequest: AccessRequest): Promise<any[]>;
     getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL>;
-    private getSecretByName;
     private getVaultKey;
     private getSecretById;
-    private getSecretValue;
+    private extractSecretName;
 }

package/dist/types/types/LLM.types.d.ts

@@ -196,6 +196,7 @@ export type TLLMModel = {
     isCustomLLM?: boolean;
     isUserCustomLLM?: boolean;
     modelId?: string;
+    modelEntryName?: string;
     tokens?: number;
     completionTokens?: number;
     components?: string[];
@@ -394,10 +395,32 @@ export type TLLMInputMessage = {
     }[];
 };
 export interface ILLMContextStore {
+    id: string;
     save(messages: any[]): Promise<void>;
     load(count?: number): Promise<any[]>;
     getMessage(message_id: string): Promise<any[]>;
 }
+/**
+ * Configuration options for Conversation helper
+ */
+export interface IConversationSettings {
+    maxContextSize?: number;
+    maxOutputTokens?: number;
+    systemPrompt?: string;
+    toolChoice?: string;
+    store?: ILLMContextStore;
+    experimentalCache?: boolean;
+    toolsStrategy?: (toolsConfig: any) => any;
+    agentId?: string;
+    agentVersion?: string;
+    baseUrl?: string;
+    /**
+     * Maximum number of tool calls allowed in a single conversation session.
+     * Prevents infinite loops in tool calling scenarios.
+     * @default 100
+     */
+    maxToolCalls?: number;
+}
 export declare enum APIKeySource {
     Smyth = "smyth-managed",
     User = "user-managed",

package/dist/types/types/VectorDB.types.d.ts

@@ -42,6 +42,10 @@ export interface IStorageVectorDataSource {
     id: string;
     candidateId: string;
     candidateRole: string;
+    datasourceSizeMb?: number;
+    createdAt?: Date;
+    chunkSize?: number;
+    chunkOverlap?: number;
 }
 export interface IStorageVectorNamespace {
     namespace: string;

package/dist/types/utils/string.utils.d.ts

@@ -22,3 +22,4 @@ export declare const kebabToCapitalize: (input: any) => any;
  * @param input
  */
 export declare const identifyMimetypeFromString: (input: string) => "application/json" | "text/plain" | "application/xml" | "text/html" | "text/css" | "text/csv" | "text/markdown" | "" | "image/svg+xml" | "application/javascript" | "application/yaml" | "application/sql";
+export declare function calcSizeMb(text: string): number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smythos/sre",
-  "version": "1.7.20",
+  "version": "1.7.41",
   "description": "Smyth Runtime Environment",
   "author": "Alaa-eddine KADDOURI",
   "license": "MIT",
@@ -68,7 +68,7 @@
     "@pinecone-database/pinecone": "^3.0.0",
     "@runware/sdk-js": "^1.1.36",
     "@smithy/smithy-client": "^4.4.3",
-    "@zilliz/milvus2-sdk-node": "^2.6.2",
+    "@zilliz/milvus2-sdk-node": "^2.6.4",
     "acorn": "^8.14.1",
     "axios": "^1.7.2",
     "chokidar": "^4.0.3",
@@ -117,7 +117,7 @@
   "scripts": {
     "gen:barrel": "ctix build",
     "build:types": "tsc --emitDeclarationOnly --declaration --outDir dist/types -p tsconfig.dts.json",
-    "build:jsbundle": "cross-env rollup -c",
+    "build:jsbundle": "cross-env NODE_OPTIONS='--max-old-space-size=8192' rollup -c",
     "build": "pnpm run build:jsbundle && pnpm run build:types",
     "test:unit": "cd ../.. && vitest run packages/core/tests/unit",
     "test:unit:watch": "cd ../.. && vitest watch packages/core/tests/unit",

package/src/Components/APIEndpoint.class.ts

@@ -246,12 +246,7 @@ async function resolveTemplateVariables(data: any, input: any, agent: Agent): Pr
         if (isKeyTemplateVar(value as string)) {
             data[key] = await parseKey(value as string, agent.teamId);
         } else if (isTemplateVar(value as string)) {
-            // Parse using input values first, then agent variables.
-            // This correctly resolves cases where input values reference agent variables with the same name.
-            // Example: agent variables { user_id: "123" }, input { user_id: "{{user_id}}" }.
-            data[key] = TemplateString(value as string)
-                .parse(input)
-                .parse(agent.agentVariables).result;
+            data[key] = TemplateString(value as string).parse(input).result;
         }
     }
 

package/src/Components/AgentPlugin.class.ts

@@ -6,6 +6,8 @@ import { TemplateString } from '@sre/helpers/TemplateString.helper';
 import { IAgent as Agent } from '@sre/types/Agent.types';
 import { Conversation } from '@sre/helpers/Conversation.helper';
 import { Component } from './Component.class';
+import { LLMInference } from '@sre/LLMManager/LLM.inference';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
 
 export class AgentPlugin extends Component {
     protected configSchema = Joi.object({
@@ -84,16 +86,31 @@ export class AgentPlugin extends Component {
 
             const conv = new Conversation(model, subAgentId, { systemPrompt: descForModel, agentVersion: version });
 
-            const result = await conv.prompt(prompt, {
+            // # Region: enhance the prompt to produce a JSON output format based on the available component outputs.
+            const llmInference: LLMInference = await LLMInference.getInstance(model, AccessCandidate.agent(agent.id));
+
+            // if the llm is undefined, then it means we removed the model from our system
+            if (!llmInference.connector) {
+                return {
+                    _error: `The model '${model}' is not available. Please try a different one.`,
+                    _debug: logger.output,
+                };
+            }
+            const enhancedPrompt = llmInference.connector.enhancePrompt(prompt, config);
+            // # End Region: prompt enhancement
+
+            const result = await conv.prompt(enhancedPrompt, {
                 'X-AGENT-ID': subAgentId,
                 'X-AGENT-VERSION': version,
                 'X-REQUEST-TAG': reqTag, //request Tag identifies the request and tells the called agent that the call comes from internal agent
                 'x-caller-session-id': agent.callerSessionId,
             });
 
-            logger.debug(`Response:\n`, result, '\n');
+            const processedResponse = llmInference.connector.postProcess(result);
+
+            logger.debug(`Response:\n`, processedResponse, '\n');
 
-            return { Response: result, _debug: logger.output };
+            return { Response: processedResponse, _debug: logger.output };
         } catch (error: any) {
             console.error('Error on running Agent Component: ', error);
             return { _error: `Error on running Agent Component!\n${error?.message || JSON.stringify(error)}`, _debug: logger.output };