@memberjunction/ai-agents 3.0.0 → 3.1.1

package/dist/base-agent.js

@@ -52,6 +52,7 @@ class BaseAgent {
         this._parentStepCounts = [];
         this._allProgressSteps = [];
         this._subAgentRuns = [];
+        this._mediaOutputs = [];
         this._payloadManager = new PayloadManager_1.PayloadManager();
         this._effectiveActions = [];
         this._dynamicActionLimits = {};
@@ -82,6 +83,163 @@ class BaseAgent {
         }
         return aiengine_1.AIEngine.Instance.GetDefaultAgentConfigurationPreset(agentId);
     }
+    promoteMediaOutputs(mediaOutputs) {
+        if (mediaOutputs && mediaOutputs.length > 0) {
+            this._mediaOutputs.push(...mediaOutputs);
+            this.logStatus(`📎 Promoted ${mediaOutputs.length} media output(s) to agent results`, true);
+        }
+    }
+    get MediaOutputs() {
+        return [...this._mediaOutputs];
+    }
+    interceptLargeBinaryContent(actionParams, actionEntity) {
+        if (!actionParams || actionParams.length === 0) {
+            return actionParams;
+        }
+        const sanitizedParams = [];
+        for (const param of actionParams) {
+            if (param.Type !== 'Output' && param.Type !== 'Both') {
+                sanitizedParams.push(param);
+                continue;
+            }
+            const paramMetadata = actionEntity?.Params?.find(p => p.Name === param.Name);
+            const valueType = paramMetadata?.ValueType;
+            const isMediaOutputParam = valueType === 'MediaOutput';
+            if (isMediaOutputParam && Array.isArray(param.Value)) {
+                const mediaItems = param.Value;
+                const references = [];
+                let extractedCount = 0;
+                for (let i = 0; i < mediaItems.length; i++) {
+                    const media = mediaItems[i];
+                    if (media.data && media.data.length > BaseAgent.LARGE_BINARY_THRESHOLD) {
+                        const refId = `media-${Date.now().toString(36)}-${i}-${Math.random().toString(36).substring(2, 8)}`;
+                        this._mediaOutputs.push({
+                            ...media,
+                            refId,
+                            persist: false
+                        });
+                        references.push(`\${media:${refId}}`);
+                        extractedCount++;
+                    }
+                }
+                if (extractedCount > 0) {
+                    sanitizedParams.push({
+                        Name: param.Name,
+                        Type: param.Type,
+                        Value: {
+                            mediaReferences: references,
+                            count: mediaItems.length,
+                            note: `${extractedCount} media item(s) extracted. Use placeholder syntax in your response: <img src="${references[0]}" alt="description" />`
+                        }
+                    });
+                    this.logStatus(`📦 Extracted ${extractedCount} ${param.Name} item(s) to media references`, true);
+                    continue;
+                }
+            }
+            if (typeof param.Value === 'string' && param.Value.length > BaseAgent.LARGE_BINARY_THRESHOLD) {
+                const base64Pattern = /^[A-Za-z0-9+/]+=*$/;
+                if (isMediaOutputParam || base64Pattern.test(param.Value.substring(0, 1000))) {
+                    const refId = `data-${Date.now().toString(36)}-${Math.random().toString(36).substring(2, 8)}`;
+                    this._mediaOutputs.push({
+                        modality: 'Image',
+                        mimeType: 'application/octet-stream',
+                        data: param.Value,
+                        label: `Media data from ${param.Name}`,
+                        refId,
+                        persist: false
+                    });
+                    sanitizedParams.push({
+                        Name: param.Name,
+                        Type: param.Type,
+                        Value: `\${media:${refId}}`
+                    });
+                    this.logStatus(`📦 Extracted large binary content from '${param.Name}' to media reference`, true);
+                    continue;
+                }
+            }
+            sanitizedParams.push(param);
+        }
+        return sanitizedParams;
+    }
+    resolveMediaPlaceholdersInString(text) {
+        const hasRefIds = this._mediaOutputs.some(m => m.refId);
+        if (!text || !hasRefIds) {
+            return text;
+        }
+        const placeholderRegex = /\$\{media:([a-z0-9-]+)\}/g;
+        return text.replace(placeholderRegex, (match, refId) => {
+            const media = this._mediaOutputs.find(m => m.refId === refId);
+            if (media?.data) {
+                media.persist = true;
+                return `data:${media.mimeType};base64,${media.data}`;
+            }
+            this.logStatus(`⚠️ Media reference '${refId}' not found in registry`, true);
+            return match;
+        });
+    }
+    resolveMediaPlaceholdersInPayload(payload) {
+        const hasRefIds = this._mediaOutputs.some(m => m.refId);
+        if (!hasRefIds) {
+            return payload;
+        }
+        const unpersisted = this._mediaOutputs.filter(m => m.refId && m.persist === false).length;
+        const resolved = this.resolveMediaPlaceholdersRecursive(payload);
+        const persistedAfter = this._mediaOutputs.filter(m => m.refId && m.persist === true).length;
+        const resolvedCount = persistedAfter - (unpersisted - this._mediaOutputs.filter(m => m.refId && m.persist === false).length);
+        if (resolvedCount > 0) {
+            this.logStatus(`✅ Resolved ${resolvedCount} media placeholder(s) in final payload`, true);
+        }
+        return resolved;
+    }
+    resolveMediaPlaceholdersRecursive(value) {
+        if (value === null || value === undefined) {
+            return value;
+        }
+        if (typeof value === 'string') {
+            return this.resolveMediaPlaceholdersInString(value);
+        }
+        if (Array.isArray(value)) {
+            return value.map(item => this.resolveMediaPlaceholdersRecursive(item));
+        }
+        if (typeof value === 'object') {
+            const result = {};
+            for (const key of Object.keys(value)) {
+                result[key] = this.resolveMediaPlaceholdersRecursive(value[key]);
+            }
+            return result;
+        }
+        return value;
+    }
+    processMessageMediaPlaceholders(message) {
+        if (!message) {
+            return message;
+        }
+        const hasRefIds = this._mediaOutputs.some(m => m.refId);
+        if (!hasRefIds) {
+            return message;
+        }
+        const placeholderRegex = /\$\{media:([a-zA-Z0-9_-]+)\}/g;
+        let match;
+        let promotedCount = 0;
+        while ((match = placeholderRegex.exec(message)) !== null) {
+            const refId = match[1];
+            const media = this._mediaOutputs.find(m => m.refId === refId);
+            if (media && media.persist !== true) {
+                media.persist = true;
+                promotedCount++;
+            }
+        }
+        if (promotedCount > 0) {
+            this.logStatus(`📎 Auto-promoted ${promotedCount} media output(s) from message placeholders`, true);
+        }
+        let cleanedMessage = message
+            .replace(/<img[^>]*src=["']\$\{media:[^}]+\}["'][^>]*\/?>/gi, '')
+            .replace(/<audio[^>]*src=["']\$\{media:[^}]+\}["'][^>]*>.*?<\/audio>/gi, '')
+            .replace(/<video[^>]*src=["']\$\{media:[^}]+\}["'][^>]*>.*?<\/video>/gi, '')
+            .replace(/\n\s*\n\s*\n/g, '\n\n')
+            .trim();
+        return cleanedMessage;
+    }
     get ValidationRetryCount() {
         return this._validationRetryCount;
     }
@@ -241,6 +399,7 @@ class BaseAgent {
             this._contextRecoveryAttempts = 0;
             this._effectiveActions = [];
             this._dynamicActionLimits = {};
+            this._mediaOutputs = [];
             this._messageLifecycleCallback = params.onMessageLifecycle;
             await this.initializeEngines(params.contextUser);
             if (params.cancellationToken?.aborted) {
@@ -321,6 +480,9 @@ class BaseAgent {
             this.logStatus(`🔄 Executing step ${stepCount + 1} for agent '${params.agent.Name}'`, true, params);
             const nextStep = await this.executeNextStep(params, config, currentNextStep, stepCount);
             stepCount++;
+            if (nextStep.promoteMediaOutputs && nextStep.promoteMediaOutputs.length > 0) {
+                this.promoteMediaOutputs(nextStep.promoteMediaOutputs);
+            }
             if (nextStep.terminate) {
                 continueExecution = false;
                 this.logStatus(`🏁 Agent '${params.agent.Name}' terminating after ${stepCount} steps with result: ${nextStep.step}`, true, params);
@@ -2822,6 +2984,13 @@ The context is now within limits. Please retry your request with the recovered c
                 throw new Error('Cancelled during sub-agent execution');
             }
             this._subAgentRuns.push(subAgentResult);
+            if (subAgentResult.mediaOutputs?.length) {
+                this._mediaOutputs.push(...subAgentResult.mediaOutputs);
+                const refCount = subAgentResult.mediaOutputs.filter(m => m.refId).length;
+                if (refCount > 0) {
+                    this.logStatus(`📦 Collected ${refCount} media reference(s) from sub-agent '${subAgentRequest.name}'`, true);
+                }
+            }
             const shouldTerminate = subAgentRequest.terminateAfter;
             const outputData = {
                 subAgentResult: {
@@ -3006,6 +3175,13 @@ The context is now within limits. Please retry your request with the recovered c
                 (0, core_1.LogStatus)(`Related sub-agent '${subAgentRequest.name}' receiving context from paths: ${contextPaths.join(', ')}`);
             }
             const subAgentResult = await this.ExecuteSubAgent(params, subAgentRequest, subAgentEntity, stepEntity, initialSubAgentPayload, contextMessage, stepCount);
+            if (subAgentResult.mediaOutputs?.length) {
+                this._mediaOutputs.push(...subAgentResult.mediaOutputs);
+                const refCount = subAgentResult.mediaOutputs.filter(m => m.refId).length;
+                if (refCount > 0) {
+                    this.logStatus(`📦 Collected ${refCount} media reference(s) from related sub-agent '${subAgentRequest.name}'`, true);
+                }
+            }
             let mergedPayload = previousDecision.newPayload;
             let currentStepPayloadChangeResult = undefined;
             if (subAgentResult.success && relationship.SubAgentOutputMapping) {
@@ -3367,10 +3543,12 @@ The context is now within limits. Please retry your request with the recovered c
             }
             const actionSummaries = actionResults.map(result => {
                 const actionResult = result.success ? result.result : null;
+                const outputParams = result.result?.Params?.filter(p => p.Type === 'Both' || p.Type === 'Output') || [];
+                const sanitizedParams = this.interceptLargeBinaryContent(outputParams, result.actionEntity);
                 return {
                     actionName: result.action.name,
                     success: result.success,
-                    params: result.result?.Params.filter(p => p.Type === 'Both' || p.Type === 'Output'),
+                    params: sanitizedParams,
                     resultCode: actionResult?.Result?.ResultCode || (result.success ? 'SUCCESS' : 'ERROR'),
                     message: result.success ? actionResult?.Message || 'Action completed' : result.error
                 };
@@ -3914,6 +4092,13 @@ The context is now within limits. Please retry your request with the recovered c
         };
     }
     async finalizeAgentRun(finalStep, payload, contextUser) {
+        const isRootAgent = this._depth === 0;
+        const resolvedPayload = (payload && isRootAgent)
+            ? this.resolveMediaPlaceholdersInPayload(payload)
+            : payload;
+        const processedMessage = (finalStep.message && isRootAgent)
+            ? this.processMessageMediaPlaceholders(finalStep.message)
+            : finalStep.message;
         if (this._agentRun) {
             this._agentRun.CompletedAt = new Date();
             this._agentRun.Success = finalStep.step === 'Success' || finalStep.step === 'Chat';
@@ -3929,11 +4114,11 @@ The context is now within limits. Please retry your request with the recovered c
             else {
                 this._agentRun.Status = 'Completed';
             }
-            this._agentRun.Result = payload ? JSON.stringify(payload) : null;
+            this._agentRun.Result = resolvedPayload ? JSON.stringify(resolvedPayload) : null;
             this._agentRun.FinalStep = finalStep.step;
-            this._agentRun.Message = finalStep.message;
-            this._agentRun.FinalPayloadObject = payload;
-            this._agentRun.FinalPayload = payload ? JSON.stringify(payload) : null;
+            this._agentRun.Message = processedMessage;
+            this._agentRun.FinalPayloadObject = resolvedPayload;
+            this._agentRun.FinalPayload = resolvedPayload ? JSON.stringify(resolvedPayload) : null;
             const tokenStats = this.calculateTokenStats();
             this._agentRun.TotalTokensUsed = tokenStats.totalTokens;
             this._agentRun.TotalPromptTokensUsed = tokenStats.promptTokens;
@@ -3944,16 +4129,20 @@ The context is now within limits. Please retry your request with the recovered c
                 (0, core_1.LogError)(`Failed to finalize agent run ${this._agentRun.ID}`);
             }
         }
+        if (finalStep.promoteMediaOutputs && finalStep.promoteMediaOutputs.length > 0) {
+            this.promoteMediaOutputs(finalStep.promoteMediaOutputs);
+        }
         return {
             success: finalStep.step === 'Success' || finalStep.step === 'Chat',
-            payload,
+            payload: resolvedPayload,
             agentRun: this._agentRun,
             responseForm: finalStep.responseForm,
             actionableCommands: finalStep.actionableCommands,
             automaticCommands: finalStep.automaticCommands,
             memoryContext: this._injectedMemory.notes.length > 0 || this._injectedMemory.examples.length > 0
                 ? this._injectedMemory
-                : undefined
+                : undefined,
+            mediaOutputs: this._mediaOutputs.length > 0 ? this._mediaOutputs : undefined
         };
     }
     calculateTokenStats() {
@@ -4378,4 +4567,5 @@ The context is now within limits. Please retry your request with the recovered c
 }
 exports.BaseAgent = BaseAgent;
 BaseAgent.MAX_VALIDATION_RETRIES = 10;
+BaseAgent.LARGE_BINARY_THRESHOLD = 10000;
 //# sourceMappingURL=base-agent.js.map