@memberjunction/ai-agents 2.113.2 → 2.115.0

package/dist/base-agent.js

@@ -477,20 +477,23 @@ class BaseAgent {
                 errorMessage: `Agent type not found for ID: ${agent.TypeID}`
             };
         }
+        const agentTypeInstance = await base_agent_type_1.BaseAgentType.GetAgentTypeInstance(agentType);
+        const requiresAgentLevelPrompts = agentTypeInstance.RequiresAgentLevelPrompts;
         const systemPrompt = engine.Prompts.find(p => p.ID === agentType.SystemPromptID);
-        if (!systemPrompt)
+        if (!systemPrompt) {
             metadataOptional = true;
+        }
         const agentPrompt = engine.AgentPrompts
             .filter(ap => ap.AgentID === agent.ID && ap.Status === 'Active')
             .sort((a, b) => a.ExecutionOrder - b.ExecutionOrder)[0];
-        if (!agentPrompt && !metadataOptional) {
+        if (!agentPrompt && !metadataOptional && requiresAgentLevelPrompts) {
             return {
                 success: false,
                 errorMessage: `No prompts configured for agent: ${agent.Name}`
             };
         }
-        const childPrompt = engine.Prompts.find(p => p.ID === agentPrompt?.PromptID);
-        if (!childPrompt && !metadataOptional) {
+        const childPrompt = agentPrompt ? engine.Prompts.find(p => p.ID === agentPrompt.PromptID) : undefined;
+        if (!childPrompt && !metadataOptional && requiresAgentLevelPrompts) {
             return {
                 success: false,
                 errorMessage: `Child prompt not found for ID: ${agentPrompt?.PromptID}`
@@ -537,7 +540,7 @@ class BaseAgent {
             this.logStatus(`🎯 Using agent default effort level: ${params.agent.DefaultPromptEffortLevel}`, true, params);
         }
         else {
-            if (childPrompt.EffortLevel !== undefined && childPrompt.EffortLevel !== null) {
+            if (childPrompt && childPrompt.EffortLevel !== undefined && childPrompt.EffortLevel !== null) {
                 promptParams.effortLevel = childPrompt.EffortLevel;
                 this.logStatus(`🎯 Using child prompt effort level: ${childPrompt.EffortLevel}`, true, params);
             }
@@ -969,6 +972,17 @@ class BaseAgent {
     }
     async hasExceededAgentRunGuardrails(params, agentRun) {
         const agent = params.agent;
+        const DEFAULT_ABSOLUTE_MAX_ITERATIONS = 5000;
+        const absoluteMaxIterations = params.absoluteMaxIterations ?? DEFAULT_ABSOLUTE_MAX_ITERATIONS;
+        if (agentRun.TotalPromptIterations && agentRun.TotalPromptIterations >= absoluteMaxIterations) {
+            return {
+                exceeded: true,
+                type: 'iterations',
+                limit: absoluteMaxIterations,
+                current: agentRun.TotalPromptIterations,
+                reason: `Absolute maximum iteration safety limit of ${absoluteMaxIterations} exceeded. Current iterations: ${agentRun.TotalPromptIterations}. This is a system-wide safety measure to prevent infinite loops.`
+            };
+        }
         if (agent.MaxCostPerRun && agentRun.TotalCost) {
             if (agentRun.TotalCost >= agent.MaxCostPerRun) {
                 return {
@@ -1026,6 +1040,12 @@ class BaseAgent {
         return { exceeded: false };
     }
     isFatalPromptError(promptResult) {
+        if (promptResult?.errorMessage) {
+            const templateErrorPattern = /Failed to render/i;
+            if (templateErrorPattern.test(promptResult.errorMessage)) {
+                return true;
+            }
+        }
         if (!promptResult?.chatResult?.errorInfo) {
             return false;
         }
@@ -1040,6 +1060,128 @@ class BaseAgent {
         ];
         return fatalErrorTypes.includes(errorInfo.errorType);
     }
+    isConfigurationError(errorMessage, config) {
+        const configErrorPatterns = [
+            {
+                pattern: /cannot read propert(y|ies) of (undefined|null)/i,
+                getMessage: () => {
+                    const propertyMatch = errorMessage.match(/reading '(\w+)'/i);
+                    const property = propertyMatch ? propertyMatch[1] : 'unknown property';
+                    let details = `Attempted to access property '${property}' on an undefined or null object.`;
+                    if (property.toLowerCase().includes('prompt')) {
+                        details += `\n\n🔧 Configuration Issue: Missing prompt configuration.`;
+                        if (config) {
+                            if (!config.childPrompt) {
+                                details += `\n   - Agent does not have a child prompt configured`;
+                                if (this.AgentTypeInstance) {
+                                    const guidance = this.AgentTypeInstance.GetPromptConfigurationGuidance();
+                                    details += `\n${guidance}`;
+                                }
+                            }
+                            if (!config.systemPrompt) {
+                                details += `\n   - Agent type does not have a system prompt configured`;
+                            }
+                        }
+                    }
+                    else if (property.toLowerCase().includes('agent')) {
+                        details += `\n\n🔧 Configuration Issue: Missing agent configuration.`;
+                        details += `\n   - Ensure agent type is properly configured`;
+                        details += `\n   - Check that agent metadata is complete`;
+                    }
+                    return details;
+                }
+            },
+            {
+                pattern: /childprompt is (undefined|null|not defined)/i,
+                getMessage: () => {
+                    let details = `Agent is missing required child prompt configuration.\n\n` +
+                        `🔧 Configuration Fix:\n`;
+                    if (this.AgentTypeInstance) {
+                        const guidance = this.AgentTypeInstance.GetPromptConfigurationGuidance();
+                        details += `${guidance}`;
+                    }
+                    else {
+                        details += `   - Ensure agent has AI Agent Prompts relationship configured\n` +
+                            `   - Verify that prompt exists in AI Prompts table and is active`;
+                    }
+                    return details;
+                }
+            },
+            {
+                pattern: /systemprompt is (undefined|null|not defined)/i,
+                getMessage: () => {
+                    return `Agent type is missing system prompt configuration.\n\n` +
+                        `🔧 Configuration Fix:\n` +
+                        `   - Check agent type's SystemPromptID in AI Agent Types table\n` +
+                        `   - Verify system prompt exists and is active\n` +
+                        `   - Some agent types (like Flow) may not require system prompts`;
+                }
+            },
+            {
+                pattern: /no prompts configured/i,
+                getMessage: () => {
+                    return `Agent has no prompts configured.\n\n` +
+                        `🔧 Configuration Fix:\n` +
+                        `   - Add at least one AI Agent Prompt relationship to this agent\n` +
+                        `   - Or for Flow agents: Add Prompt-type steps to the agent's step graph`;
+                }
+            },
+            {
+                pattern: /agent type not found/i,
+                getMessage: () => {
+                    return `Agent references an agent type that doesn't exist.\n\n` +
+                        `🔧 Configuration Fix:\n` +
+                        `   - Verify TypeID in AI Agents table matches existing AI Agent Types record\n` +
+                        `   - Check that AIEngine.Instance.AgentTypes includes the required type\n` +
+                        `   - Ensure Config() has been called to load agent types`;
+                }
+            },
+            {
+                pattern: /child prompt not found/i,
+                getMessage: () => {
+                    return `Referenced child prompt doesn't exist or isn't loaded.\n\n` +
+                        `🔧 Configuration Fix:\n` +
+                        `   - Check AI Agent Prompts relationship has valid PromptID\n` +
+                        `   - Verify prompt exists in AI Prompts table\n` +
+                        `   - Ensure AIEngine.Instance.Prompts includes the prompt`;
+                }
+            },
+            {
+                pattern: /agent configuration/i,
+                getMessage: () => {
+                    return `General agent configuration error.\n\n` +
+                        `🔧 Configuration Check:\n` +
+                        `   - Review agent metadata in AI Agents table\n` +
+                        `   - Check all foreign key relationships are valid\n` +
+                        `   - Verify agent type is properly configured`;
+                }
+            },
+            {
+                pattern: /Failed to render.*child prompt templates/i,
+                getMessage: () => {
+                    return `Template rendering error: ${errorMessage}\n\n` +
+                        `🔧 Template Configuration Fix:\n` +
+                        `   - Check nunjucks template syntax in prompt content\n` +
+                        `   - Use {% raw %}{{}}{% endraw %} for literal braces in examples\n` +
+                        `   - Verify all template variables are properly defined\n` +
+                        `   - Ensure template variables referenced in content exist in context`;
+                }
+            }
+        ];
+        for (const { pattern, getMessage } of configErrorPatterns) {
+            if (pattern.test(errorMessage)) {
+                const detailedMessage = getMessage();
+                return {
+                    isConfigError: true,
+                    detailedMessage: `Configuration Error: ${detailedMessage}\n\nOriginal Error: ${errorMessage}`
+                };
+            }
+        }
+        return {
+            isConfigError: false,
+            detailedMessage: errorMessage
+        };
+    }
     contentToString(content) {
         if (typeof content === 'string') {
             return content;
@@ -1522,10 +1664,12 @@ The context is now within limits. Please retry your request with the recovered c
         if (contextMessage) {
             messages.push(contextMessage);
         }
-        messages.push({
-            role: 'user',
-            content: subAgentRequest.message
-        });
+        if (subAgentRequest.message) {
+            messages.push({
+                role: 'user',
+                content: subAgentRequest.message
+            });
+        }
         return messages;
     }
     async ExecuteSubAgent(params, subAgentRequest, subAgent, stepEntity, payload, contextMessage) {
@@ -1554,8 +1698,8 @@ The context is now within limits. Please retry your request with the recovered c
                 configurationId: params.configurationId,
                 effortLevel: params.effortLevel,
                 data: {
-                    ...subAgentRequest.templateParameters,
                     ...params.data,
+                    ...subAgentRequest.templateParameters,
                 },
                 context: params.context,
                 verbose: params.verbose,
@@ -1850,57 +1994,28 @@ The context is now within limits. Please retry your request with the recovered c
         };
     }
     async finalizeStepEntity(stepEntity, success, errorMessage, outputData) {
-        stepEntity.Status = success ? 'Completed' : 'Failed';
-        stepEntity.CompletedAt = new Date();
-        stepEntity.Success = success;
-        stepEntity.ErrorMessage = errorMessage || null;
-        if (outputData) {
-            stepEntity.OutputData = JSON.stringify({
-                ...outputData,
-                context: {
-                    success,
-                    durationMs: stepEntity.CompletedAt.getTime() - stepEntity.StartedAt.getTime(),
-                    errorMessage
-                }
-            });
-        }
-        if (!await stepEntity.Save()) {
-            console.error('Failed to update agent run step record');
-        }
-    }
-    getValueFromPath(obj, path) {
-        const parts = path.split('.');
-        let current = obj;
-        for (const part of parts) {
-            if (!part)
-                continue;
-            const arrayMatch = part.match(/^([^[]+)\[(\d+)\]$/);
-            if (arrayMatch) {
-                const arrayName = arrayMatch[1];
-                const index = parseInt(arrayMatch[2], 10);
-                if (current && typeof current === 'object' && arrayName in current) {
-                    current = current[arrayName];
-                    if (Array.isArray(current) && index >= 0 && index < current.length) {
-                        current = current[index];
-                    }
-                    else {
-                        return undefined;
+        try {
+            stepEntity.Status = success ? 'Completed' : 'Failed';
+            stepEntity.CompletedAt = new Date();
+            stepEntity.Success = success;
+            stepEntity.ErrorMessage = errorMessage || null;
+            if (outputData) {
+                stepEntity.OutputData = JSON.stringify({
+                    ...(0, global_1.CopyScalarsAndArrays)(outputData, true),
+                    context: {
+                        success,
+                        durationMs: stepEntity.CompletedAt.getTime() - stepEntity.StartedAt.getTime(),
+                        errorMessage
                     }
-                }
-                else {
-                    return undefined;
-                }
+                });
             }
-            else {
-                if (current && typeof current === 'object' && part in current) {
-                    current = current[part];
-                }
-                else {
-                    return undefined;
-                }
+            if (!await stepEntity.Save()) {
+                console.error('Failed to update agent run step record');
             }
         }
-        return current;
+        catch (e) {
+            console.error('Failed to update agent run step record', e);
+        }
     }
     resolveLoopParams(params, item, index, payload) {
         const resolved = {};
@@ -1982,7 +2097,28 @@ The context is now within limits. Please retry your request with the recovered c
                     };
                 }
                 else {
-                    return await this.executePromptStep(params, config, previousDecision);
+                    const fallbackStep = await this.AgentTypeInstance.HandleStepFallback(previousDecision, config, params, previousDecision.newPayload || previousDecision.previousPayload, this.AgentTypeState);
+                    if (fallbackStep) {
+                        if (fallbackStep.terminate) {
+                            const terminationMessage = `Agent completed successfully. Agent type '${config.agentType.Name}' terminated execution.`;
+                            const stepEntity = await this.createStepEntity({
+                                stepType: 'Decision',
+                                stepName: terminationMessage,
+                                contextUser: params.contextUser,
+                                payloadAtStart: fallbackStep.previousPayload,
+                                payloadAtEnd: fallbackStep.newPayload
+                            });
+                            await this.finalizeStepEntity(stepEntity, true, undefined, {
+                                reason: 'Agent type intervention - Success',
+                                agentType: config.agentType.Name,
+                                decision: 'Success'
+                            });
+                        }
+                        return fallbackStep;
+                    }
+                    else {
+                        return await this.executePromptStep(params, config, previousDecision);
+                    }
                 }
             case 'Failed':
                 if (previousDecision.terminate) {
@@ -1994,7 +2130,41 @@ The context is now within limits. Please retry your request with the recovered c
                     };
                 }
                 else {
-                    return await this.executePromptStep(params, config, previousDecision);
+                    const fallbackStep = await this.AgentTypeInstance.HandleStepFallback(previousDecision, config, params, previousDecision.newPayload || previousDecision.previousPayload, this.AgentTypeState);
+                    if (fallbackStep) {
+                        if (fallbackStep.terminate) {
+                            const errorDetails = fallbackStep.errorMessage || 'No error details provided';
+                            const terminationMessage = `Agent failed. Agent type '${config.agentType.Name}' terminated execution: ${errorDetails}`;
+                            const stepEntity = await this.createStepEntity({
+                                stepType: 'Decision',
+                                stepName: terminationMessage,
+                                contextUser: params.contextUser,
+                                payloadAtStart: fallbackStep.previousPayload,
+                                payloadAtEnd: fallbackStep.newPayload
+                            });
+                            await this.finalizeStepEntity(stepEntity, false, errorDetails, {
+                                reason: 'Agent type intervention - Failed',
+                                agentType: config.agentType.Name,
+                                decision: 'Failed'
+                            });
+                            if (this.AgentRun) {
+                                this.AgentRun.ErrorMessage = terminationMessage;
+                                await this.AgentRun.Save();
+                            }
+                            this.logError(`Agent type '${config.agentType.Name}' handling Failed step with termination`, {
+                                agent: params.agent,
+                                category: 'AgentExecution',
+                                metadata: {
+                                    errorDetails,
+                                    agentType: config.agentType.Name
+                                }
+                            });
+                        }
+                        return fallbackStep;
+                    }
+                    else {
+                        return await this.executePromptStep(params, config, previousDecision);
+                    }
                 }
             case 'ForEach':
                 return await this.executeForEachLoop(params, config, previousDecision);
@@ -2004,6 +2174,9 @@ The context is now within limits. Please retry your request with the recovered c
                 throw new Error(`Unsupported next step: ${previousDecision.step}`);
         }
     }
+    getDefaultPayloadSelfWritePaths() {
+        return undefined;
+    }
     async executePromptStep(params, config, previousDecision) {
         const promptToUse = await this.AgentTypeInstance.GetPromptForStep(params, config, previousDecision?.newPayload || previousDecision?.previousPayload, this.AgentTypeState, previousDecision);
         const promptId = promptToUse?.ID;
@@ -2116,7 +2289,7 @@ The context is now within limits. Please retry your request with the recovered c
             if (initialNextStep.payloadChangeRequest) {
                 const allowedPaths = params.agent.PayloadSelfWritePaths
                     ? JSON.parse(params.agent.PayloadSelfWritePaths)
-                    : undefined;
+                    : this.getDefaultPayloadSelfWritePaths();
                 const changeResult = this._payloadManager.applyAgentChangeRequest(payload, initialNextStep.payloadChangeRequest, {
                     validateChanges: true,
                     logChanges: true,
@@ -2165,24 +2338,107 @@ The context is now within limits. Please retry your request with the recovered c
         catch (error) {
             const payload = stepEntity.PayloadAtEnd ? JSON.parse(stepEntity.PayloadAtEnd) : previousDecision?.newPayload || params.payload;
             stepEntity.PayloadAtEnd = payload ? JSON.stringify(payload) : null;
-            await this.finalizeStepEntity(stepEntity, false, error.message);
             const errString = error?.message || error || 'Unknown error';
+            const configCheck = this.isConfigurationError(errString, config);
+            const isConfigurationError = configCheck.isConfigError;
+            const detailedErrorMessage = configCheck.detailedMessage;
+            const guardailCheck = await this.hasExceededAgentRunGuardrails(params, this.AgentRun);
+            const guardrailsExceeded = guardailCheck && guardailCheck.exceeded;
+            const shouldTerminate = isConfigurationError || guardrailsExceeded;
+            let finalErrorMessage;
+            if (isConfigurationError) {
+                finalErrorMessage = detailedErrorMessage;
+                this.logError(`❌ Configuration error in prompt execution (will terminate):\n${detailedErrorMessage}`, {
+                    agent: params.agent,
+                    category: 'AgentConfiguration',
+                    metadata: {
+                        error: (0, global_1.CopyScalarsAndArrays)(error, true)
+                    }
+                });
+            }
+            else if (guardrailsExceeded) {
+                finalErrorMessage = `Guardrails exceeded: ${guardailCheck.reason}\n\nOriginal error: ${errString}`;
+                this.logError(`⛔ Guardrails exceeded during prompt execution: ${guardailCheck.reason}`, {
+                    agent: params.agent,
+                    category: 'Guardrails',
+                    metadata: {
+                        guardrailType: guardailCheck.type,
+                        limit: guardailCheck.limit,
+                        current: guardailCheck.current,
+                        originalError: errString
+                    }
+                });
+            }
+            else {
+                finalErrorMessage = `Prompt execution failed: ${errString}`;
+                this.logError(`⚠️ Prompt execution error (will retry if guardrails allow): ${errString}`, {
+                    agent: params.agent,
+                    category: 'PromptExecution',
+                    metadata: {
+                        attemptNumber: this._agentRun?.TotalPromptIterations || 0,
+                        retryable: true,
+                        error: (0, global_1.CopyScalarsAndArrays)(error, true)
+                    }
+                });
+            }
+            await this.finalizeStepEntity(stepEntity, false, finalErrorMessage);
             const errorNextStep = {
-                errorMessage: `Prompt execution failed: ${errString}`,
+                errorMessage: finalErrorMessage,
                 step: 'Failed',
-                terminate: false,
+                terminate: shouldTerminate,
                 previousPayload: payload,
                 newPayload: payload
             };
-            const guardailCheck = await this.hasExceededAgentRunGuardrails(params, this.AgentRun);
-            if (guardailCheck && guardailCheck.exceeded) {
-                errorNextStep.terminate = true;
-            }
-            ;
             return errorNextStep;
         }
     }
-    async executeChildSubAgentStep(params, previousDecision) {
+    computeUpstreamDownstreamPaths(params, subAgentEntity, subAgentRequest) {
+        let downstreamPaths = ['*'];
+        let upstreamPaths = ['*'];
+        try {
+            if (subAgentEntity.PayloadDownstreamPaths) {
+                downstreamPaths = JSON.parse(subAgentEntity.PayloadDownstreamPaths);
+            }
+            if (subAgentEntity.PayloadUpstreamPaths) {
+                upstreamPaths = JSON.parse(subAgentEntity.PayloadUpstreamPaths);
+            }
+        }
+        catch (parseError) {
+            this.logError(`Failed to parse payload paths for sub-agent ${subAgentRequest.name}: ${parseError.message}`, {
+                category: 'SubAgentExecution',
+                metadata: {
+                    agentName: params.agent.Name,
+                    subAgentName: subAgentRequest.name,
+                    subAgentId: subAgentEntity.ID,
+                    downstreamPaths: subAgentEntity.PayloadDownstreamPaths,
+                    upstreamPaths: subAgentEntity.PayloadUpstreamPaths
+                }
+            });
+        }
+        return { downstreamPaths, upstreamPaths };
+    }
+    async computeChildSubAgentPayload(params, subAgentEntity, downstreamPaths, subAgentRequest, previousDecision) {
+        let downstreamPayload = this._payloadManager.extractDownstreamPayload(subAgentRequest.name, previousDecision.newPayload, downstreamPaths);
+        let scopedPayload = downstreamPayload;
+        if (subAgentEntity.PayloadScope) {
+            scopedPayload = this._payloadManager.applyPayloadScope(downstreamPayload, subAgentEntity.PayloadScope);
+            if (scopedPayload === null) {
+                const errorMessage = `Critical: Failed to extract payload scope '${subAgentEntity.PayloadScope}' for sub-agent '${subAgentRequest.name}'. The specified path does not exist in the payload.`;
+                this.logError(errorMessage, {
+                    category: 'SubAgentExecution',
+                    metadata: {
+                        agentName: params.agent.Name,
+                        subAgentName: subAgentRequest.name,
+                        payloadScope: subAgentEntity.PayloadScope,
+                        availableKeys: Object.keys(downstreamPayload || {})
+                    }
+                });
+                throw new Error(errorMessage);
+            }
+        }
+        return scopedPayload;
+    }
+    async executeChildSubAgentStep(params, previousDecision, parentStepId, subAgentPayloadOverride) {
         const subAgentRequest = previousDecision.subAgent;
         if (params.cancellationToken?.aborted) {
             throw new Error('Cancelled before sub-agent execution');
@@ -2214,48 +2470,16 @@ The context is now within limits. Please retry your request with the recovered c
         if (!subAgentEntity) {
             throw new Error(`Sub-agent '${subAgentRequest.name}' not found`);
         }
-        const stepEntity = await this.createStepEntity({ stepType: 'Sub-Agent', stepName: `Execute Sub-Agent: ${subAgentRequest.name}`, contextUser: params.contextUser, targetId: subAgentEntity.ID, inputData, payloadAtStart: previousDecision.newPayload });
+        const stepEntity = await this.createStepEntity({ stepType: 'Sub-Agent', stepName: `Execute Sub-Agent: ${subAgentRequest.name}`, contextUser: params.contextUser, targetId: subAgentEntity.ID, inputData, payloadAtStart: previousDecision.newPayload, parentId: parentStepId });
         this.incrementExecutionCount(subAgentEntity.ID);
         try {
-            let downstreamPaths = ['*'];
-            let upstreamPaths = ['*'];
-            try {
-                if (subAgentEntity.PayloadDownstreamPaths) {
-                    downstreamPaths = JSON.parse(subAgentEntity.PayloadDownstreamPaths);
-                }
-                if (subAgentEntity.PayloadUpstreamPaths) {
-                    upstreamPaths = JSON.parse(subAgentEntity.PayloadUpstreamPaths);
-                }
+            const { downstreamPaths, upstreamPaths } = this.computeUpstreamDownstreamPaths(params, subAgentEntity, subAgentRequest);
+            let scopedPayload = null;
+            if (!subAgentPayloadOverride) {
+                scopedPayload = await this.computeChildSubAgentPayload(params, subAgentEntity, downstreamPaths, subAgentRequest, previousDecision);
             }
-            catch (parseError) {
-                this.logError(`Failed to parse payload paths for sub-agent ${subAgentRequest.name}: ${parseError.message}`, {
-                    category: 'SubAgentExecution',
-                    metadata: {
-                        agentName: params.agent.Name,
-                        subAgentName: subAgentRequest.name,
-                        subAgentId: subAgentEntity.ID,
-                        downstreamPaths: subAgentEntity.PayloadDownstreamPaths,
-                        upstreamPaths: subAgentEntity.PayloadUpstreamPaths
-                    }
-                });
-            }
-            let downstreamPayload = this._payloadManager.extractDownstreamPayload(subAgentRequest.name, previousDecision.newPayload, downstreamPaths);
-            let scopedPayload = downstreamPayload;
-            if (subAgentEntity.PayloadScope) {
-                scopedPayload = this._payloadManager.applyPayloadScope(downstreamPayload, subAgentEntity.PayloadScope);
-                if (scopedPayload === null) {
-                    const errorMessage = `Critical: Failed to extract payload scope '${subAgentEntity.PayloadScope}' for sub-agent '${subAgentRequest.name}'. The specified path does not exist in the payload.`;
-                    this.logError(errorMessage, {
-                        category: 'SubAgentExecution',
-                        metadata: {
-                            agentName: params.agent.Name,
-                            subAgentName: subAgentRequest.name,
-                            payloadScope: subAgentEntity.PayloadScope,
-                            availableKeys: Object.keys(downstreamPayload || {})
-                        }
-                    });
-                    throw new Error(errorMessage);
-                }
+            else {
+                scopedPayload = subAgentPayloadOverride;
             }
             stepEntity.PayloadAtStart = JSON.stringify(previousDecision.newPayload);
             const subAgentResult = await this.ExecuteSubAgent(params, subAgentRequest, subAgentEntity, stepEntity, scopedPayload, undefined);
@@ -2404,7 +2628,7 @@ The context is now within limits. Please retry your request with the recovered c
             };
         }
     }
-    async processSubAgentStep(params, previousDecision) {
+    async processSubAgentStep(params, previousDecision, parentStepId, subAgentPayloadOverride) {
         const subAgentRequest = previousDecision.subAgent;
         const name = subAgentRequest?.name;
         if (!name) {
@@ -2420,7 +2644,7 @@ The context is now within limits. Please retry your request with the recovered c
             a.Status === 'Active');
         const childAgent = childAgents.find(a => a.Name.trim().toLowerCase() === name.trim().toLowerCase());
         if (childAgent) {
-            return await this.executeChildSubAgentStep(params, previousDecision);
+            return await this.executeChildSubAgentStep(params, previousDecision, parentStepId, subAgentPayloadOverride);
         }
         const activeRelationships = aiengine_1.AIEngine.Instance.AgentRelationships.filter(ar => ar.AgentID === params.agent.ID &&
             ar.Status === 'Active');
@@ -2428,7 +2652,7 @@ The context is now within limits. Please retry your request with the recovered c
             const relatedAgent = aiengine_1.AIEngine.Instance.Agents.find(a => a.ID === relationship.SubAgentID &&
                 a.Status === 'Active');
             if (relatedAgent && relatedAgent.Name.trim().toLowerCase() === name.trim().toLowerCase()) {
-                return await this.executeRelatedSubAgentStep(params, previousDecision, relatedAgent, relationship);
+                return await this.executeRelatedSubAgentStep(params, previousDecision, relatedAgent, relationship, parentStepId, subAgentPayloadOverride);
             }
         }
         this.logError(`Sub-agent '${name}' not found or not active for agent '${params.agent.Name}'`, {
@@ -2443,7 +2667,7 @@ The context is now within limits. Please retry your request with the recovered c
             newPayload: previousDecision?.newPayload
         };
     }
-    async executeRelatedSubAgentStep(params, previousDecision, subAgentEntity, relationship) {
+    async executeRelatedSubAgentStep(params, previousDecision, subAgentEntity, relationship, parentStepId, subAgentPayloadOverride) {
         const subAgentRequest = previousDecision.subAgent;
         if (params.cancellationToken?.aborted) {
             throw new Error('Cancelled before related sub-agent execution');
@@ -2478,13 +2702,14 @@ The context is now within limits. Please retry your request with the recovered c
             contextUser: params.contextUser,
             targetId: subAgentEntity.ID,
             inputData,
-            payloadAtStart: previousDecision.newPayload
+            payloadAtStart: previousDecision.newPayload,
+            parentId: parentStepId
         });
         this.incrementExecutionCount(subAgentEntity.ID);
         try {
             stepEntity.PayloadAtStart = JSON.stringify(previousDecision.newPayload);
-            let initialSubAgentPayload = undefined;
-            if (relationship.SubAgentInputMapping) {
+            let initialSubAgentPayload = subAgentPayloadOverride;
+            if (!initialSubAgentPayload && relationship.SubAgentInputMapping) {
                 const mapped = this.applySubAgentInputMapping(previousDecision.newPayload, relationship.SubAgentInputMapping);
                 if (mapped && Object.keys(mapped).length > 0) {
                     initialSubAgentPayload = mapped;
@@ -2607,12 +2832,13 @@ The context is now within limits. Please retry your request with the recovered c
                     let current = updateObj;
                     for (let i = 0; i < pathParts.length - 1; i++) {
                         const part = pathParts[i];
-                        if (!(part in current)) {
-                            current[part] = {};
+                        const cleanPart = part.endsWith('[]') ? part.slice(0, -2) : part;
+                        if (!(cleanPart in current)) {
+                            current[cleanPart] = {};
                         }
-                        current = current[part];
+                        current = current[cleanPart];
                     }
-                    current[pathParts[pathParts.length - 1]] = value;
+                    this.setMappedValue(current, pathParts[pathParts.length - 1], value);
                 }
             }
             if (Object.keys(updateObj).length === 0) {
@@ -2627,6 +2853,23 @@ The context is now within limits. Please retry your request with the recovered c
             return null;
         }
     }
+    setMappedValue(target, key, value) {
+        const isArrayAppend = key.endsWith('[]');
+        const actualKey = isArrayAppend ? key.slice(0, -2) : key;
+        if (isArrayAppend) {
+            if (!(actualKey in target)) {
+                target[actualKey] = [];
+            }
+            if (!Array.isArray(target[actualKey])) {
+                throw new Error(`Cannot append to '${actualKey}': target is not an array. ` +
+                    `Use '${actualKey}' without [] suffix for property update.`);
+            }
+            target[actualKey].push(value);
+        }
+        else {
+            target[actualKey] = value;
+        }
+    }
     applySubAgentInputMapping(parentPayload, mappingConfig) {
         try {
             const mapping = JSON.parse(mappingConfig);
@@ -2784,7 +3027,7 @@ The context is now within limits. Please retry your request with the recovered c
             const agentActions = aiengine_1.AIEngine.Instance.AgentActions.filter(aa => aa.AgentID === params.agent.ID);
             try {
                 const currentPayload = previousDecision?.newPayload || previousDecision?.previousPayload || params.payload;
-                await this.AgentTypeInstance.PreProcessActionStep(actions, currentPayload, this.AgentTypeState, previousDecision);
+                await this.AgentTypeInstance.PreProcessActionStep(actions, currentPayload, this.AgentTypeState, previousDecision, params);
             }
             catch (error) {
                 (0, core_1.LogError)(`Error in PreProcessActionStep: ${error.message}`);
@@ -2793,7 +3036,7 @@ The context is now within limits. Please retry your request with the recovered c
             const baseStepNumber = (this._agentRun.Steps?.length || 0) + 1;
             let lastStep = undefined;
             const actionPromises = actions.map(async (aa) => {
-                const actionEntity = actionEngine.Actions.find(a => a.Name === aa.name && agentActions.some(aa => aa.ActionID === a.ID));
+                const actionEntity = actionEngine.Actions.find(a => a.Name === aa.name && agentActions.some(agentAction => agentAction.ActionID === a.ID));
                 if (!actionEntity) {
                     throw new Error(`Action "${aa.name}" Not Found for Agent "${params.agent.Name}"`);
                 }
@@ -2892,7 +3135,7 @@ The context is now within limits. Please retry your request with the recovered c
                 if (payloadChangeRequest) {
                     const allowedPaths = params.agent.PayloadSelfWritePaths
                         ? JSON.parse(params.agent.PayloadSelfWritePaths)
-                        : undefined;
+                        : this.getDefaultPayloadSelfWritePaths();
                     const changeResult = this._payloadManager.applyAgentChangeRequest(currentPayload, payloadChangeRequest, {
                         validateChanges: true,
                         logChanges: true,
@@ -3147,10 +3390,17 @@ The context is now within limits. Please retry your request with the recovered c
     async executeSingleForEachIteration(forEach, item, index, currentPayload, parentStepId, params, config) {
         try {
             const beforeHook = this.AgentTypeInstance.BeforeLoopIteration?.({ item, itemVariable: forEach.itemVariable, index, payload: currentPayload, loopType: 'ForEach', actionParams: forEach.action?.params || {} });
-            const resolvedParams = beforeHook?.actionParams || forEach.action?.params || {};
+            let resolvedParams = beforeHook?.actionParams || forEach.action?.params || {};
             let result;
             if (forEach.action) {
                 const matchedAction = this.findAgentActionForLoop(forEach.action.name, params.agent.ID, params.agent.Name, params);
+                const context = {
+                    item,
+                    index,
+                    payload: currentPayload,
+                    data: params.data
+                };
+                resolvedParams = this.resolveTemplates(resolvedParams, context, forEach.itemVariable);
                 const resolvedAction = {
                     name: matchedAction.Action,
                     params: resolvedParams
@@ -3160,7 +3410,7 @@ The context is now within limits. Please retry your request with the recovered c
             }
             else if (forEach.subAgent) {
                 const subAgentStep = { step: 'Sub-Agent', subAgent: forEach.subAgent, newPayload: currentPayload, previousPayload: currentPayload };
-                result = await this.processSubAgentStep(params, subAgentStep);
+                result = await this.processSubAgentStep(params, subAgentStep, parentStepId, item);
             }
             else {
                 throw new Error('ForEach missing action/subAgent');
@@ -3283,10 +3533,17 @@ The context is now within limits. Please retry your request with the recovered c
     async executeSingleWhileIteration(whileOp, attemptContext, index, currentPayload, parentStepId, params, config) {
         try {
             const beforeHook = this.AgentTypeInstance.BeforeLoopIteration?.({ item: attemptContext, index, itemVariable: whileOp.itemVariable, payload: currentPayload, loopType: 'While', actionParams: whileOp.action?.params || {} });
-            const resolvedParams = beforeHook?.actionParams || whileOp.action?.params || {};
+            let resolvedParams = beforeHook?.actionParams || whileOp.action?.params || {};
             let result;
             if (whileOp.action) {
                 const matchedAction = this.findAgentActionForLoop(whileOp.action.name, params.agent.ID, params.agent.Name, params);
+                const context = {
+                    item: attemptContext,
+                    index,
+                    payload: currentPayload,
+                    data: params.data
+                };
+                resolvedParams = this.resolveTemplates(resolvedParams, context, whileOp.itemVariable || 'item');
                 const resolvedAction = {
                     name: matchedAction.Action,
                     params: resolvedParams
@@ -3296,7 +3553,7 @@ The context is now within limits. Please retry your request with the recovered c
             }
             else if (whileOp.subAgent) {
                 const subAgentStep = { step: 'Sub-Agent', subAgent: whileOp.subAgent, newPayload: currentPayload, previousPayload: currentPayload };
-                result = await this.processSubAgentStep(params, subAgentStep);
+                result = await this.processSubAgentStep(params, subAgentStep, parentStepId, attemptContext);
             }
             else {
                 throw new Error('While missing action/subAgent');
@@ -3739,6 +3996,79 @@ The context is now within limits. Please retry your request with the recovered c
             console.log(`[Turn ${currentTurn}] Expanded message at index ${messageIndex}`);
         }
     }
+    resolveTemplates(obj, context, itemVariable) {
+        const result = {};
+        for (const [key, value] of Object.entries(obj)) {
+            if (typeof value === 'string') {
+                result[key] = this.resolveValueFromContext(value, context, itemVariable);
+            }
+            else if (Array.isArray(value)) {
+                result[key] = value.map(v => typeof v === 'string' ? this.resolveValueFromContext(v, context, itemVariable) : v);
+            }
+            else if (typeof value === 'object' && value !== null) {
+                result[key] = this.resolveTemplates(value, context, itemVariable);
+            }
+            else {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    resolveValueFromContext(value, context, itemVariable) {
+        const trimmedValue = value.trim();
+        if (trimmedValue.startsWith('{{') && trimmedValue.endsWith('}}')) {
+            value = trimmedValue.substring(2, trimmedValue.length - 2).trim();
+        }
+        const ivToLower = itemVariable?.trim().toLowerCase();
+        if (value?.toLowerCase().startsWith(`${ivToLower}.`)) {
+            const path = value.substring(ivToLower.length + 1);
+            return this.getValueFromPath(context.item, path);
+        }
+        for (const [varName, varValue] of Object.entries(context)) {
+            if (value === varName) {
+                return varValue;
+            }
+            if (value?.trim().toLowerCase().startsWith(`${varName}.`)) {
+                const path = value.substring(varName.length + 1);
+                return this.getValueFromPath(varValue, path);
+            }
+        }
+        return value;
+    }
+    getValueFromPath(obj, path) {
+        const parts = path.split('.');
+        let current = obj;
+        for (const part of parts) {
+            if (!part)
+                continue;
+            const arrayMatch = part.match(/^([^[]+)\[(\d+)\]$/);
+            if (arrayMatch) {
+                const arrayName = arrayMatch[1];
+                const index = parseInt(arrayMatch[2], 10);
+                if (current && typeof current === 'object' && arrayName in current) {
+                    current = current[arrayName];
+                    if (Array.isArray(current) && index >= 0 && index < current.length) {
+                        current = current[index];
+                    }
+                    else {
+                        return undefined;
+                    }
+                }
+                else {
+                    return undefined;
+                }
+            }
+            else {
+                if (current && typeof current === 'object' && part in current) {
+                    current = current[part];
+                }
+                else {
+                    return undefined;
+                }
+            }
+        }
+        return current;
+    }
 }
 exports.BaseAgent = BaseAgent;
 BaseAgent.MAX_VALIDATION_RETRIES = 10;