@memberjunction/ai-agents 5.17.0 → 5.18.0

package/dist/base-agent.js

@@ -22,6 +22,7 @@ import { RerankerService } from '@memberjunction/ai-reranker';
 import { AIPromptParams, ChildPromptParam, ConversationUtility, parseAssignmentStrategy } from '@memberjunction/ai-core-plus';
 import { AgentRunner } from './AgentRunner.js';
 import { PayloadManager } from './PayloadManager.js';
+import { ScratchpadManager } from './ScratchpadManager.js';
 import { AgentDataPreloader } from './AgentDataPreloader.js';
 import { ConversationMessageResolver } from './utils/ConversationMessageResolver.js';
 import _ from 'lodash';
@@ -158,6 +159,13 @@ export class BaseAgent {
          * @private
          */
         this._payloadManager = new PayloadManager();
+        /**
+         * Scratchpad manager for private agent working memory (notes + task list).
+         * Instantiated per agent run, garbage collected when the run ends.
+         * @private
+         * @since 2.46.0
+         */
+        this._scratchpadManager = new ScratchpadManager();
         /**
          * Effective actions available to this agent after applying actionChanges.
          * Populated during gatherPromptTemplateData() and used for validation in executeActionsStep().
@@ -818,6 +826,8 @@ export class BaseAgent {
             if (params.convertUIMarkupToPlainText !== false) {
                 this.convertUIMarkupInMessages(wrappedParams.conversationMessages);
             }
+            // Reset scratchpad for each new execution (ephemeral per run)
+            this._scratchpadManager.Clear();
             await this.initializeStartingPayload(wrappedParams);
             // Check for cancellation at start
             if (params.cancellationToken?.aborted) {
@@ -1506,6 +1516,16 @@ export class BaseAgent {
             agentId: params.agent.ID,
             agentRunId: this._agentRun?.ID
         });
+        // Inject scratchpad template variables if scratchpad is enabled
+        if (promptParams.data) {
+            const agentTypePromptParams = promptParams.data.__agentTypePromptParams;
+            const scratchpadEnabled = agentTypePromptParams?.includeScratchpadDocs !== false;
+            if (scratchpadEnabled && this._scratchpadManager) {
+                promptParams.data['_SCRATCHPAD_NOTES'] = this._scratchpadManager.GetNotes() || '_(no notes yet)_';
+                promptParams.data['_SCRATCHPAD_TASKS'] = this._scratchpadManager.ToPromptString();
+                promptParams.data['_SCRATCHPAD_TASK_SUMMARY'] = this._scratchpadManager.GetTaskSummary();
+            }
+        }
         // Only set up child prompts if we have a system prompt
         if (systemPrompt) {
             // Setup child prompt parameters
@@ -2879,24 +2899,21 @@ The context is now within limits. Please retry your request with the recovered c
      * @protected
      */
     createActionResultMessage(actions, results) {
-        const resultSummary = actions.map((action, index) => {
+        const actionSummaries = actions.map((action, index) => {
             const result = results[index];
-            const outputParams = result.Params?.filter((p) => p.Type === 'Output' || p.Type === 'Both') || [];
+            const outputParams = result.Params?.filter(p => p.Type === 'Output' || p.Type === 'Both') || [];
             return {
                 actionName: action.name,
                 success: result.Success,
+                params: outputParams,
                 resultCode: result.Result?.ResultCode || 'N/A',
-                message: result.Message || null,
+                message: result.Message || '(no message)',
                 aiDirectives: result.AIDirectives,
-                outputs: outputParams.reduce((acc, param) => {
-                    acc[param.Name] = param.Value;
-                    return acc;
-                }, {})
             };
         });
         return {
             role: 'user',
-            content: `Action results:\n${JSON.stringify(resultSummary, null, 2)}`
+            content: `Action results:\n${this.formatActionResultsAsMarkdown(actionSummaries)}`
         };
     }
     /**
@@ -2910,7 +2927,7 @@ The context is now within limits. Please retry your request with the recovered c
     createSubAgentResultMessage(subAgent, result) {
         return {
             role: 'user',
-            content: `Sub-agent '${subAgent.name}' result:\n${typeof result === 'string' ? result : JSON.stringify(result, null, 2)}`
+            content: this.formatSubAgentResultAsMarkdown(subAgent.name, result)
         };
     }
     /**
@@ -3068,7 +3085,8 @@ The context is now within limits. Please retry your request with the recovered c
                 responseForms: true,
                 commands: true,
                 forEach: true,
-                while: true
+                while: true,
+                scratchpad: true
             };
         }
         const responseType = params.includeResponseTypeDefinition;
@@ -3078,7 +3096,8 @@ The context is now within limits. Please retry your request with the recovered c
             { docsFlag: 'includeResponseFormDocs', responseTypeKey: 'responseForms' },
             { docsFlag: 'includeCommandDocs', responseTypeKey: 'commands' },
             { docsFlag: 'includeForEachDocs', responseTypeKey: 'forEach' },
-            { docsFlag: 'includeWhileDocs', responseTypeKey: 'while' }
+            { docsFlag: 'includeWhileDocs', responseTypeKey: 'while' },
+            { docsFlag: 'includeScratchpadDocs', responseTypeKey: 'scratchpad' }
         ];
         for (const { docsFlag, responseTypeKey } of alignmentMappings) {
             // Check if the user explicitly set this response type property
@@ -3370,27 +3389,20 @@ The context is now within limits. Please retry your request with the recovered c
         }
     }
     /**
-     * Formats sub-agent details for inclusion in prompt context.
+     * Formats sub-agent details as compact markdown for inclusion in prompt context.
      *
      * @param {MJAIAgentEntityExtended[]} subAgents - Array of sub-agent entities
-     * @returns {string} JSON formatted string with sub-agent details
+     * @returns {string} Markdown formatted string with sub-agent details
      * @private
      */
     formatSubAgentDetails(subAgents) {
-        return JSON.stringify(subAgents.map(sa => {
-            const result = {
-                Name: sa.Name,
-                Description: sa.Description
-            };
+        return subAgents.map(sa => {
+            let line = `- **${sa.Name}** — ${sa.Description}`;
             if (sa.ExecutionMode !== 'Sequential') {
-                // no need to include these two attributes for sub-agents
-                // that are sequential and the order is implied via the array order
-                // saves tokens
-                result['ExecutionMode'] = sa.ExecutionMode;
-                result['ExecutionOrder'] = sa.ExecutionOrder;
+                line += ` _(${sa.ExecutionMode}, order: ${sa.ExecutionOrder})_`;
             }
-            return result;
-        }), null, 2);
+            return line;
+        }).join('\n');
     }
     /**
      * Utility method to get agent prompt parameters for a given agent. This gets the
@@ -3422,53 +3434,178 @@ The context is now within limits. Please retry your request with the recovered c
         })), null, 2);
     }
     /**
-     * Formats action details for inclusion in prompt context.
+     * Formats action details as compact markdown for inclusion in prompt context.
+     * Produces ~75% fewer tokens than the previous JSON format while preserving
+     * all information the LLM needs to invoke actions correctly.
      *
      * @param {MJActionEntityExtended[]} actions - Array of action entities
-     * @returns {string} JSON formatted string with comprehensive action details
+     * @returns {string} Markdown formatted string with action details
      * @private
      */
     formatActionDetails(actions) {
-        return JSON.stringify(actions.map(action => ({
-            Name: action.Name,
-            Description: action.Description,
-            // Parameters with detailed information
-            Parameters: {
-                Input: action.Params
-                    .filter(p => p.Type.trim().toLowerCase() === 'input' || p.Type.trim().toLowerCase() === 'both')
-                    .map(param => this.formatActionParameter(param)),
-                Output: action.Params
-                    .filter(p => p.Type.trim().toLowerCase() === 'output' || p.Type.trim().toLowerCase() === 'both')
-                    .map(param => this.formatActionParameter(param))
-            },
-            // Result codes with detailed information
-            ResultCodes: action.ResultCodes.map(rc => ({
-                Code: rc.ResultCode,
-                IsSuccess: rc.IsSuccess,
-                Description: rc.Description || 'No description provided',
-            })),
-            // Additional metadata
-            Category: action.CategoryID ? this.getActionCategoryName(action.CategoryID) : null,
-            Status: action.Status
-        })), null, 2);
+        return actions.map(action => {
+            const lines = [];
+            lines.push(`### ${action.Name}`);
+            lines.push(action.Description);
+            const inputParams = action.Params
+                .filter(p => {
+                const t = p.Type.trim().toLowerCase();
+                return t === 'input' || t === 'both';
+            });
+            const outputParams = action.Params
+                .filter(p => {
+                const t = p.Type.trim().toLowerCase();
+                return t === 'output' || t === 'both';
+            });
+            if (inputParams.length > 0) {
+                lines.push(`**Input:** ${inputParams.map(p => this.formatActionParameter(p)).join(', ')}`);
+            }
+            if (outputParams.length > 0) {
+                lines.push(`**Output:** ${outputParams.map(p => this.formatActionParameter(p)).join(', ')}`);
+            }
+            if (action.ResultCodes.length > 0) {
+                const rcParts = action.ResultCodes.map(rc => {
+                    const marker = rc.IsSuccess ? '✓' : '✗';
+                    const desc = rc.Description && rc.Description.toLowerCase() !== rc.ResultCode.toLowerCase()
+                        ? ` ${rc.Description}`
+                        : '';
+                    return `${rc.ResultCode} ${marker}${desc}`;
+                });
+                lines.push(`**Results:** ${rcParts.join(' · ')}`);
+            }
+            return lines.join('\n');
+        }).join('\n\n');
     }
     /**
-     * Formats a single action parameter for display.
+     * Formats a single action parameter as a compact inline string.
+     * Uses \* suffix for required, (array) when IsArray, and only shows
+     * ValueType when it is not Scalar/Other.
      *
-     * @param {any} param - The action parameter to format
-     * @returns {object} Formatted parameter object
+     * @param {MJActionParamEntity} param - The action parameter to format
+     * @returns {string} Compact inline string, e.g. `` `To`\* — Email address ``
      * @private
      */
     formatActionParameter(param) {
-        return {
-            Name: param.Name,
-            Type: param.Type,
-            IsRequired: param.IsRequired,
-            IsArray: param.IsArray,
-            DefaultValue: param.DefaultValue,
-            Description: param.Description,
-            ValueType: param.ValueType
-        };
+        const requiredMarker = param.IsRequired ? '\\*' : '';
+        const parts = [];
+        if (param.IsArray) {
+            parts.push('array');
+        }
+        const vt = param.ValueType?.trim();
+        if (vt && vt !== 'Scalar' && vt !== 'Other') {
+            parts.push(vt);
+        }
+        const suffix = parts.length > 0 ? ` (${parts.join(', ')})` : '';
+        let defaultStr = '';
+        if (param.DefaultValue != null && param.DefaultValue !== '') {
+            defaultStr = ` (default: ${JSON.stringify(param.DefaultValue)})`;
+        }
+        const desc = param.Description ? ` — ${param.Description}` : '';
+        return `\`${param.Name}\`${requiredMarker}${suffix}${desc}${defaultStr}`;
+    }
+    /**
+     * Formats an array of action result summaries as compact markdown instead of
+     * pretty-printed JSON.  This typically saves 60-70 % of the tokens that the
+     * old `JSON.stringify(actionSummaries, null, 2)` approach consumed, while
+     * remaining equally parseable by LLMs.
+     *
+     * Format per action:
+     * ```
+     * ## ActionName ✓          (or ✗ for failure)
+     * **Result:** RESULT_CODE — human message
+     * **Output:**
+     * • `ParamName`: value
+     * ```
+     *
+     * AI directives are intentionally omitted here because they are surfaced
+     * as a separate high-priority message (see the caller).
+     *
+     * @param actionSummaries - The action summary objects built by executeActionStep
+     * @returns Compact markdown string
+     * @private
+     */
+    formatActionResultsAsMarkdown(actionSummaries) {
+        return actionSummaries.map(a => {
+            const marker = a.success ? '✓' : '✗';
+            const lines = [];
+            lines.push(`## ${a.actionName} ${marker}`);
+            lines.push(`**Result:** ${a.resultCode} — ${a.message || '(no message)'}`);
+            // Format output params as bullet list
+            if (a.params && a.params.length > 0) {
+                lines.push('**Output:**');
+                for (const p of a.params) {
+                    lines.push(`• \`${p.Name}\`: ${this.formatParamValueForResult(p.Value)}`);
+                }
+            }
+            return lines.join('\n');
+        }).join('\n\n');
+    }
+    /**
+     * Formats an ExecuteAgentResult as compact markdown for inclusion in
+     * conversation messages.  Extracts only the fields that are meaningful
+     * to the calling agent (success, status, error, and payload) and
+     * presents them in a concise format.
+     *
+     * @param subAgentName - Display name of the sub-agent
+     * @param result       - The execution result from the sub-agent
+     * @returns Compact markdown string
+     * @private
+     */
+    formatSubAgentResultAsMarkdown(subAgentName, result) {
+        const marker = result.success ? '✓' : '✗';
+        const lines = [];
+        lines.push(`## Sub-agent: ${subAgentName} ${marker}`);
+        // Status / error from the agent run
+        const status = result.agentRun?.Status || (result.success ? 'Completed' : 'Failed');
+        lines.push(`**Status:** ${status}`);
+        if (!result.success && result.agentRun?.ErrorMessage) {
+            lines.push(`**Error:** ${result.agentRun.ErrorMessage}`);
+        }
+        // Payload — the actual result data the parent agent cares about
+        if (result.payload != null) {
+            const payloadStr = typeof result.payload === 'string'
+                ? result.payload
+                : JSON.stringify(result.payload);
+            // For very large payloads, truncate to keep context manageable
+            if (payloadStr.length > 4000) {
+                lines.push(`**Payload** (truncated):\n${payloadStr.substring(0, 4000)}…`);
+            }
+            else {
+                lines.push(`**Payload:**\n${payloadStr}`);
+            }
+        }
+        return lines.join('\n');
+    }
+    /**
+     * Formats a single output parameter value for inclusion in action result
+     * markdown.  Scalars are shown inline with backtick formatting; objects and
+     * arrays use compact (single-line) JSON to avoid the indentation overhead
+     * of pretty-printed JSON.  Very long values are truncated.
+     *
+     * @param value - The parameter value (any type)
+     * @param maxLength - Maximum character length before truncation (default 500)
+     * @returns Formatted string
+     * @private
+     */
+    formatParamValueForResult(value, maxLength = 500) {
+        if (value === null || value === undefined) {
+            return '`null`';
+        }
+        if (typeof value === 'boolean' || typeof value === 'number') {
+            return `\`${String(value)}\``;
+        }
+        let stringValue;
+        if (typeof value === 'string') {
+            stringValue = value;
+        }
+        else {
+            // Compact JSON (no pretty-printing) for objects/arrays
+            stringValue = JSON.stringify(value);
+        }
+        if (stringValue.length > maxLength) {
+            return `${stringValue.substring(0, maxLength)}…`;
+        }
+        return stringValue;
     }
     /**
      * Formats a parameter value for display in action execution messages.
@@ -4203,7 +4340,8 @@ The context is now within limits. Please retry your request with the recovered c
         const promptToUse = await this.AgentTypeInstance.GetPromptForStep(params, config, previousDecision?.newPayload || previousDecision?.previousPayload, this.AgentTypeState, previousDecision);
         const promptId = promptToUse?.ID;
         const promptName = promptToUse?.Name;
-        // Prepare input data for the step
+        // Prepare input data for the step (includes scratchpad snapshot before LLM response)
+        const scratchpadSnapshotBeforeStep = this._scratchpadManager.HasContent() ? this._scratchpadManager.ToJSON() : undefined;
         const inputData = {
             promptId: promptId,
             promptName: promptName,
@@ -4213,6 +4351,7 @@ The context is now within limits. Please retry your request with the recovered c
                 instructions: previousDecision.retryInstructions
             } : undefined,
             conversationMessages: params.conversationMessages,
+            ...(scratchpadSnapshotBeforeStep && { scratchpad: scratchpadSnapshotBeforeStep }),
         };
         // Prepare prompt parameters
         const payload = previousDecision?.newPayload || params.payload;
@@ -4376,6 +4515,18 @@ The context is now within limits. Please retry your request with the recovered c
                 // Set the final payload - the changeResult already respects the allowed paths
                 finalPayload = changeResult.result;
             }
+            // Apply scratchpad changes if provided (zero turn cost — processed inline)
+            if (initialNextStep.scratchpad) {
+                this._scratchpadManager.ApplyScratchpadChanges(initialNextStep.scratchpad);
+                // Enforce task limit using the default (50) — the prompt params were already
+                // used during prompt preparation. We use a simple default here to avoid
+                // re-fetching prompt params just for this value.
+                const maxTasks = 50;
+                const pruned = this._scratchpadManager.EnforceTaskLimit(maxTasks);
+                if (pruned > 0 && (params.verbose === true || IsVerboseLoggingEnabled())) {
+                    LogStatus(`Scratchpad: pruned ${pruned} completed tasks (limit: ${maxTasks})`);
+                }
+            }
             // now that we have processed the payload, we can process the next step which does validation and changes the next step if
             // validation fails
             const updatedNextStep = await this.processNextStep(initialNextStep, params, config.agentType, promptResult, finalPayload, stepEntity);
@@ -4394,6 +4545,10 @@ The context is now within limits. Please retry your request with the recovered c
                 ...(currentStepPayloadChangeResult && {
                     payloadChangeResult: currentStepPayloadChangeResult
                 }),
+                // Include scratchpad snapshot after changes for audit/training data
+                ...(this._scratchpadManager.HasContent() && {
+                    scratchpad: this._scratchpadManager.ToJSON()
+                }),
                 // Include memory attribution for observability
                 // This tracks which notes/examples were injected and influenced this step
                 ...(this._injectedMemory && (this._injectedMemory.notes.length > 0 || this._injectedMemory.examples.length > 0) && {
@@ -5483,14 +5638,17 @@ The context is now within limits. Please retry your request with the recovered c
                     success: result.success,
                     params: sanitizedParams,
                     resultCode: actionResult?.Result?.ResultCode || (result.success ? 'SUCCESS' : 'ERROR'),
-                    message: result.success ? actionResult?.Message || 'Action completed' : result.error,
+                    message: result.success ? actionResult?.Message || 'Action completed' : result.error || 'Unknown error',
                     aiDirectives: result.success ? actionResult?.AIDirectives : undefined
                 };
             });
             // Check if any actions failed
             const failedActions = actionSummaries.filter(a => !a.success);
-            // Add user message with the results
-            const resultsMessage = (failedActions.length > 0 ? `${failedActions.length} of ${actionSummaries.length} failed:` : `Action results:`) + `\n${JSON.stringify(actionSummaries, null, 2)}`;
+            // Add user message with the results — compact markdown instead of JSON
+            const header = failedActions.length > 0
+                ? `${failedActions.length} of ${actionSummaries.length} action(s) failed:`
+                : `Action results:`;
+            const resultsMessage = `${header}\n${this.formatActionResultsAsMarkdown(actionSummaries)}`;
             // Build metadata from AI Agent Actions configuration
             // If multiple actions, use the most restrictive (shortest) expiration settings
             let metadata;
@@ -7046,8 +7204,16 @@ The context is now within limits. Please retry your request with the recovered c
         // and we trim whitespace first
         const trimmedValue = value.trim();
         if (trimmedValue.startsWith('{{') && trimmedValue.endsWith('}}')) {
+            // Single-variable case: "{{city.name}}" — strip wrappers, fall through to
+            // single-variable resolution below which can return non-string types (objects, numbers)
             value = trimmedValue.substring(2, trimmedValue.length - 2).trim();
         }
+        else if (trimmedValue.includes('{{')) {
+            // Inline template interpolation: "text {{var.prop}} more {{var2.field}}"
+            // The string contains embedded {{}} expressions mixed with literal text.
+            // Each expression is resolved individually and stringified back into the template.
+            return this.resolveInlineTemplateExpressions(trimmedValue, context, itemVariable);
+        }
         // Check itemVariable reference (the custom loop variable name like "entityName" or "user")
         // This handles both direct references and path-based property access
         const ivToLower = itemVariable?.trim().toLowerCase();
@@ -7079,6 +7245,32 @@ The context is now within limits. Please retry your request with the recovered c
         // Static value - no variable reference found, return as literal string
         return value;
     }
+    /**
+     * Resolves multiple {{expression}} placeholders embedded within a literal string.
+     *
+     * Unlike the single-variable path (which can return objects/numbers), this always
+     * returns a string because the resolved values are interpolated back into surrounding text.
+     *
+     * @example
+     * // Given itemVariable="cityInfo", context.item={city:"Tokyo", country:"Japan"}
+     * resolveInlineTemplateExpressions(
+     *   "largest company in {{cityInfo.city}} {{cityInfo.country}}",
+     *   context, "cityInfo"
+     * )
+     * // → "largest company in Tokyo Japan"
+     */
+    resolveInlineTemplateExpressions(template, context, itemVariable) {
+        const expressionPattern = /\{\{\s*([^}]+?)\s*\}\}/g;
+        return template.replace(expressionPattern, (_match, expr) => {
+            // Resolve each expression using the single-variable path (without {{ }} wrappers)
+            const resolved = this.resolveValueFromContext(expr.trim(), context, itemVariable);
+            // If the expression didn't resolve (returned unchanged), keep original {{ }} for transparency
+            if (resolved === expr.trim()) {
+                return _match;
+            }
+            return String(resolved ?? '');
+        });
+    }
     /**
      * Helper to get value from nested object path - extracts from LoopAgentType
      */