@memberjunction/ai-agents 2.108.0 → 2.110.0

package/README.md

@@ -268,6 +268,302 @@ The callback is invoked:
 
 ## Advanced Features
 
+### Agent Data Preloading
+
+Agents can declaratively preload reference data without requiring custom application code or action calls. Data sources are configured through the `AIAgentDataSource` entity and automatically loaded before agent execution.
+
+#### Overview
+
+Data preloading solves the common problem of agents needing access to reference data (like entity lists, configuration values, or initial state) that doesn't change during execution. Instead of:
+- Writing custom application code to load data
+- Having agents call actions to fetch data (which bloats conversation context)
+- Manually passing the same data to every agent invocation
+
+Agents can now specify data sources that are automatically loaded and injected into the appropriate destination (`data`, `context`, or `payload`).
+
+#### Three Destination Types
+
+**1. Data Destination** - For Nunjucks templates in prompts (visible to LLMs)
+```typescript
+// Configuration
+{
+  "Name": "ALL_ENTITIES",
+  "SourceType": "RunView",
+  "EntityName": "Entities",
+  "OrderBy": "Name ASC",
+  "DestinationType": "Data",
+  "DestinationPath": null  // Uses "ALL_ENTITIES" at root level
+}
+
+// Result in agent prompt:
+// params.data.ALL_ENTITIES = [{ Name: "Users", ... }, { Name: "Entities", ... }]
+
+// Prompt can use Nunjucks:
+// You have access to {{ALL_ENTITIES.length}} entities:
+// {% for entity in ALL_ENTITIES %}
+// - {{entity.Name}}: {{entity.Description}}
+// {% endfor %}
+```
+
+**2. Context Destination** - For actions only (NOT visible to LLMs)
+```typescript
+// Configuration
+{
+  "Name": "ORG_SETTINGS",
+  "SourceType": "RunView",
+  "EntityName": "Organization Settings",
+  "ExtraFilter": "OrgID='${context.organizationId}'",
+  "DestinationType": "Context",
+  "DestinationPath": "organization.settings"
+}
+
+// Result:
+// params.context.organization.settings = { apiEndpoint: "...", features: [...] }
+
+// Actions can access context, but prompts/LLMs cannot
+// This keeps API keys and sensitive configuration away from LLMs
+```
+
+**3. Payload Destination** - For agent state initialization
+```typescript
+// Configuration
+{
+  "Name": "CustomerOrders",
+  "SourceType": "RunQuery",
+  "QueryName": "Recent Orders by Customer",
+  "Parameters": JSON.stringify({ customerId: "{{context.customerId}}" }),
+  "DestinationType": "Payload",
+  "DestinationPath": "analysis.orders.recent"
+}
+
+// Result:
+// params.payload.analysis.orders.recent = [{ OrderID: "123", ... }]
+
+// Agent starts with rich initial state without caller manually loading it
+```
+
+#### Data Source Types
+
+**RunView Data Sources** - Query entities with filters
+```typescript
+{
+  "Name": "ACTIVE_MODELS",
+  "SourceType": "RunView",
+  "EntityName": "AI Models",
+  "ExtraFilter": "IsActive=1 AND Vendor='OpenAI'",
+  "OrderBy": "Priority DESC",
+  "FieldsToRetrieve": JSON.stringify(["ID", "Name", "Vendor", "MaxInputTokens"]),
+  "ResultType": "simple",  // or "entity_object"
+  "MaxRows": 100,
+  "DestinationType": "Data"
+}
+```
+
+**RunQuery Data Sources** - Execute stored queries
+```typescript
+{
+  "Name": "MONTHLY_STATS",
+  "SourceType": "RunQuery",
+  "QueryName": "Monthly Analytics",
+  "CategoryPath": "/Reports/Analytics",
+  "Parameters": JSON.stringify({
+    month: "{{context.currentMonth}}",
+    year: "{{context.currentYear}}"
+  }),
+  "DestinationType": "Payload",
+  "DestinationPath": "stats.monthly"
+}
+```
+
+#### Path Support
+
+The `DestinationPath` field supports nested paths using dot notation:
+
+```typescript
+// Simple root-level
+{
+  "Name": "ENTITIES",
+  "DestinationPath": null  // Uses "ENTITIES" at root
+}
+// Result: data.ENTITIES
+
+// Nested paths
+{
+  "Name": "ModelList",
+  "DestinationPath": "config.ai.models"
+}
+// Result: data.config.ai.models
+
+// Deep nesting
+{
+  "Name": "CustomerData",
+  "DestinationPath": "analysis.customer.profile.orders"
+}
+// Result: payload.analysis.customer.profile.orders
+```
+
+#### Caching Policies
+
+Data sources support three caching strategies:
+
+**1. None** - No caching (default)
+```typescript
+{
+  "CachePolicy": "None"
+  // Data is loaded fresh every time
+}
+```
+
+**2. PerRun** - Cache for duration of a single agent run
+```typescript
+{
+  "CachePolicy": "PerRun"
+  // Multiple data sources with same AgentID+Name share cached data within one run
+  // Cache is cleared when agent run completes
+}
+```
+
+**3. PerAgent** - Global cache with TTL
+```typescript
+{
+  "CachePolicy": "PerAgent",
+  "CacheTimeoutSeconds": 3600  // 1 hour
+  // Cached across all runs for this agent until TTL expires
+  // Good for rarely-changing reference data like entity lists
+}
+```
+
+#### Execution Control
+
+**Disable data preloading** for specific executions:
+```typescript
+const result = await runner.RunAgent({
+    agent: myAgent,
+    conversationMessages: messages,
+    contextUser: user,
+    disableDataPreloading: true  // Skip automatic data preloading
+});
+```
+
+**Caller precedence**: Caller-provided data always takes precedence over preloaded data:
+```typescript
+const result = await runner.RunAgent({
+    agent: myAgent,
+    conversationMessages: messages,
+    contextUser: user,
+    data: {
+        CUSTOM_ENTITIES: myEntities  // Overrides preloaded CUSTOM_ENTITIES
+    }
+});
+```
+
+#### Configuration Examples
+
+**Database Research Agent** - Preload entity metadata
+```typescript
+// Data source 1: All entities for reference
+{
+  "AgentID": "database-research-agent-id",
+  "Name": "ALL_ENTITIES",
+  "SourceType": "RunView",
+  "EntityName": "Entities",
+  "OrderBy": "Name ASC",
+  "FieldsToRetrieve": JSON.stringify(["ID", "Name", "SchemaName", "Description", "BaseView"]),
+  "DestinationType": "Data",
+  "ExecutionOrder": 1,
+  "Status": "Active",
+  "CachePolicy": "PerAgent",
+  "CacheTimeoutSeconds": 3600
+}
+
+// Data source 2: Schema information
+{
+  "AgentID": "database-research-agent-id",
+  "Name": "SCHEMA_INFO",
+  "SourceType": "RunView",
+  "EntityName": "Entity Fields",
+  "DestinationType": "Data",
+  "DestinationPath": "schema.fields",
+  "ExecutionOrder": 2,
+  "Status": "Active",
+  "CachePolicy": "PerAgent",
+  "CacheTimeoutSeconds": 3600
+}
+```
+
+**Customer Service Agent** - Preload customer context
+```typescript
+// Preload customer data into payload
+{
+  "AgentID": "customer-service-agent-id",
+  "Name": "CUSTOMER_PROFILE",
+  "SourceType": "RunView",
+  "EntityName": "Customers",
+  "ExtraFilter": "ID='{{context.customerId}}'",
+  "DestinationType": "Payload",
+  "DestinationPath": "customer.profile",
+  "Status": "Active",
+  "CachePolicy": "PerRun"
+}
+
+// Preload recent orders
+{
+  "AgentID": "customer-service-agent-id",
+  "Name": "RECENT_ORDERS",
+  "SourceType": "RunQuery",
+  "QueryName": "Recent Orders by Customer",
+  "Parameters": JSON.stringify({ customerId: "{{context.customerId}}", days: 30 }),
+  "DestinationType": "Payload",
+  "DestinationPath": "customer.orders",
+  "Status": "Active",
+  "CachePolicy": "PerRun"
+}
+
+// Preload organization settings (for actions)
+{
+  "AgentID": "customer-service-agent-id",
+  "Name": "ORG_CONFIG",
+  "SourceType": "RunView",
+  "EntityName": "Organization Settings",
+  "ExtraFilter": "OrgID='{{context.organizationId}}'",
+  "DestinationType": "Context",
+  "DestinationPath": "organization.config",
+  "Status": "Active",
+  "CachePolicy": "PerAgent",
+  "CacheTimeoutSeconds": 1800
+}
+```
+
+#### Benefits
+
+- **Declarative**: Configure data preloading through metadata, not code
+- **Reusable**: Same agent works across different environments
+- **Efficient**: Caching reduces redundant database queries
+- **Clean Separation**: Keeps data in appropriate destinations (data/context/payload)
+- **Flexible**: Supports both RunView and RunQuery with full parameter control
+- **Secure**: Context destination keeps sensitive data away from LLMs
+- **Performance**: Multiple caching strategies for different use cases
+
+#### Database Schema
+
+The `AIAgentDataSource` table includes:
+- **AgentID**: The agent using this data source
+- **Name**: Variable name (used as fallback if DestinationPath is null)
+- **SourceType**: RunView or RunQuery
+- **EntityName**, **ExtraFilter**, **OrderBy**, **FieldsToRetrieve**, **ResultType**: RunView parameters
+- **QueryName**, **CategoryPath**, **Parameters**: RunQuery parameters
+- **MaxRows**: Limit results (applies to both source types)
+- **DestinationType**: Data, Context, or Payload
+- **DestinationPath**: Nested path using dot notation (optional)
+- **ExecutionOrder**: Order to execute when multiple sources exist
+- **Status**: Active or Disabled
+- **CachePolicy**: None, PerRun, or PerAgent
+- **CacheTimeoutSeconds**: TTL for PerAgent cache
+
+**Unique Constraint**: `AgentID + Name + DestinationType + DestinationPath`
+- Allows same Name across different destinations/paths
+- Example: "ENTITIES" can exist in both Data and Payload destinations
+
 ### Payload Scoping for Sub-Agents
 
 The framework now supports narrowing the payload that sub-agents work with through the `PayloadScope` field:
@@ -987,70 +1283,309 @@ export class DecisionTreeAgent extends BaseAgentType {
 }
 ```
 
-### Flow Agent Configuration
+## Flow Agent Type - Deterministic Workflows
+
+Flow agents execute **deterministic, graph-based workflows** where the execution path is determined by boolean conditions evaluated against the payload and step results. Unlike Loop agents that rely on LLM decision-making at each step, Flow agents follow predefined paths through a directed graph.
+
+### When to Use Flow Agents
+
+Flow agents are ideal for:
+- **Predictable workflows** with well-defined decision points
+- **Approval processes** with conditional routing
+- **Data pipelines** with validation and transformation steps
+- **Hybrid workflows** combining deterministic logic with AI prompts
+- **Multi-step processes** where you need guaranteed execution order
+
+### Core Concepts
+
+#### 1. Workflow Steps (AIAgentStep)
+
+Steps are the nodes in your workflow graph. Each step represents an action to perform:
+
+```typescript
+// Three types of steps:
+{
+    Name: 'ValidateInput',
+    StepType: 'Action',           // Execute a MJ Action
+    ActionID: 'validation-action-id',
+    StartingStep: true,           // Marks this as an entry point
+    Sequence: 0,                  // For parallel starting steps
+    Status: 'Active',             // Active, Disabled, or Pending
+    TimeoutSeconds: 30            // Optional timeout
+}
+
+{
+    Name: 'AnalyzeData',
+    StepType: 'Prompt',           // Execute an AI prompt
+    PromptID: 'analysis-prompt-id',
+    Description: 'Analyze data quality and completeness'
+}
+
+{
+    Name: 'ProcessWithSubAgent',
+    StepType: 'Sub-Agent',        // Invoke another agent
+    SubAgentID: 'processing-agent-id'
+}
+```
+
+#### 2. Workflow Paths (AIAgentStepPath)
+
+Paths are the edges connecting your workflow nodes. They determine the flow:
+
+```typescript
+{
+    OriginStepID: 'step-a-id',
+    DestinationStepID: 'step-b-id',
+    Condition: 'payload.amount > 1000 && payload.approved === true',
+    Priority: 10                  // Higher priority paths evaluated first
+}
+
+// Path without condition (always valid)
+{
+    OriginStepID: 'step-a-id',
+    DestinationStepID: 'default-step-id',
+    Condition: null,              // No condition = always valid
+    Priority: 0                   // Lower priority = fallback
+}
+```
+
+#### 3. Action Input/Output Mapping
+
+**Action Input Mapping** (`ActionInputMapping`) - Maps payload values to action parameters:
+
+```typescript
+// In AIAgentStep.ActionInputMapping
+{
+    "customerId": "payload.customer.id",              // Map from payload
+    "orderDate": "static:2024-01-01",                 // Static value
+    "includeDetails": true,                           // Boolean literal
+    "maxResults": 100,                                // Numeric literal
+    "filters": {                                      // Nested object
+        "status": "payload.filters.orderStatus",
+        "region": "static:US-WEST"
+    },
+    "itemIds": "payload.order.items"                  // Can map arrays
+}
+
+// Supports nested resolution
+{
+    "searchParams": {
+        "query": "payload.searchTerm",
+        "filters": {
+            "category": "payload.category",
+            "tags": "payload.selectedTags"
+        },
+        "options": {
+            "maxResults": 50,
+            "includeMetadata": true
+        }
+    }
+}
+```
+
+**Action Output Mapping** (`ActionOutputMapping`) - Maps action results back to payload:
+
+```typescript
+// In AIAgentStep.ActionOutputMapping
+{
+    "userId": "payload.customer.id",                  // Map specific output param
+    "orderTotal": "payload.order.total",              // Nested path in payload
+    "metadata": "payload.action.lastResult",          // Arbitrary nesting
+    "*": "payload.rawResults.fullData"                // Wildcard = entire result
+}
+
+// Case-insensitive output parameter matching
+// If action returns { UserId: "123" }, it matches "userId" in mapping
+```
+
+#### 4. Prompt Result Merging
+
+When a Prompt step executes, its JSON response is **deep merged** into the payload:
+
+```typescript
+// Before prompt execution
+payload = {
+    decision: {
+        status: "pending",
+        reviewerId: "user-123"
+    },
+    metadata: { startTime: "..." }
+};
+
+// Prompt returns
+promptResponse = {
+    decision: {
+        approved: true,
+        confidence: 0.95
+    }
+};
+
+// After deep merge (preserves existing keys!)
+payload = {
+    decision: {
+        approved: true,        // NEW from prompt
+        confidence: 0.95,      // NEW from prompt
+        status: "pending",     // PRESERVED from before
+        reviewerId: "user-123" // PRESERVED from before
+    },
+    metadata: { startTime: "..." }  // PRESERVED
+};
+```
+
+**Why Deep Merge?**
+- **Preserves context** - Existing payload data isn't lost
+- **Incremental updates** - Prompts can add fields without destroying structure
+- **Composable decisions** - Multiple prompts can build up complex objects
 
-Flow agents enable deterministic workflow execution through graph-based step definitions:
+**Special Prompt Response Handling**:
+```typescript
+// If prompt response contains Chat step request
+{
+    "nextStep": { "type": "Chat" },
+    "message": "I need more information from the user",
+    "taskComplete": false
+}
+// OR
+{
+    "taskComplete": true,
+    "message": "Here's the final result..."
+}
+
+// Flow agent returns Chat step to bubble message to user
+// This allows prompts within flows to communicate with users
+```
+
+### Complete Flow Agent Example
 
 ```typescript
-// Database configuration for a Flow agent workflow
-// AIAgentStep records define the workflow nodes
-const approvalWorkflowSteps = [
+// Database configuration for a complete approval workflow
+// 1. Define the workflow steps
+const steps = [
     {
-        Name: 'CheckAmount',
+        Name: 'ValidateRequest',
         StepType: 'Action',
-        ActionID: checkAmountActionId,
+        ActionID: validateActionId,
         StartingStep: true,
-        Sequence: 0
+        Sequence: 0,
+        ActionInputMapping: JSON.stringify({
+            "requestData": "payload.request",
+            "validationRules": "payload.rules"
+        }),
+        ActionOutputMapping: JSON.stringify({
+            "isValid": "payload.validation.isValid",
+            "errors": "payload.validation.errors"
+        })
+    },
+    {
+        Name: 'CheckAmount',
+        StepType: 'Prompt',
+        PromptID: amountCheckPromptId,
+        Description: 'AI analyzes amount and risk factors'
+        // Prompt returns: { risk: "low"|"medium"|"high", reasoning: "..." }
+        // Deep merged into payload.risk and payload.reasoning
     },
     {
         Name: 'AutoApprove',
         StepType: 'Action',
         ActionID: approveActionId,
+        ActionInputMapping: JSON.stringify({
+            "requestId": "payload.request.id",
+            "approvedBy": "static:SYSTEM_AUTO"
+        }),
         ActionOutputMapping: JSON.stringify({
-            'approvalId': 'payload.approval.id',
-            'timestamp': 'payload.approval.approvedAt'
+            "approvalId": "payload.approval.id",
+            "timestamp": "payload.approval.timestamp"
         })
     },
     {
         Name: 'ManagerReview',
-        StepType: 'Prompt',
-        PromptID: managerReviewPromptId,
-        Description: 'Get manager approval decision'
+        StepType: 'Sub-Agent',
+        SubAgentID: managerReviewAgentId
+        // Sub-agent payload inherits and can modify parent payload
     },
     {
-        Name: 'NotifyRejection',
+        Name: 'NotifyUser',
         StepType: 'Action',
-        ActionID: sendNotificationActionId
+        ActionID: notificationActionId,
+        ActionInputMapping: JSON.stringify({
+            "userId": "payload.request.userId",
+            "message": "payload.approval.notificationMessage",
+            "channel": "static:email"
+        })
     }
 ];
 
-// AIAgentStepPath records define the workflow edges
-const approvalWorkflowPaths = [
+// 2. Define the workflow paths
+const paths = [
+    // From validation
     {
-        OriginStepID: checkAmountStep.ID,
-        DestinationStepID: autoApproveStep.ID,
-        Condition: 'payload.amount <= 1000',
+        OriginStepID: validateStepId,
+        DestinationStepID: checkAmountStepId,
+        Condition: 'payload.validation.isValid === true',
         Priority: 10
     },
     {
-        OriginStepID: checkAmountStep.ID,
-        DestinationStepID: managerReviewStep.ID,
-        Condition: 'payload.amount > 1000',
+        OriginStepID: validateStepId,
+        DestinationStepID: notifyUserStepId,
+        Condition: 'payload.validation.isValid === false',
+        Priority: 10
+    },
+
+    // From AI risk assessment
+    {
+        OriginStepID: checkAmountStepId,
+        DestinationStepID: autoApproveStepId,
+        Condition: 'payload.risk === "low" && payload.request.amount <= 1000',
         Priority: 10
     },
     {
-        OriginStepID: managerReviewStep.ID,
-        DestinationStepID: autoApproveStep.ID,
-        Condition: 'stepResult.approved === true',
+        OriginStepID: checkAmountStepId,
+        DestinationStepID: managerReviewStepId,
+        Condition: 'payload.risk === "medium" || payload.risk === "high"',
         Priority: 10
     },
+
+    // From manager review
     {
-        OriginStepID: managerReviewStep.ID,
-        DestinationStepID: notifyRejectionStep.ID,
-        Condition: 'stepResult.approved === false',
+        OriginStepID: managerReviewStepId,
+        DestinationStepID: autoApproveStepId,
+        Condition: 'payload.managerDecision.approved === true',
         Priority: 10
+    },
+    {
+        OriginStepID: managerReviewStepId,
+        DestinationStepID: notifyUserStepId,
+        Condition: 'payload.managerDecision.approved === false',
+        Priority: 5
+    },
+
+    // Final notification after approval
+    {
+        OriginStepID: autoApproveStepId,
+        DestinationStepID: notifyUserStepId,
+        Condition: null,  // Always execute
+        Priority: 0
     }
 ];
+
+// 3. Execute the flow agent
+const result = await runner.RunAgent({
+    agent: flowAgentEntity,
+    conversationMessages: messages,
+    contextUser: user,
+    payload: {
+        request: {
+            id: "req-123",
+            userId: "user-456",
+            amount: 5000,
+            description: "Equipment purchase"
+        },
+        rules: {
+            maxAutoApprove: 1000,
+            requiresManagerReview: true
+        }
+    }
+});
 ```
 
 ### Flow Agent Features

package/dist/AgentDataPreloader.d.ts

@@ -0,0 +1,25 @@
+import { UserInfo } from '@memberjunction/core';
+export interface PreloadedDataResult {
+    data: Record<string, unknown>;
+    context: Record<string, unknown>;
+    payload: Record<string, unknown>;
+}
+export declare class AgentDataPreloader {
+    private static _instance;
+    private _perAgentCache;
+    private _perRunCache;
+    private constructor();
+    static get Instance(): AgentDataPreloader;
+    PreloadAgentData(agentId: string, contextUser: UserInfo, runId?: string): Promise<PreloadedDataResult>;
+    clearRunCache(runId: string): void;
+    clearAgentCache(): void;
+    private loadDataSourcesForAgent;
+    private executeDataSource;
+    private executeRunView;
+    private executeRunQuery;
+    private getCachedData;
+    private cacheData;
+    private getPerAgentCacheKey;
+}
+export declare function LoadAgentDataPreloader(): AgentDataPreloader;
+//# sourceMappingURL=AgentDataPreloader.d.ts.map

package/dist/AgentDataPreloader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgentDataPreloader.d.ts","sourceRoot":"","sources":["../src/AgentDataPreloader.ts"],"names":[],"mappings":"AAcA,OAAO,EAAqE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAQnH,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AA4BD,qBAAa,kBAAkB;IAC3B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAmC;IAK3D,OAAO,CAAC,cAAc,CAAsC;IAK5D,OAAO,CAAC,YAAY,CAAgD;IAKpE,OAAO;IAOP,WAAkB,QAAQ,IAAI,kBAAkB,CAK/C;IA4BY,gBAAgB,CACzB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,QAAQ,EACrB,KAAK,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,mBAAmB,CAAC;IA8ExB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAalC,eAAe,IAAI,IAAI;YAchB,uBAAuB;YAuBvB,iBAAiB;YAsCjB,cAAc;YAwCd,eAAe;IAuC7B,OAAO,CAAC,aAAa;IA2CrB,OAAO,CAAC,SAAS;IAwCjB,OAAO,CAAC,mBAAmB;CAG9B;AAKD,wBAAgB,sBAAsB,uBAGrC"}