@memberjunction/ai-agents 2.107.0 → 2.109.0

package/README.md

@@ -268,6 +268,302 @@ The callback is invoked:
 
 ## Advanced Features
 
+### Agent Data Preloading
+
+Agents can declaratively preload reference data without requiring custom application code or action calls. Data sources are configured through the `AIAgentDataSource` entity and automatically loaded before agent execution.
+
+#### Overview
+
+Data preloading solves the common problem of agents needing access to reference data (like entity lists, configuration values, or initial state) that doesn't change during execution. Instead of:
+- Writing custom application code to load data
+- Having agents call actions to fetch data (which bloats conversation context)
+- Manually passing the same data to every agent invocation
+
+Agents can now specify data sources that are automatically loaded and injected into the appropriate destination (`data`, `context`, or `payload`).
+
+#### Three Destination Types
+
+**1. Data Destination** - For Nunjucks templates in prompts (visible to LLMs)
+```typescript
+// Configuration
+{
+  "Name": "ALL_ENTITIES",
+  "SourceType": "RunView",
+  "EntityName": "Entities",
+  "OrderBy": "Name ASC",
+  "DestinationType": "Data",
+  "DestinationPath": null  // Uses "ALL_ENTITIES" at root level
+}
+
+// Result in agent prompt:
+// params.data.ALL_ENTITIES = [{ Name: "Users", ... }, { Name: "Entities", ... }]
+
+// Prompt can use Nunjucks:
+// You have access to {{ALL_ENTITIES.length}} entities:
+// {% for entity in ALL_ENTITIES %}
+// - {{entity.Name}}: {{entity.Description}}
+// {% endfor %}
+```
+
+**2. Context Destination** - For actions only (NOT visible to LLMs)
+```typescript
+// Configuration
+{
+  "Name": "ORG_SETTINGS",
+  "SourceType": "RunView",
+  "EntityName": "Organization Settings",
+  "ExtraFilter": "OrgID='${context.organizationId}'",
+  "DestinationType": "Context",
+  "DestinationPath": "organization.settings"
+}
+
+// Result:
+// params.context.organization.settings = { apiEndpoint: "...", features: [...] }
+
+// Actions can access context, but prompts/LLMs cannot
+// This keeps API keys and sensitive configuration away from LLMs
+```
+
+**3. Payload Destination** - For agent state initialization
+```typescript
+// Configuration
+{
+  "Name": "CustomerOrders",
+  "SourceType": "RunQuery",
+  "QueryName": "Recent Orders by Customer",
+  "Parameters": JSON.stringify({ customerId: "{{context.customerId}}" }),
+  "DestinationType": "Payload",
+  "DestinationPath": "analysis.orders.recent"
+}
+
+// Result:
+// params.payload.analysis.orders.recent = [{ OrderID: "123", ... }]
+
+// Agent starts with rich initial state without caller manually loading it
+```
+
+#### Data Source Types
+
+**RunView Data Sources** - Query entities with filters
+```typescript
+{
+  "Name": "ACTIVE_MODELS",
+  "SourceType": "RunView",
+  "EntityName": "AI Models",
+  "ExtraFilter": "IsActive=1 AND Vendor='OpenAI'",
+  "OrderBy": "Priority DESC",
+  "FieldsToRetrieve": JSON.stringify(["ID", "Name", "Vendor", "MaxInputTokens"]),
+  "ResultType": "simple",  // or "entity_object"
+  "MaxRows": 100,
+  "DestinationType": "Data"
+}
+```
+
+**RunQuery Data Sources** - Execute stored queries
+```typescript
+{
+  "Name": "MONTHLY_STATS",
+  "SourceType": "RunQuery",
+  "QueryName": "Monthly Analytics",
+  "CategoryPath": "/Reports/Analytics",
+  "Parameters": JSON.stringify({
+    month: "{{context.currentMonth}}",
+    year: "{{context.currentYear}}"
+  }),
+  "DestinationType": "Payload",
+  "DestinationPath": "stats.monthly"
+}
+```
+
+#### Path Support
+
+The `DestinationPath` field supports nested paths using dot notation:
+
+```typescript
+// Simple root-level
+{
+  "Name": "ENTITIES",
+  "DestinationPath": null  // Uses "ENTITIES" at root
+}
+// Result: data.ENTITIES
+
+// Nested paths
+{
+  "Name": "ModelList",
+  "DestinationPath": "config.ai.models"
+}
+// Result: data.config.ai.models
+
+// Deep nesting
+{
+  "Name": "CustomerData",
+  "DestinationPath": "analysis.customer.profile.orders"
+}
+// Result: payload.analysis.customer.profile.orders
+```
+
+#### Caching Policies
+
+Data sources support three caching strategies:
+
+**1. None** - No caching (default)
+```typescript
+{
+  "CachePolicy": "None"
+  // Data is loaded fresh every time
+}
+```
+
+**2. PerRun** - Cache for duration of a single agent run
+```typescript
+{
+  "CachePolicy": "PerRun"
+  // Multiple data sources with same AgentID+Name share cached data within one run
+  // Cache is cleared when agent run completes
+}
+```
+
+**3. PerAgent** - Global cache with TTL
+```typescript
+{
+  "CachePolicy": "PerAgent",
+  "CacheTimeoutSeconds": 3600  // 1 hour
+  // Cached across all runs for this agent until TTL expires
+  // Good for rarely-changing reference data like entity lists
+}
+```
+
+#### Execution Control
+
+**Disable data preloading** for specific executions:
+```typescript
+const result = await runner.RunAgent({
+    agent: myAgent,
+    conversationMessages: messages,
+    contextUser: user,
+    disableDataPreloading: true  // Skip automatic data preloading
+});
+```
+
+**Caller precedence**: Caller-provided data always takes precedence over preloaded data:
+```typescript
+const result = await runner.RunAgent({
+    agent: myAgent,
+    conversationMessages: messages,
+    contextUser: user,
+    data: {
+        CUSTOM_ENTITIES: myEntities  // Overrides preloaded CUSTOM_ENTITIES
+    }
+});
+```
+
+#### Configuration Examples
+
+**Database Research Agent** - Preload entity metadata
+```typescript
+// Data source 1: All entities for reference
+{
+  "AgentID": "database-research-agent-id",
+  "Name": "ALL_ENTITIES",
+  "SourceType": "RunView",
+  "EntityName": "Entities",
+  "OrderBy": "Name ASC",
+  "FieldsToRetrieve": JSON.stringify(["ID", "Name", "SchemaName", "Description", "BaseView"]),
+  "DestinationType": "Data",
+  "ExecutionOrder": 1,
+  "Status": "Active",
+  "CachePolicy": "PerAgent",
+  "CacheTimeoutSeconds": 3600
+}
+
+// Data source 2: Schema information
+{
+  "AgentID": "database-research-agent-id",
+  "Name": "SCHEMA_INFO",
+  "SourceType": "RunView",
+  "EntityName": "Entity Fields",
+  "DestinationType": "Data",
+  "DestinationPath": "schema.fields",
+  "ExecutionOrder": 2,
+  "Status": "Active",
+  "CachePolicy": "PerAgent",
+  "CacheTimeoutSeconds": 3600
+}
+```
+
+**Customer Service Agent** - Preload customer context
+```typescript
+// Preload customer data into payload
+{
+  "AgentID": "customer-service-agent-id",
+  "Name": "CUSTOMER_PROFILE",
+  "SourceType": "RunView",
+  "EntityName": "Customers",
+  "ExtraFilter": "ID='{{context.customerId}}'",
+  "DestinationType": "Payload",
+  "DestinationPath": "customer.profile",
+  "Status": "Active",
+  "CachePolicy": "PerRun"
+}
+
+// Preload recent orders
+{
+  "AgentID": "customer-service-agent-id",
+  "Name": "RECENT_ORDERS",
+  "SourceType": "RunQuery",
+  "QueryName": "Recent Orders by Customer",
+  "Parameters": JSON.stringify({ customerId: "{{context.customerId}}", days: 30 }),
+  "DestinationType": "Payload",
+  "DestinationPath": "customer.orders",
+  "Status": "Active",
+  "CachePolicy": "PerRun"
+}
+
+// Preload organization settings (for actions)
+{
+  "AgentID": "customer-service-agent-id",
+  "Name": "ORG_CONFIG",
+  "SourceType": "RunView",
+  "EntityName": "Organization Settings",
+  "ExtraFilter": "OrgID='{{context.organizationId}}'",
+  "DestinationType": "Context",
+  "DestinationPath": "organization.config",
+  "Status": "Active",
+  "CachePolicy": "PerAgent",
+  "CacheTimeoutSeconds": 1800
+}
+```
+
+#### Benefits
+
+- **Declarative**: Configure data preloading through metadata, not code
+- **Reusable**: Same agent works across different environments
+- **Efficient**: Caching reduces redundant database queries
+- **Clean Separation**: Keeps data in appropriate destinations (data/context/payload)
+- **Flexible**: Supports both RunView and RunQuery with full parameter control
+- **Secure**: Context destination keeps sensitive data away from LLMs
+- **Performance**: Multiple caching strategies for different use cases
+
+#### Database Schema
+
+The `AIAgentDataSource` table includes:
+- **AgentID**: The agent using this data source
+- **Name**: Variable name (used as fallback if DestinationPath is null)
+- **SourceType**: RunView or RunQuery
+- **EntityName**, **ExtraFilter**, **OrderBy**, **FieldsToRetrieve**, **ResultType**: RunView parameters
+- **QueryName**, **CategoryPath**, **Parameters**: RunQuery parameters
+- **MaxRows**: Limit results (applies to both source types)
+- **DestinationType**: Data, Context, or Payload
+- **DestinationPath**: Nested path using dot notation (optional)
+- **ExecutionOrder**: Order to execute when multiple sources exist
+- **Status**: Active or Disabled
+- **CachePolicy**: None, PerRun, or PerAgent
+- **CacheTimeoutSeconds**: TTL for PerAgent cache
+
+**Unique Constraint**: `AgentID + Name + DestinationType + DestinationPath`
+- Allows same Name across different destinations/paths
+- Example: "ENTITIES" can exist in both Data and Payload destinations
+
 ### Payload Scoping for Sub-Agents
 
 The framework now supports narrowing the payload that sub-agents work with through the `PayloadScope` field:
@@ -498,10 +794,164 @@ const result = await agent.ExecutePrompt({
 ### Context Management
 Agents automatically manage conversation context:
 - Maintains message history across steps
+- **Intelligent message expiration** - Automatically compacts or removes old action results
 - Compresses context when approaching token limits
 - Handles placeholder replacement in prompts
 - Preserves important context during compression
 
+### Message Expiration and Compaction
+
+The framework provides sophisticated message lifecycle management to prevent context bloat from large action results:
+
+**Per-Action Configuration** (in `AIAgentAction` table):
+- `ResultExpirationTurns`: Number of turns before message expires (e.g., 2)
+- `ResultExpirationMode`: 'None' | 'Remove' | 'Compact'
+- `CompactMode`: 'First N Chars' | 'AI Summary'
+- `CompactLength`: Character limit for 'First N Chars' mode
+- `CompactPromptID`: Custom AI prompt for 'AI Summary' mode
+
+**How It Works**:
+```typescript
+// Configure a Google Search action to compact results after 2 turns
+await agentAction.Save({
+    ResultExpirationTurns: 2,
+    ResultExpirationMode: 'Compact',
+    CompactMode: 'First N Chars',
+    CompactLength: 500
+});
+
+// Turn 1: Action returns 10,000 char search results
+// Turn 2: Results still in conversation (turn 1, limit 2)
+// Turn 3: Results still in conversation (turn 2, limit 2)
+// Turn 4: Results compacted to 500 chars (turn 3 > limit 2)
+//         Original content preserved in metadata for expansion
+```
+
+**Compaction Modes**:
+1. **First N Chars**: Fast truncation with annotation
+   ```
+   First 500 chars of result...
+
+   [Compacted: showing first 500 of 10000 characters. Agent can request expansion if needed.]
+   ```
+
+2. **AI Summary**: Intelligent LLM-based summarization
+   ```
+   [AI Summary of 10000 chars. Agent can request full expansion if needed.]
+
+   Search found 47 results for "MemberJunction". Top results include...
+   ```
+
+**Message Expansion**:
+Agents can restore compacted messages when needed:
+```typescript
+// In agent's JSON response
+{
+    "taskComplete": false,
+    "nextStep": {
+        "type": "Retry",
+        "messageIndex": 5,  // Index of compacted message
+        "reason": "Need full search results to answer user's question about item #47"
+    }
+}
+```
+
+**Runtime Override**:
+Test different expiration strategies without modifying database:
+```typescript
+const result = await runner.RunAgent({
+    agent: myAgent,
+    conversationMessages: messages,
+    contextUser: user,
+    messageExpirationOverride: {
+        expirationTurns: 1,
+        expirationMode: 'Compact',
+        compactMode: 'First N Chars',
+        compactLength: 200,
+        preserveOriginalContent: true
+    }
+});
+```
+
+**Lifecycle Monitoring**:
+Track message compaction for debugging and token savings analysis:
+```typescript
+const result = await runner.RunAgent({
+    agent: myAgent,
+    conversationMessages: messages,
+    contextUser: user,
+    onMessageLifecycle: (event) => {
+        console.log(`[Turn ${event.turn}] ${event.type}: ${event.reason}`);
+        if (event.tokensSaved) {
+            console.log(`  Tokens saved: ${event.tokensSaved}`);
+        }
+    }
+});
+// Output:
+// [Turn 3] message-compacted: Compacted using First N Chars (saved 2375 tokens)
+// [Turn 5] message-removed: Removed due to expiration
+```
+
+**Prompt Lookup Hierarchy**:
+For AI Summary mode, prompts are resolved in this order:
+1. Runtime override (`messageExpirationOverride.compactPromptId`)
+2. Agent action configuration (`AIAgentAction.CompactPromptID`)
+3. Action default (`Action.DefaultCompactPromptID`)
+4. System default ("Compact Agent Message" prompt)
+
+**Benefits**:
+- **Addresses Large Action Results**: Automatically handles the most common cause of context bloat
+- **Configurable Per-Action**: Different expiration strategies for different action types
+- **Non-Destructive**: Original content preserved in metadata for on-demand expansion
+- **Token Savings**: Reduces context window usage by 70-95% for large results
+- **Agent-Aware**: Agents can detect compacted messages and request full expansion when needed
+
+### Context Length Recovery
+
+When a prompt execution fails due to context length overflow (even after model failover), BaseAgent provides **one-time automatic recovery** instead of immediately terminating. This gives the agent an opportunity to adapt its approach.
+
+**How It Works**:
+1. **Prompt fails with ContextLengthExceeded** → Detected as fatal error
+2. **First occurrence**: Recovery is attempted automatically (once per run)
+3. **Last user message is trimmed** using smart strategies:
+   - JSON arrays: Keeps first 10 items with truncation notice
+   - CSV data: Keeps header + first 10 rows
+   - Plain text: Keeps first 1000 characters
+4. **Agent receives clear guidance** explaining what happened and recommended actions
+5. **Agent gets Retry step** to choose alternative approach (e.g., more specific filters, batch requests)
+6. **If recovery fails again**: Normal fatal error handling (agent terminates)
+
+**Example Recovery Message**:
+```
+⚠️ CONTEXT OVERFLOW RECOVERY ⚠️
+
+The previous step returned a result that exceeded the context window (147,532 characters truncated).
+
+Here is a PARTIAL result from the previous action:
+---
+[First 10 items from JSON array...]
+... (487 more items truncated due to context length)
+---
+
+❗ THE ABOVE IS INCOMPLETE - the full result was too large for the context window.
+
+RECOMMENDED ACTIONS:
+1. Use a different action with more specific filters to get smaller result sets
+2. Request data in batches or pages instead of all at once
+3. Ask the user to clarify scope to narrow the query
+4. If you need the full data, acknowledge the limitation and ask the user how to proceed
+
+Please choose an alternative approach to complete your task.
+```
+
+**Benefits**:
+- **Resilient**: Agents can adapt instead of failing immediately
+- **Informative**: Clear explanation of what went wrong and how to recover
+- **Safe**: ONE-TIME recovery prevents infinite loops
+- **Smart**: Preserves data structure when possible (JSON, CSV)
+
+This feature is particularly useful when agents call actions that can return very large datasets (e.g., "Get Entity List" without filters).
+
 ### Action Integration
 ```typescript
 // In agent type's DetermineNextStep

package/dist/AgentDataPreloader.d.ts

@@ -0,0 +1,25 @@
+import { UserInfo } from '@memberjunction/core';
+export interface PreloadedDataResult {
+    data: Record<string, unknown>;
+    context: Record<string, unknown>;
+    payload: Record<string, unknown>;
+}
+export declare class AgentDataPreloader {
+    private static _instance;
+    private _perAgentCache;
+    private _perRunCache;
+    private constructor();
+    static get Instance(): AgentDataPreloader;
+    PreloadAgentData(agentId: string, contextUser: UserInfo, runId?: string): Promise<PreloadedDataResult>;
+    clearRunCache(runId: string): void;
+    clearAgentCache(): void;
+    private loadDataSourcesForAgent;
+    private executeDataSource;
+    private executeRunView;
+    private executeRunQuery;
+    private getCachedData;
+    private cacheData;
+    private getPerAgentCacheKey;
+}
+export declare function LoadAgentDataPreloader(): AgentDataPreloader;
+//# sourceMappingURL=AgentDataPreloader.d.ts.map

package/dist/AgentDataPreloader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgentDataPreloader.d.ts","sourceRoot":"","sources":["../src/AgentDataPreloader.ts"],"names":[],"mappings":"AAcA,OAAO,EAAqE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAQnH,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AA4BD,qBAAa,kBAAkB;IAC3B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAmC;IAK3D,OAAO,CAAC,cAAc,CAAsC;IAK5D,OAAO,CAAC,YAAY,CAAgD;IAKpE,OAAO;IAOP,WAAkB,QAAQ,IAAI,kBAAkB,CAK/C;IA4BY,gBAAgB,CACzB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,QAAQ,EACrB,KAAK,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,mBAAmB,CAAC;IA8ExB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAalC,eAAe,IAAI,IAAI;YAchB,uBAAuB;YAuBvB,iBAAiB;YAsCjB,cAAc;YAwCd,eAAe;IAuC7B,OAAO,CAAC,aAAa;IA2CrB,OAAO,CAAC,SAAS;IAwCjB,OAAO,CAAC,mBAAmB;CAG9B;AAKD,wBAAgB,sBAAsB,uBAGrC"}