@memberjunction/ai-agents 3.2.0 → 3.4.0

package/dist/memory-manager-agent.js

@@ -1,61 +1,181 @@
 "use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MemoryManagerAgent = void 0;
 const base_agent_1 = require("./base-agent");
+const global_1 = require("@memberjunction/global");
 const core_1 = require("@memberjunction/core");
 const ai_prompts_1 = require("@memberjunction/ai-prompts");
 const ai_core_plus_1 = require("@memberjunction/ai-core-plus");
 const aiengine_1 = require("@memberjunction/aiengine");
-class MemoryManagerAgent extends base_agent_1.BaseAgent {
+const EXTRACTION_CONFIG = {
+    maxNotesPerRun: 1000,
+    maxNotesPerConversation: 1000,
+    maxNotesPerMessage: 1000,
+    minConfidenceThreshold: 80,
+    minContentLength: 10,
+    cooldownHours: 24
+};
+let MemoryManagerAgent = class MemoryManagerAgent extends base_agent_1.BaseAgent {
+    constructor() {
+        super(...arguments);
+        this._verbose = false;
+        this._agentRunID = null;
+        this._stepCounter = 0;
+        this._contextUser = null;
+    }
+    async CreateRunStep(stepType, stepName, inputData, targetId) {
+        if (!this._agentRunID || !this._contextUser) {
+            return null;
+        }
+        try {
+            const md = new core_1.Metadata();
+            const step = await md.GetEntityObject('MJ: AI Agent Run Steps', this._contextUser);
+            step.AgentRunID = this._agentRunID;
+            step.StepNumber = ++this._stepCounter;
+            step.StepType = stepType;
+            step.StepName = stepName;
+            step.Status = 'Running';
+            step.StartedAt = new Date();
+            if (targetId) {
+                step.TargetID = targetId;
+            }
+            if (inputData) {
+                step.InputData = JSON.stringify(inputData);
+            }
+            if (await step.Save()) {
+                return step;
+            }
+            else {
+                (0, core_1.LogError)(`Memory Manager: Failed to create run step: ${JSON.stringify(step.LatestResult)}`);
+                return null;
+            }
+        }
+        catch (error) {
+            (0, core_1.LogError)('Memory Manager: Exception creating run step:', error);
+            return null;
+        }
+    }
+    async FinalizeRunStep(step, success, outputData, targetLogId, errorMessage) {
+        if (!step) {
+            return;
+        }
+        try {
+            step.Status = success ? 'Completed' : 'Failed';
+            step.CompletedAt = new Date();
+            step.Success = success;
+            if (errorMessage) {
+                step.ErrorMessage = errorMessage;
+            }
+            if (targetLogId) {
+                step.TargetLogID = targetLogId;
+            }
+            if (outputData) {
+                step.OutputData = JSON.stringify(outputData);
+            }
+            if (!await step.Save()) {
+                (0, core_1.LogError)(`Memory Manager: Failed to finalize run step: ${JSON.stringify(step.LatestResult)}`);
+            }
+        }
+        catch (error) {
+            (0, core_1.LogError)('Memory Manager: Exception finalizing run step:', error);
+        }
+    }
     async GetLastRunTime(agentId, contextUser) {
         const rv = new core_1.RunView();
         const result = await rv.RunView({
             EntityName: 'MJ: AI Agent Runs',
-            ExtraFilter: `AgentID='${agentId}' AND Status='Success'`,
+            ExtraFilter: `AgentID='${agentId}' AND Status='Completed'`,
             OrderBy: 'StartedAt DESC',
             MaxRows: 1,
             ResultType: 'entity_object'
         }, contextUser);
         if (result.Success && result.Results && result.Results.length > 0) {
-            return result.Results[0].StartedAt;
+            const startedAt = result.Results[0].StartedAt;
+            return startedAt instanceof Date ? startedAt : new Date(startedAt);
         }
         return null;
     }
     async LoadAgentsUsingMemory(contextUser) {
-        const filteredAgents = aiengine_1.AIEngine.Instance.Agents.filter(a => a.Status === 'Active' && (a.InjectNotes || a.InjectExamples));
+        const allAgents = aiengine_1.AIEngine.Instance.Agents;
+        const filteredAgents = allAgents.filter(a => a.Status === 'Active' && (a.InjectNotes || a.InjectExamples));
+        if (this._verbose) {
+            (0, core_1.LogStatus)(`Memory Manager: AIEngine has ${allAgents.length} total agents cached`);
+            if (filteredAgents.length > 0) {
+                const agentNames = filteredAgents.map(a => `${a.Name} (InjectNotes=${a.InjectNotes}, InjectExamples=${a.InjectExamples})`).join(', ');
+                (0, core_1.LogStatus)(`Memory Manager: Agents with memory enabled: ${agentNames}`);
+            }
+            const sage = allAgents.find(a => a.Name === 'Sage');
+            if (sage) {
+                (0, core_1.LogStatus)(`Memory Manager: Sage agent - ID=${sage.ID}, InjectNotes=${sage.InjectNotes}, InjectExamples=${sage.InjectExamples}, Status=${sage.Status}`);
+            }
+        }
+        const sage = allAgents.find(a => a.Name === 'Sage');
+        if (!sage) {
+            (0, core_1.LogStatus)(`Memory Manager: WARNING - Sage agent not found in AIEngine cache!`);
+        }
         return filteredAgents;
     }
-    async LoadHighQualityConversationDetails(since, agentsUsingMemory, contextUser) {
+    async LoadConversationsWithNewActivity(since, agentsUsingMemory, contextUser) {
         if (agentsUsingMemory.length === 0) {
             (0, core_1.LogStatus)('Memory Manager: No agents have memory injection enabled - skipping');
             return [];
         }
-        const rv = new core_1.RunView();
-        const sinceFilter = since ? `AND cd.__mj_CreatedAt >= '${since.toISOString()}'` : '';
-        const agentIdFilter = agentsUsingMemory.map(a => `'${a.ID}'`).join(',');
-        const filter = `
-            Status = 'Complete'
-            AND Role = 'AI'
-            ${sinceFilter}
-            AND EXISTS (
-                SELECT 1 FROM ConversationDetailRating cdr
-                WHERE cdr.ConversationDetailID = cd.ID
-                AND cdr.Rating >= 8
-            )
-            AND EXISTS (
-                SELECT 1 FROM AIAgentRun ar
-                WHERE ar.ConversationID = cd.ConversationID
-                AND ar.AgentID IN (${agentIdFilter})
-            )
-        `.trim().replace(/\s+/g, ' ');
-        const result = await rv.RunView({
-            EntityName: 'Conversation Details',
-            ExtraFilter: filter,
-            OrderBy: '__mj_CreatedAt DESC',
-            MaxRows: 100,
-            ResultType: 'entity_object'
+        const agentIds = agentsUsingMemory.map(a => `'${a.ID}'`).join(',');
+        const rq = new core_1.RunQuery();
+        const result = await rq.RunQuery({
+            QueryName: 'GetConversationsForMemoryManager',
+            CategoryPath: '/MJ/AI/Agents/',
+            Parameters: {
+                since: since?.toISOString() || null,
+                agentIds: agentIds
+            },
+            MaxRows: 1000
         }, contextUser);
-        return result.Success ? (result.Results || []) : [];
+        if (!result.Success || !result.Results || result.Results.length === 0) {
+            (0, core_1.LogStatus)('Memory Manager: No conversations with new activity found');
+            return [];
+        }
+        const conversations = [];
+        for (const row of result.Results) {
+            let messages = [];
+            if (row.MessagesJSON) {
+                try {
+                    const parsedMessages = JSON.parse(row.MessagesJSON);
+                    messages = parsedMessages.map(m => ({
+                        id: m.id,
+                        role: m.role,
+                        message: m.message,
+                        createdAt: new Date(m.createdAt),
+                        rating: m.rating,
+                        ratingComment: m.ratingComment
+                    }));
+                }
+                catch (e) {
+                    (0, core_1.LogError)(`Memory Manager: Failed to parse MessagesJSON for conversation ${row.ConversationID}:`, e);
+                    continue;
+                }
+            }
+            conversations.push({
+                conversationId: row.ConversationID,
+                userId: row.UserID,
+                agentRunId: row.AgentRunID,
+                messages: messages,
+                hasPositiveRating: row.HasPositiveRating === 1,
+                hasNegativeRating: row.HasNegativeRating === 1,
+                isUnrated: row.IsUnrated === 1
+            });
+        }
+        const totalMessages = conversations.reduce((sum, c) => sum + c.messages.length, 0);
+        if (this._verbose) {
+            (0, core_1.LogStatus)(`Memory Manager: Loaded ${conversations.length} conversations with ${totalMessages} total messages (positive: ${conversations.filter(c => c.hasPositiveRating).length}, negative: ${conversations.filter(c => c.hasNegativeRating).length}, unrated: ${conversations.filter(c => c.isUnrated).length})`);
+        }
+        return conversations;
     }
     async LoadHighValueAgentRuns(since, contextUser) {
         const rv = new core_1.RunView();
@@ -63,14 +183,14 @@ class MemoryManagerAgent extends base_agent_1.BaseAgent {
         const filter = `
             ID IN (
                 SELECT DISTINCT ar.ID
-                FROM AIAgentRun ar
-                INNER JOIN Conversation c ON ar.ConversationID = c.ID
-                INNER JOIN ConversationDetail cd ON cd.ConversationID = c.ID
-                INNER JOIN ConversationDetailArtifact cda ON cda.ConversationDetailID = cd.ID
-                INNER JOIN ArtifactVersion av ON av.ID = cda.ArtifactVersionID
+                FROM __mj.AIAgentRun ar
+                INNER JOIN __mj.Conversation c ON ar.ConversationID = c.ID
+                INNER JOIN __mj.ConversationDetail cd ON cd.ConversationID = c.ID
+                INNER JOIN __mj.ConversationDetailArtifact cda ON cda.ConversationDetailID = cd.ID
+                INNER JOIN __mj.ArtifactVersion av ON av.ID = cda.ArtifactVersionID
                 WHERE EXISTS (
                     SELECT 1
-                    FROM ArtifactUse au
+                    FROM __mj.ArtifactUse au
                     WHERE au.ArtifactVersionID = av.ID
                     ${sinceFilter}
                     GROUP BY au.ArtifactVersionID
@@ -90,25 +210,26 @@ class MemoryManagerAgent extends base_agent_1.BaseAgent {
         }, contextUser);
         return result.Success ? (result.Results || []) : [];
     }
-    async ExtractNotes(conversationDetails, contextUser) {
-        if (conversationDetails.length === 0) {
+    async ExtractNotesFromConversations(conversations, contextUser) {
+        if (conversations.length === 0) {
             return [];
         }
         const allNotes = aiengine_1.AIEngine.Instance.AgentNotes;
         const existingNotes = allNotes.filter(n => n.Status === 'Active');
-        const detailsByConversation = new Map();
-        for (const detail of conversationDetails) {
-            const existing = detailsByConversation.get(detail.ConversationID) || [];
-            existing.push(detail);
-            detailsByConversation.set(detail.ConversationID, existing);
-        }
-        const conversationThreads = Array.from(detailsByConversation.entries()).map(([convId, details]) => ({
-            conversationId: convId,
-            messages: details.map(d => ({
-                id: d.ID,
-                role: d.Role,
-                message: d.Message,
-                createdAt: d.__mj_CreatedAt
+        const conversationThreads = conversations.map(conv => ({
+            conversationId: conv.conversationId,
+            userId: conv.userId,
+            agentRunId: conv.agentRunId,
+            hasPositiveRating: conv.hasPositiveRating,
+            hasNegativeRating: conv.hasNegativeRating,
+            isUnrated: conv.isUnrated,
+            messages: conv.messages.map(msg => ({
+                id: msg.id,
+                role: msg.role,
+                message: msg.message,
+                createdAt: msg.createdAt,
+                rating: msg.rating,
+                ratingComment: msg.ratingComment
             }))
         }));
         const promptData = {
@@ -122,22 +243,208 @@ class MemoryManagerAgent extends base_agent_1.BaseAgent {
                 companyId: n.CompanyID
             }))
         };
+        return this.executeNoteExtraction(promptData, existingNotes, contextUser);
+    }
+    async executeNoteExtraction(promptData, existingNotes, contextUser) {
+        const conversationThreads = promptData.conversationThreads;
         const prompt = aiengine_1.AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Extract Notes' && p.Category === 'MJ: System');
         if (!prompt) {
             (0, core_1.LogError)('Memory Manager note extraction prompt not found');
+            if (this._verbose) {
+                (0, core_1.LogStatus)(`Memory Manager: Available prompts in MJ: System category: ${aiengine_1.AIEngine.Instance.Prompts.filter(p => p.Category === 'MJ: System').map(p => p.Name).join(', ')}`);
+            }
             return [];
         }
+        if (this._verbose) {
+            (0, core_1.LogStatus)(`Memory Manager: Found extraction prompt "${prompt.Name}" (ID: ${prompt.ID})`);
+            (0, core_1.LogStatus)(`Memory Manager: Sending ${conversationThreads.length} conversation threads with ${conversationThreads.reduce((sum, t) => sum + t.messages.length, 0)} total messages for note extraction`);
+            if (conversationThreads.length > 0) {
+                const firstThread = conversationThreads[0];
+                (0, core_1.LogStatus)(`Memory Manager: Sample thread (conv ${firstThread.conversationId}): ${firstThread.messages.map(m => `[${m.role}] ${m.message?.substring(0, 80)}...`).join(' | ')}`);
+            }
+        }
+        const step3 = await this.CreateRunStep('Prompt', 'Extract Notes from Conversations', {
+            conversationCount: conversationThreads.length,
+            messageCount: conversationThreads.reduce((sum, t) => sum + t.messages.length, 0),
+            existingNoteCount: existingNotes.length
+        }, prompt.ID);
         const runner = new ai_prompts_1.AIPromptRunner();
         const params = new ai_core_plus_1.AIPromptParams();
         params.prompt = prompt;
         params.data = promptData;
         params.contextUser = contextUser;
         const result = await runner.ExecutePrompt(params);
+        await this.FinalizeRunStep(step3, result.success, {
+            success: result.success,
+            rawNoteCount: result.result && typeof result.result !== 'string' ? (result.result.notes?.length || 0) : 0
+        }, result.promptRun?.ID, result.errorMessage || undefined);
+        if (this._verbose) {
+            (0, core_1.LogStatus)(`Memory Manager: Extraction prompt result - success: ${result.success}, hasResult: ${!!result.result}`);
+            if (result.errorMessage) {
+                (0, core_1.LogStatus)(`Memory Manager: Extraction error: ${result.errorMessage}`);
+            }
+            if (result.result) {
+                (0, core_1.LogStatus)(`Memory Manager: Raw extraction result: ${JSON.stringify(result.result)}`);
+            }
+        }
         if (!result.success || !result.result) {
             (0, core_1.LogError)('Failed to extract notes:', result.errorMessage);
             return [];
         }
-        return (result.result.notes || []).filter(n => n.confidence >= 70);
+        let parsedResult;
+        if (typeof result.result === 'string') {
+            try {
+                parsedResult = JSON.parse(result.result);
+                if (this._verbose) {
+                    (0, core_1.LogStatus)(`Memory Manager: Parsed string result into object with ${parsedResult.notes?.length || 0} notes`);
+                }
+            }
+            catch (e) {
+                (0, core_1.LogError)('Failed to parse extraction result as JSON:', e);
+                return [];
+            }
+        }
+        else {
+            parsedResult = result.result;
+        }
+        if (this._verbose && parsedResult.notes) {
+            (0, core_1.LogStatus)(`Memory Manager: LLM returned ${parsedResult.notes.length} raw notes before filtering`);
+            for (const note of parsedResult.notes) {
+                (0, core_1.LogStatus)(`Memory Manager: Raw note: [${note.type}] "${note.content}" (confidence: ${note.confidence})`);
+            }
+        }
+        const candidateNotes = (parsedResult.notes || []).filter(n => n.confidence >= EXTRACTION_CONFIG.minConfidenceThreshold &&
+            n.content && n.content.length >= EXTRACTION_CONFIG.minContentLength);
+        if (candidateNotes.length === 0) {
+            if (this._verbose) {
+                (0, core_1.LogStatus)('Memory Manager: No candidates passed confidence/length thresholds');
+            }
+            return [];
+        }
+        if (this._verbose) {
+            (0, core_1.LogStatus)(`Memory Manager: ${candidateNotes.length} candidates passed confidence threshold (>=${EXTRACTION_CONFIG.minConfidenceThreshold})`);
+            for (const note of candidateNotes) {
+                (0, core_1.LogStatus)(`Memory Manager: Extracted note: [${note.type}] "${note.content}" (confidence: ${note.confidence})`);
+            }
+        }
+        const notesByMessage = new Map();
+        const noteCountByConversation = new Map();
+        for (const note of candidateNotes) {
+            const msgId = note.sourceConversationDetailId || 'unknown';
+            const convId = note.sourceConversationId || 'unknown';
+            const existingForMsg = notesByMessage.get(msgId) || [];
+            const convCount = noteCountByConversation.get(convId) || 0;
+            if (existingForMsg.length >= EXTRACTION_CONFIG.maxNotesPerMessage) {
+                if (this._verbose) {
+                    (0, core_1.LogStatus)(`Memory Manager: Skipping note (max ${EXTRACTION_CONFIG.maxNotesPerMessage} per message reached)`);
+                }
+            }
+            else if (convCount >= EXTRACTION_CONFIG.maxNotesPerConversation) {
+                if (this._verbose) {
+                    (0, core_1.LogStatus)(`Memory Manager: Skipping note (max ${EXTRACTION_CONFIG.maxNotesPerConversation} per conversation reached)`);
+                }
+            }
+            else {
+                existingForMsg.push(note);
+                notesByMessage.set(msgId, existingForMsg);
+                noteCountByConversation.set(convId, convCount + 1);
+            }
+        }
+        const sparseCandidates = Array.from(notesByMessage.values()).flat();
+        if (this._verbose) {
+            (0, core_1.LogStatus)(`Memory Manager: ${sparseCandidates.length} candidates after sparsity filter`);
+        }
+        const step4 = await this.CreateRunStep('Decision', 'Deduplicate Note Candidates', {
+            candidateCount: sparseCandidates.length,
+            existingNoteCount: existingNotes.length
+        });
+        const approvedNotes = [];
+        let dedupeRejectedCount = 0;
+        let dedupeLlmCallCount = 0;
+        const dedupePrompt = aiengine_1.AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Deduplicate Note' && p.Category === 'MJ: System');
+        for (const candidate of sparseCandidates) {
+            if (approvedNotes.length >= EXTRACTION_CONFIG.maxNotesPerRun) {
+                if (this._verbose) {
+                    (0, core_1.LogStatus)(`Memory Manager: Stopping - max notes per run (${EXTRACTION_CONFIG.maxNotesPerRun}) reached`);
+                }
+                break;
+            }
+            const normalizedContent = candidate.content.toLowerCase().trim();
+            const exactMatch = existingNotes.find(n => n.Note && n.Note.toLowerCase().trim() === normalizedContent);
+            if (exactMatch) {
+                dedupeRejectedCount++;
+                if (this._verbose) {
+                    (0, core_1.LogStatus)(`Memory Manager: Skipping exact duplicate: "${candidate.content.substring(0, 50)}..."`);
+                }
+                continue;
+            }
+            const similarNotes = await aiengine_1.AIEngine.Instance.FindSimilarAgentNotes(candidate.content, candidate.agentId, candidate.userId, candidate.companyId, 10, 0.85);
+            if (dedupePrompt && similarNotes.length > 0) {
+                const dedupeParams = new ai_core_plus_1.AIPromptParams();
+                dedupeParams.prompt = dedupePrompt;
+                dedupeParams.data = {
+                    candidateNote: candidate,
+                    similarNotes: similarNotes.map(s => ({
+                        type: s.note.Type,
+                        content: s.note.Note,
+                        agentId: s.note.AgentID,
+                        userId: s.note.UserID,
+                        companyId: s.note.CompanyID,
+                        similarity: s.similarity
+                    }))
+                };
+                dedupeParams.contextUser = contextUser;
+                const dedupeResult = await runner.ExecutePrompt(dedupeParams);
+                dedupeLlmCallCount++;
+                if (dedupeResult.success && dedupeResult.result && dedupeResult.result.shouldAdd) {
+                    approvedNotes.push(candidate);
+                    if (this._verbose) {
+                        (0, core_1.LogStatus)(`Memory Manager: Approved note - ${dedupeResult.result.reason}`);
+                    }
+                }
+                else {
+                    dedupeRejectedCount++;
+                    if (this._verbose) {
+                        (0, core_1.LogStatus)(`Memory Manager: Skipped duplicate note - ${dedupeResult.result?.reason || 'too similar to existing notes'}`);
+                    }
+                }
+            }
+            else {
+                approvedNotes.push(candidate);
+                if (this._verbose) {
+                    (0, core_1.LogStatus)(`Memory Manager: Approved note (no similar notes found): "${candidate.content.substring(0, 50)}..."`);
+                }
+            }
+        }
+        await this.FinalizeRunStep(step4, true, {
+            approvedCount: approvedNotes.length,
+            rejectedCount: dedupeRejectedCount,
+            llmCallCount: dedupeLlmCallCount
+        });
+        if (this._verbose) {
+            (0, core_1.LogStatus)(`Memory Manager: Final approved notes: ${approvedNotes.length}`);
+        }
+        const conversationContext = new Map();
+        for (const thread of conversationThreads) {
+            conversationContext.set(thread.conversationId, {
+                userId: thread.userId || null,
+                agentRunId: thread.agentRunId || null
+            });
+        }
+        for (const note of approvedNotes) {
+            if (note.sourceConversationId) {
+                const ctx = conversationContext.get(note.sourceConversationId);
+                if (ctx) {
+                    if (!note.userId && ctx.userId) {
+                        note.userId = ctx.userId;
+                    }
+                    if (!note.sourceAgentRunId && ctx.agentRunId) {
+                        note.sourceAgentRunId = ctx.agentRunId;
+                    }
+                }
+            }
+        }
+        return approvedNotes;
     }
     async ExtractExamples(conversationDetails, contextUser) {
         if (conversationDetails.length === 0) {
@@ -155,6 +462,9 @@ class MemoryManagerAgent extends base_agent_1.BaseAgent {
             (0, core_1.LogError)('Memory Manager example extraction prompt not found');
             return [];
         }
+        const step5 = await this.CreateRunStep('Prompt', 'Extract Examples from Conversations', {
+            qaPairCount: qaPairs.length
+        }, extractPrompt.ID);
         const runner = new ai_prompts_1.AIPromptRunner();
         const extractParams = new ai_core_plus_1.AIPromptParams();
         extractParams.prompt = extractPrompt;
@@ -162,15 +472,40 @@ class MemoryManagerAgent extends base_agent_1.BaseAgent {
         extractParams.contextUser = contextUser;
         const extractResult = await runner.ExecutePrompt(extractParams);
         if (!extractResult.success || !extractResult.result) {
+            await this.FinalizeRunStep(step5, false, {
+                success: false
+            }, extractResult.promptRun?.ID, extractResult.errorMessage || undefined);
             (0, core_1.LogError)('Failed to extract examples:', extractResult.errorMessage);
             return [];
         }
-        const candidateExamples = (extractResult.result.examples || [])
+        let parsedResult;
+        if (typeof extractResult.result === 'string') {
+            try {
+                parsedResult = JSON.parse(extractResult.result);
+            }
+            catch (e) {
+                (0, core_1.LogError)('Failed to parse example extraction result as JSON:', e);
+                return [];
+            }
+        }
+        else {
+            parsedResult = extractResult.result;
+        }
+        const candidateExamples = (parsedResult.examples || [])
             .filter(e => e.successScore >= 70 && e.confidence >= 70);
+        await this.FinalizeRunStep(step5, true, {
+            rawExampleCount: parsedResult.examples?.length || 0,
+            candidateCount: candidateExamples.length
+        }, extractResult.promptRun?.ID);
         if (candidateExamples.length === 0) {
             return [];
         }
+        const step6 = await this.CreateRunStep('Decision', 'Deduplicate Example Candidates', {
+            candidateCount: candidateExamples.length
+        });
         const approvedExamples = [];
+        let exampleDedupeRejectedCount = 0;
+        let exampleDedupeLlmCallCount = 0;
         for (const candidate of candidateExamples) {
             const similarExamples = await aiengine_1.AIEngine.Instance.FindSimilarAgentExamples(candidate.exampleInput, candidate.agentId, candidate.userId, candidate.companyId, 5, 0.7);
             const dedupePrompt = aiengine_1.AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Deduplicate Example' && p.Category === 'MJ: System');
@@ -188,65 +523,182 @@ class MemoryManagerAgent extends base_agent_1.BaseAgent {
                 };
                 dedupeParams.contextUser = contextUser;
                 const dedupeResult = await runner.ExecutePrompt(dedupeParams);
+                exampleDedupeLlmCallCount++;
                 if (dedupeResult.success && dedupeResult.result && dedupeResult.result.shouldAdd) {
                     approvedExamples.push(candidate);
-                    (0, core_1.LogStatus)(`Memory Manager: Approved example - ${dedupeResult.result.reason}`);
+                    if (this._verbose) {
+                        (0, core_1.LogStatus)(`Memory Manager: Approved example - ${dedupeResult.result.reason}`);
+                    }
                 }
                 else {
-                    (0, core_1.LogStatus)(`Memory Manager: Skipped duplicate example - ${dedupeResult.result?.reason || 'too similar'}`);
+                    exampleDedupeRejectedCount++;
+                    if (this._verbose) {
+                        (0, core_1.LogStatus)(`Memory Manager: Skipped duplicate example - ${dedupeResult.result?.reason || 'too similar'}`);
+                    }
                 }
             }
             else {
                 approvedExamples.push(candidate);
             }
         }
+        await this.FinalizeRunStep(step6, true, {
+            approvedCount: approvedExamples.length,
+            rejectedCount: exampleDedupeRejectedCount,
+            llmCallCount: exampleDedupeLlmCallCount
+        });
         return approvedExamples;
     }
+    isValidUUID(str) {
+        const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+        return uuidRegex.test(str);
+    }
     async CreateNoteRecords(extractedNotes, contextUser) {
+        const step7 = await this.CreateRunStep('Decision', 'Create Note Records', {
+            noteCount: extractedNotes.length
+        });
         let created = 0;
+        let merged = 0;
+        let failed = 0;
         const md = new core_1.Metadata();
+        const rv = new core_1.RunView();
+        const runCache = new Map();
+        const aiNoteTypeId = aiengine_1.AIEngine.Instance.AgenteNoteTypeIDByName('AI');
+        if (!aiNoteTypeId) {
+            (0, core_1.LogError)('Memory Manager: Could not find "AI" note type - cannot create notes');
+            await this.FinalizeRunStep(step7, false, {
+                created: 0,
+                merged: 0,
+                failed: extractedNotes.length
+            }, undefined, 'Could not find "AI" note type');
+            return 0;
+        }
         for (const extracted of extractedNotes) {
             try {
+                let sourceRun = null;
+                if (extracted.sourceAgentRunId) {
+                    if (!runCache.has(extracted.sourceAgentRunId)) {
+                        const runResult = await rv.RunView({
+                            EntityName: 'MJ: AI Agent Runs',
+                            ExtraFilter: `ID='${extracted.sourceAgentRunId}'`,
+                            MaxRows: 1,
+                            ResultType: 'entity_object'
+                        }, contextUser);
+                        runCache.set(extracted.sourceAgentRunId, runResult.Success && runResult.Results?.length > 0 ? runResult.Results[0] : null);
+                    }
+                    sourceRun = runCache.get(extracted.sourceAgentRunId) || null;
+                }
                 if (extracted.mergeWithExistingId) {
                     const existingNote = await md.GetEntityObject('AI Agent Notes', contextUser);
                     if (await existingNote.Load(extracted.mergeWithExistingId)) {
                         existingNote.Note = extracted.content;
                         existingNote.Type = extracted.type;
-                        await existingNote.Save();
-                        created++;
+                        existingNote.AgentNoteTypeID = aiNoteTypeId;
+                        if (await existingNote.Save()) {
+                            merged++;
+                        }
+                        else {
+                            failed++;
+                        }
+                    }
+                    else {
+                        failed++;
                     }
                 }
                 else {
                     const note = await md.GetEntityObject('AI Agent Notes', contextUser);
-                    note.AgentID = extracted.agentId || null;
-                    note.UserID = extracted.userId || null;
-                    note.CompanyID = extracted.companyId || null;
-                    note.AgentNoteTypeID = extracted.noteTypeId;
+                    const isValidUUID = (id) => {
+                        if (!id)
+                            return false;
+                        return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id);
+                    };
+                    let agentId = isValidUUID(extracted.agentId) ? extracted.agentId : null;
+                    if (!agentId && sourceRun?.AgentID) {
+                        agentId = sourceRun.AgentID;
+                    }
+                    note.AgentID = agentId;
+                    note.UserID = isValidUUID(extracted.userId) ? extracted.userId : null;
+                    note.CompanyID = isValidUUID(extracted.companyId) ? extracted.companyId : null;
+                    note.AgentNoteTypeID = aiNoteTypeId;
                     note.Type = extracted.type;
                     note.Note = extracted.content;
                     note.IsAutoGenerated = true;
                     note.Status = 'Active';
+                    note.AccessCount = 1;
                     note.SourceConversationID = extracted.sourceConversationId || null;
-                    note.SourceConversationDetailID = extracted.sourceConversationDetailId || null;
+                    note.SourceConversationDetailID = isValidUUID(extracted.sourceConversationDetailId) ? extracted.sourceConversationDetailId : null;
                     note.SourceAIAgentRunID = extracted.sourceAgentRunId || null;
-                    if (await note.Save()) {
+                    const scopeLevel = extracted.scopeLevel || 'contact';
+                    if (scopeLevel === 'global') {
+                        note.PrimaryScopeEntityID = null;
+                        note.PrimaryScopeRecordID = null;
+                        note.SecondaryScopes = null;
+                    }
+                    else if (scopeLevel === 'organization' && sourceRun?.PrimaryScopeEntityID) {
+                        note.PrimaryScopeEntityID = sourceRun.PrimaryScopeEntityID;
+                        note.PrimaryScopeRecordID = sourceRun.PrimaryScopeRecordID;
+                        note.SecondaryScopes = null;
+                    }
+                    else if (sourceRun) {
+                        note.PrimaryScopeEntityID = sourceRun.PrimaryScopeEntityID;
+                        note.PrimaryScopeRecordID = sourceRun.PrimaryScopeRecordID;
+                        note.SecondaryScopes = sourceRun.SecondaryScopes;
+                    }
+                    const saveResult = await note.Save();
+                    if (saveResult) {
                         created++;
                     }
+                    else {
+                        failed++;
+                        (0, core_1.LogError)(`Memory Manager: Failed to save note - Validation errors: ${JSON.stringify(note.LatestResult)}`);
+                    }
                 }
             }
             catch (error) {
-                (0, core_1.LogError)('Failed to create note:', error);
+                failed++;
+                (0, core_1.LogError)('Memory Manager: Exception creating note:', error);
             }
         }
-        return created;
+        await this.FinalizeRunStep(step7, failed === 0 || created > 0 || merged > 0, {
+            created,
+            merged,
+            failed
+        });
+        return created + merged;
     }
     async CreateExampleRecords(extractedExamples, contextUser) {
+        const step8 = await this.CreateRunStep('Decision', 'Create Example Records', {
+            exampleCount: extractedExamples.length
+        });
         let created = 0;
+        let skipped = 0;
+        let failed = 0;
         const md = new core_1.Metadata();
+        const rv = new core_1.RunView();
+        const runCache = new Map();
         for (const extracted of extractedExamples) {
             try {
+                let sourceRun = null;
+                if (extracted.sourceAgentRunId) {
+                    if (!runCache.has(extracted.sourceAgentRunId)) {
+                        const runResult = await rv.RunView({
+                            EntityName: 'MJ: AI Agent Runs',
+                            ExtraFilter: `ID='${extracted.sourceAgentRunId}'`,
+                            MaxRows: 1,
+                            ResultType: 'entity_object'
+                        }, contextUser);
+                        runCache.set(extracted.sourceAgentRunId, runResult.Success && runResult.Results?.length > 0 ? runResult.Results[0] : null);
+                    }
+                    sourceRun = runCache.get(extracted.sourceAgentRunId) || null;
+                }
                 const example = await md.GetEntityObject('MJ: AI Agent Examples', contextUser);
-                example.AgentID = extracted.agentId;
+                if (!sourceRun?.AgentID) {
+                    skipped++;
+                    if (this._verbose) {
+                        (0, core_1.LogStatus)(`Memory Manager: Skipping example - no source run to inherit AgentID from`);
+                    }
+                    continue;
+                }
+                example.AgentID = sourceRun.AgentID;
                 example.UserID = extracted.userId || null;
                 example.CompanyID = extracted.companyId || null;
                 example.Type = extracted.type;
@@ -256,26 +708,60 @@ class MemoryManagerAgent extends base_agent_1.BaseAgent {
                 example.SuccessScore = extracted.successScore;
                 example.Status = 'Active';
                 example.SourceConversationID = extracted.sourceConversationId || null;
-                example.SourceConversationDetailID = extracted.sourceConversationDetailId || null;
+                example.SourceConversationDetailID = this.isValidUUID(extracted.sourceConversationDetailId) ? extracted.sourceConversationDetailId : null;
                 example.SourceAIAgentRunID = extracted.sourceAgentRunId || null;
+                if (sourceRun && sourceRun.PrimaryScopeEntityID) {
+                    const scopeLevel = extracted.scopeLevel || 'contact';
+                    if (scopeLevel === 'global') {
+                        example.PrimaryScopeEntityID = null;
+                        example.PrimaryScopeRecordID = null;
+                        example.SecondaryScopes = null;
+                    }
+                    else if (scopeLevel === 'organization') {
+                        example.PrimaryScopeEntityID = sourceRun.PrimaryScopeEntityID;
+                        example.PrimaryScopeRecordID = sourceRun.PrimaryScopeRecordID;
+                        example.SecondaryScopes = null;
+                    }
+                    else {
+                        example.PrimaryScopeEntityID = sourceRun.PrimaryScopeEntityID;
+                        example.PrimaryScopeRecordID = sourceRun.PrimaryScopeRecordID;
+                        example.SecondaryScopes = sourceRun.SecondaryScopes;
+                    }
+                }
                 if (await example.Save()) {
                     created++;
                 }
+                else {
+                    failed++;
+                    (0, core_1.LogError)(`Memory Manager: Failed to save example - Validation errors: ${JSON.stringify(example.LatestResult)}`);
+                }
             }
             catch (error) {
-                (0, core_1.LogError)('Failed to create example:', error);
+                failed++;
+                (0, core_1.LogError)('Memory Manager: Exception creating example:', error);
             }
         }
+        await this.FinalizeRunStep(step8, failed === 0 || created > 0, {
+            created,
+            skipped,
+            failed
+        });
         return created;
     }
     async executeAgentInternal(params, config) {
         try {
+            this._verbose = params.verbose ?? false;
+            this._agentRunID = this.AgentRun?.ID || null;
+            this._stepCounter = 0;
+            this._contextUser = params.contextUser || null;
             (0, core_1.LogStatus)('Memory Manager: Starting analysis cycle');
             const lastRunTime = await this.GetLastRunTime(params.agent.ID, params.contextUser);
-            const sinceMessage = lastRunTime ? `since ${lastRunTime.toISOString()}` : 'all history';
-            (0, core_1.LogStatus)(`Memory Manager: Processing ${sinceMessage}`);
             const agentsUsingMemory = await this.LoadAgentsUsingMemory(params.contextUser);
-            (0, core_1.LogStatus)(`Memory Manager: Found ${agentsUsingMemory.length} agents with memory injection enabled`);
+            if (this._verbose) {
+                const sinceMessage = lastRunTime ? `since ${lastRunTime.toISOString()}` : 'all history';
+                (0, core_1.LogStatus)(`Memory Manager: Processing ${sinceMessage}`);
+                (0, core_1.LogStatus)(`Memory Manager: Found ${agentsUsingMemory.length} agents with memory injection enabled`);
+            }
             if (agentsUsingMemory.length === 0) {
                 const finalStep = {
                     terminate: true,
@@ -284,34 +770,109 @@ class MemoryManagerAgent extends base_agent_1.BaseAgent {
                 };
                 return { finalStep, stepCount: 1 };
             }
-            const conversationDetails = await this.LoadHighQualityConversationDetails(lastRunTime, agentsUsingMemory, params.contextUser);
-            (0, core_1.LogStatus)(`Memory Manager: Found ${conversationDetails.length} high-quality conversation details`);
+            const step1 = await this.CreateRunStep('Decision', 'Load Conversations With New Activity', {
+                since: lastRunTime?.toISOString() || null,
+                agentCount: agentsUsingMemory.length,
+                agentIds: agentsUsingMemory.map(a => a.ID)
+            });
+            const conversations = await this.LoadConversationsWithNewActivity(lastRunTime, agentsUsingMemory, params.contextUser);
+            const totalMessages = conversations.reduce((sum, c) => sum + c.messages.length, 0);
+            await this.FinalizeRunStep(step1, true, {
+                conversationCount: conversations.length,
+                totalMessages,
+                positiveCount: conversations.filter(c => c.hasPositiveRating).length,
+                negativeCount: conversations.filter(c => c.hasNegativeRating).length,
+                unratedCount: conversations.filter(c => c.isUnrated).length
+            });
+            if (this._verbose) {
+                (0, core_1.LogStatus)(`Memory Manager: Found ${conversations.length} conversations with new activity`);
+            }
+            const step2 = await this.CreateRunStep('Decision', 'Load High-Value Agent Runs', {
+                since: lastRunTime?.toISOString() || null
+            });
             const agentRuns = await this.LoadHighValueAgentRuns(lastRunTime, params.contextUser);
-            (0, core_1.LogStatus)(`Memory Manager: Found ${agentRuns.length} high-value agent runs`);
-            if (conversationDetails.length === 0 && agentRuns.length === 0) {
-                (0, core_1.LogStatus)('Memory Manager: No data to process');
+            await this.FinalizeRunStep(step2, true, {
+                runCount: agentRuns.length
+            });
+            if (this._verbose) {
+                (0, core_1.LogStatus)(`Memory Manager: Found ${agentRuns.length} high-value agent runs`);
+            }
+            if (conversations.length === 0 && agentRuns.length === 0) {
+                if (this._verbose) {
+                    (0, core_1.LogStatus)('Memory Manager: No data to process');
+                }
                 const finalStep = {
                     terminate: true,
                     step: 'Success',
-                    message: 'No high-quality conversation details or agent runs to process'
+                    message: 'No conversations with new activity or agent runs to process'
                 };
                 return { finalStep, stepCount: 1 };
             }
-            const extractedNotes = await this.ExtractNotes(conversationDetails, params.contextUser);
-            (0, core_1.LogStatus)(`Memory Manager: Extracted ${extractedNotes.length} potential notes`);
+            const extractedNotes = await this.ExtractNotesFromConversations(conversations, params.contextUser);
+            if (this._verbose) {
+                (0, core_1.LogStatus)(`Memory Manager: Extracted ${extractedNotes.length} potential notes`);
+            }
+            const conversationDetails = conversations.flatMap(conv => conv.messages.map(msg => ({
+                ID: msg.id,
+                ConversationID: conv.conversationId,
+                Role: msg.role,
+                Message: msg.message,
+                Status: 'Complete',
+                __mj_CreatedAt: msg.createdAt
+            })));
             const extractedExamples = await this.ExtractExamples(conversationDetails, params.contextUser);
-            (0, core_1.LogStatus)(`Memory Manager: Extracted ${extractedExamples.length} potential examples`);
+            if (this._verbose) {
+                (0, core_1.LogStatus)(`Memory Manager: Extracted ${extractedExamples.length} potential examples`);
+            }
+            const conversationContextForExamples = new Map();
+            for (const conv of conversations) {
+                conversationContextForExamples.set(conv.conversationId, {
+                    userId: conv.userId || null,
+                    agentRunId: conv.agentRunId || null
+                });
+            }
+            for (const example of extractedExamples) {
+                if (example.sourceConversationId) {
+                    const ctx = conversationContextForExamples.get(example.sourceConversationId);
+                    if (ctx) {
+                        if (!example.userId && ctx.userId) {
+                            example.userId = ctx.userId;
+                            if (this._verbose) {
+                                (0, core_1.LogStatus)(`Memory Manager: Enriched example with userId from conversation: ${ctx.userId}`);
+                            }
+                        }
+                        if (!example.sourceAgentRunId && ctx.agentRunId) {
+                            example.sourceAgentRunId = ctx.agentRunId;
+                            if (this._verbose) {
+                                (0, core_1.LogStatus)(`Memory Manager: Enriched example with agentRunId from conversation: ${ctx.agentRunId}`);
+                            }
+                        }
+                    }
+                }
+                if (example.agentId && !this.isValidUUID(example.agentId)) {
+                    if (this._verbose) {
+                        (0, core_1.LogStatus)(`Memory Manager: Clearing invalid agentId "${example.agentId}" from example`);
+                    }
+                    example.agentId = '';
+                }
+            }
             const notesCreated = await this.CreateNoteRecords(extractedNotes, params.contextUser);
             const examplesCreated = await this.CreateExampleRecords(extractedExamples, params.contextUser);
             (0, core_1.LogStatus)(`Memory Manager: Created ${notesCreated} notes and ${examplesCreated} examples`);
+            if (notesCreated > 0 || examplesCreated > 0) {
+                (0, core_1.LogStatus)(`Memory Manager: Refreshing AIEngine memory cache...`);
+                await aiengine_1.AIEngine.Instance.Config(true, params.contextUser);
+                (0, core_1.LogStatus)(`Memory Manager: Memory cache refreshed`);
+            }
             const finalStep = {
                 terminate: true,
                 step: 'Success',
-                message: `Processed ${conversationDetails.length} conversation details and ${agentRuns.length} agent runs. Created ${notesCreated} notes and ${examplesCreated} examples.`,
+                message: `Processed ${conversations.length} conversations (${totalMessages} messages) and ${agentRuns.length} agent runs. Created ${notesCreated} notes and ${examplesCreated} examples.`,
                 newPayload: {
                     notesCreated,
                     examplesCreated,
-                    conversationDetailsProcessed: conversationDetails.length,
+                    conversationsProcessed: conversations.length,
+                    messagesProcessed: totalMessages,
                     agentRunsProcessed: agentRuns.length
                 }
             };
@@ -327,6 +888,9 @@ class MemoryManagerAgent extends base_agent_1.BaseAgent {
             return { finalStep, stepCount: 1 };
         }
     }
-}
+};
 exports.MemoryManagerAgent = MemoryManagerAgent;
+exports.MemoryManagerAgent = MemoryManagerAgent = __decorate([
+    (0, global_1.RegisterClass)(base_agent_1.BaseAgent, 'MemoryManagerAgent')
+], MemoryManagerAgent);
 //# sourceMappingURL=memory-manager-agent.js.map